diff --git a/api/core/workflow/nodes/knowledge_retrieval_v2/knowledge_retrieval_v2_node.py b/api/core/workflow/nodes/knowledge_retrieval_v2/knowledge_retrieval_v2_node.py index 67d110c825f..a81c4dd65ea 100644 --- a/api/core/workflow/nodes/knowledge_retrieval_v2/knowledge_retrieval_v2_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval_v2/knowledge_retrieval_v2_node.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging import math import time +import uuid from collections.abc import Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager @@ -379,6 +380,7 @@ class KnowledgeRetrievalV2Node(Node[KnowledgeRetrievalV2NodeData]): service = self._service() payload = KnowledgeFSRetrievalTestPayload( query=query, + query_id=uuid.uuid4(), queryImages=list(query_images), mode=self._node_data.mode, include_text=True, diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 0d4f1dea0d9..5961aeb87d8 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,7 +1,7 @@ { "schemaVersion": 5, - "subtreeTree": "1d3eabf30c7ef2aef0383595be622670c2ec56a8", - "openapiSha256": "5f9bee6a3593d8bd05308c7a57ab3d02ef451133cde9347a40543fae934a9a59", + "subtreeTree": "ca8a11624a77a972ea79c9dddd2817f887588cf5", + "openapiSha256": "aab7dfcb8163af398c5e308b7e48f2d7225898d7cedecfaaf677b804fa5572d2", "capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", "productOperationManifestSha256": "5d1241a83bcca12ebbd848928dd3cdda0d2ecaeb5f5336e955eb24a8c8db175b", diff --git a/api/services/knowledge_fs/product_dto.py b/api/services/knowledge_fs/product_dto.py index 4369c3e5051..9b8da401f04 100644 --- a/api/services/knowledge_fs/product_dto.py +++ b/api/services/knowledge_fs/product_dto.py @@ -2993,6 +2993,11 @@ class KnowledgeFSRetrievalMetadataFilters(BaseModel): class KnowledgeFSRetrievalTestPayload(BaseModel): query: str = Field(default="", max_length=16_000) + query_id: UUID | None = Field( + default=None, + validation_alias=AliasChoices("query_id", "queryId"), + serialization_alias="queryId", + ) query_images: list[KnowledgeFSRetrievalQueryImageReference] = Field( default_factory=list, max_length=4, diff --git a/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval_v2/test_knowledge_retrieval_v2_node.py b/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval_v2/test_knowledge_retrieval_v2_node.py index 7dad5ecbb1c..a4d8e5f9d70 100644 --- a/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval_v2/test_knowledge_retrieval_v2_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/knowledge_retrieval_v2/test_knowledge_retrieval_v2_node.py @@ -411,6 +411,9 @@ def test_multi_space_retrieval_uses_one_system_reranker_and_returns_mixed_metric assert rerank_manager.default_calls assert rerank_manager.explicit_calls == [] assert all(call["payload"].include_text is True for call in service.calls) # type: ignore[attr-defined] + query_ids = {call["payload"].query_id for call in service.calls} # type: ignore[attr-defined] + assert len(query_ids) == 1 + assert query_ids.pop().version == 4 # type: ignore[union-attr] assert all( call["payload"].filters.document_types == ["handbook"] # type: ignore[attr-defined] for call in service.calls @@ -1113,8 +1116,9 @@ def test_threshold_filtered_results_are_not_recorded_as_a_raw_retrieval_miss( "enqueue_workflow_failed_retrieval_capture", lambda **kwargs: dispatched.append(kwargs), ) + service = RecordingCapabilityService({"space-a": _response(mode="fast", score=0.91, space="a", text="A")}) result = _node( - service=RecordingCapabilityService({"space-a": _response(mode="fast", score=0.91, space="a", text="A")}), + service=service, spaces=["space-a"], rerank_model_manager=RecordingRerankModelManager(RecordingRerankModel({"A": 0.42})), node_data_overrides={"score_threshold": 0.8}, @@ -1123,6 +1127,9 @@ def test_threshold_filtered_results_are_not_recorded_as_a_raw_retrieval_miss( assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.outputs["result"].value == [] assert result.outputs["metrics"].value["workflow_rerank"]["output_count"] == 0 + assert "scoreThreshold" not in service.calls[0]["payload"].model_dump( # type: ignore[attr-defined] + mode="json", by_alias=True + ) assert dispatched == [] diff --git a/api/tests/unit_tests/services/test_knowledge_fs_app_execution_capability.py b/api/tests/unit_tests/services/test_knowledge_fs_app_execution_capability.py index 2ef874cd639..376ce7b13b8 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_app_execution_capability.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_app_execution_capability.py @@ -328,6 +328,7 @@ def test_run_retrieval_issues_read_capability_and_calls_bounded_product_operatio "includeText": True, "mode": "fast", "query": "camera", + "queryId": "10000000-0000-4000-8000-000000000010", "queryImages": [ { "accessGrant": "short-lived-grant", @@ -347,6 +348,7 @@ def test_run_retrieval_issues_read_capability_and_calls_bounded_product_operatio "includeText": True, "mode": "fast", "query": "camera", + "queryId": "10000000-0000-4000-8000-000000000010", "queryImages": [{"uploadFileId": "00000000-0000-4000-8000-000000000001"}], } assert len(request.headers) == 1 diff --git a/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py b/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py index 4adfad5b495..d870aa9628a 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py @@ -429,6 +429,7 @@ def test_connector_initial_source_requires_an_exact_credential_binding() -> None def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() -> None: + query_id = "10000000-0000-4000-8000-000000000010" payload = KnowledgeFSRetrievalTestPayload.model_validate( { "filters": { @@ -439,6 +440,7 @@ def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() -> "includeText": True, "mode": "deep", "query": " camera evidence ", + "queryId": query_id, } ) @@ -457,11 +459,14 @@ def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() -> "includeText": True, "mode": "deep", "query": "camera evidence", + "queryId": query_id, } with pytest.raises(ValidationError): KnowledgeFSRetrievalTestPayload(query="camera", mode="auto") # type: ignore[arg-type] with pytest.raises(ValidationError): KnowledgeFSRetrievalMetadataFilters(tags=[f"tag-{index}" for index in range(101)]) + with pytest.raises(ValidationError): + KnowledgeFSRetrievalTestPayload(query="camera", queryId="not-a-uuid") def test_retrieval_test_payload_accepts_transient_workflow_image_grants() -> None: diff --git a/knowledge-fs/.harness/changes/2026-09-04-workflow-query-overview.md b/knowledge-fs/.harness/changes/2026-09-04-workflow-query-overview.md new file mode 100644 index 00000000000..1af2d638d35 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-04-workflow-query-overview.md @@ -0,0 +1,52 @@ +# Count Workflow retrievals in Knowledge Space Overview + +## Why + +The Knowledge Retrieval v2 node called the published retrieval-test operation directly. That path +returned evidence to the Workflow but did not create the durable `query.requested` and AnswerTrace +facts consumed by Knowledge Space Overview, so App/Workflow traffic was absent from query counts +and outcome charts. + +## What changed + +- Each Knowledge Retrieval v2 node execution now creates one UUID workflow query identity and sends + it to every selected KnowledgeFS space. KnowledgeFS derives a separate deterministic AnswerTrace + ID for each space and keeps the workflow query ID in terminal metadata for cross-space linkage. +- Capability v2 retrieval-test requests from a `workflow` caller persist a `query.requested` + activity and a terminal `query.generate` AnswerTrace under that identity. +- Outcome classification uses only the selected space's published retrieval profile and its + mode-final score threshold. The Workflow node's optional post-retrieval threshold is not sent to + KnowledgeFS and does not change the Overview outcome. +- Interactive retrieval tests and legacy Workflow requests without the new business query identity + remain excluded, preventing manual tests or whole-run transport trace collisions from inflating + the dashboard. +- Pure-image requests against text-only spaces stay on the existing retrieval path but are excluded + from Overview because there is no traceable text or resolved image payload to persist. + +## Verification + +- Test-first regressions failed before the request contract and persistence path were implemented. +- Focused retrieval handler suite: 17 passed, including a shared workflow query across two spaces + using the real recorder and in-memory repository, plus the text-only pure-image compatibility case. +- KnowledgeFS API full suite: 5,020 passed, 3 skipped. +- Dify Knowledge Retrieval v2 node, KnowledgeFS DTO, and App execution capability: 124 passed. +- KnowledgeFS contract generator: 36 passed; the reviewed subtree/OpenAPI lock was regenerated and + checked with a temporary Git index. +- `pnpm --filter @knowledge/api typecheck`, `pnpm openapi:export:test`, focused Biome, Ruff, Pyrefly, + and `git diff --check` passed. +- Browser acceptance ran the `KR v2 Reset verification` Workflow against control space + `01a0556a-8e43-7e60-8be5-69fe8228f825`. The successful request persisted activity and terminal + trace `2ecedddd-87c0-4d87-9231-74e81d4a77fd` as `answered`; the 24-hour Overview visibly moved + from 6 to 7 queries and showed the `dify-app` request in Recent activity. +- The App-threshold boundary has automated coverage: a Workflow threshold-filtered output does not + send `scoreThreshold` to KnowledgeFS, while the KFS handler classifies from its published profile. + A second browser attempt with a temporary App threshold of `0.99` reached KFS but the local model + runtime returned unavailable before a terminal trace; the temporary App setting was restored. +- Final runtime health and queue checks are recorded before handoff. + +## Risks and follow-up + +- Historical Workflow runs are not backfilled; the new query identity is available only after both + Dify and KnowledgeFS contain this contract change. +- Failed retrieval execution still leaves the durable request as unanswered, matching the existing + Overview query lifecycle rather than fabricating a no-evidence terminal result. diff --git a/knowledge-fs/packages/api/src/index.ts b/knowledge-fs/packages/api/src/index.ts index 3ffb5bffa62..3f3dc8ff23c 100644 --- a/knowledge-fs/packages/api/src/index.ts +++ b/knowledge-fs/packages/api/src/index.ts @@ -1916,9 +1916,11 @@ export function createKnowledgeGateway({ }); registerRetrievalTestHandlers({ + answerTraceRecorder, app, ...(retrievalTestExecutor ? { executor: retrievalTestExecutor } : {}), ...(modelInputModalityResolver ? { modelInputModalityResolver } : {}), + overview: overviewRepository, ...(queryImageResolver ? { queryImageResolver } : {}), ...(retrievalExecutionLeases ? { retrievalExecutionLeases } : {}), ...(runtimeSnapshotResolver ? { runtimeSnapshotResolver } : {}), diff --git a/knowledge-fs/packages/api/src/retrieval-test-handlers.test.ts b/knowledge-fs/packages/api/src/retrieval-test-handlers.test.ts index 4bfeaa633ed..bcc9b421fc9 100644 --- a/knowledge-fs/packages/api/src/retrieval-test-handlers.test.ts +++ b/knowledge-fs/packages/api/src/retrieval-test-handlers.test.ts @@ -1,10 +1,19 @@ +import { OpenAPIHono } from "@hono/zod-openapi"; import { createNodePlatformAdapter } from "@knowledge/adapters/node"; import type { KnowledgeSpaceRetrievalProfile } from "@knowledge/core"; import { describe, expect, it, vi } from "vitest"; +import { type AnswerTraceRecorder, createAnswerTraceRecorder } from "./answer-trace-recorder"; +import { createInMemoryAnswerTraceRepository } from "./answer-trace-repository"; import { createStaticAuthVerifier } from "./auth"; +import type { DifyCapabilityV2SanitizedGrant } from "./dify-capability-v2-grant"; +import type { KnowledgeGatewayEnv } from "./gateway-openapi-contracts"; import type { QueryGenerator } from "./gateway-sse-responses"; import { createKnowledgeGateway } from "./index"; +import { + type KnowledgeSpaceOverviewRepository, + deterministicKnowledgeSpaceActivityId, +} from "./knowledge-space-overview"; import { createInMemoryKnowledgeSpaceRepository } from "./knowledge-space-repository"; import type { PublishedKnowledgeSpaceRuntimeSnapshot } from "./published-knowledge-space-runtime-snapshot"; import { KNOWLEDGE_FS_QUERY_IMAGE_GRANTS_HEADER } from "./query-images"; @@ -14,6 +23,7 @@ import { type RetrievalTestResult, createRetrievalTestExecutor, } from "./retrieval-test"; +import { registerRetrievalTestHandlers } from "./retrieval-test-handlers"; import { RetrievalTestMetricsSchema, RetrievalTestRequestSchema, @@ -21,7 +31,9 @@ import { } from "./retrieval-test-routes"; const SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42"; +const SECOND_SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44"; const TOKEN = "owner-token"; +const WORKFLOW_QUERY_ID = "10000000-0000-4000-8000-000000000010"; const reasoningSelection = { model: "reasoning-1", pluginId: "plugin/reasoning", @@ -47,6 +59,161 @@ const retrievalProfile: KnowledgeSpaceRetrievalProfile = { }; describe("retrieval test route", () => { + it.each([ + { + expectedOutcome: "answered", + result: retrievalResult("fast"), + }, + { + expectedOutcome: "low-confidence", + result: emptyRetrievalResult({ scoreThresholdFilteredCandidates: 2 }), + }, + { + expectedOutcome: "no-evidence", + result: emptyRetrievalResult(), + }, + ] as const)( + "records workflow retrieval as $expectedOutcome for Overview", + async ({ expectedOutcome, result }) => { + const { answerTraceRecorder, app, overview } = workflowTelemetryApp(result); + const expectedTraceId = deterministicKnowledgeSpaceActivityId( + "workflow.answer-trace", + "tenant-1", + SPACE_ID, + WORKFLOW_QUERY_ID, + ); + + const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, { + body: JSON.stringify({ query: "workflow camera query", queryId: WORKFLOW_QUERY_ID }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + + expect(response.status).toBe(200); + expect(overview.appendActivity).toHaveBeenCalledWith( + expect.objectContaining({ + action: "query.requested", + details: expect.objectContaining({ mode: "fast" }), + resource: { id: expectedTraceId, type: "query" }, + }), + ); + expect(answerTraceRecorder.record).toHaveBeenCalledWith( + expect.objectContaining({ + capabilityGrantId: "10000000-0000-4000-8000-000000000011", + traceId: expectedTraceId, + steps: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + queryOutcome: expectedOutcome, + workflowQueryId: WORKFLOW_QUERY_ID, + }), + name: "query.generate", + status: "ok", + }), + ], + }), + ); + }, + ); + + it("keeps interactive retrieval tests out of Overview query traffic", async () => { + const { answerTraceRecorder, app, overview } = workflowTelemetryApp(retrievalResult("fast"), { + callerKind: "interactive", + }); + + const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, { + body: JSON.stringify({ query: "manual camera query", queryId: WORKFLOW_QUERY_ID }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + + expect(response.status).toBe(200); + expect(overview.appendActivity).not.toHaveBeenCalled(); + expect(answerTraceRecorder.record).not.toHaveBeenCalled(); + }); + + it("keeps legacy workflow requests without a business query id out of Overview", async () => { + const { answerTraceRecorder, app, overview } = workflowTelemetryApp(retrievalResult("fast")); + + const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, { + body: JSON.stringify({ query: "legacy workflow camera query" }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + + expect(response.status).toBe(200); + expect(overview.appendActivity).not.toHaveBeenCalled(); + expect(answerTraceRecorder.record).not.toHaveBeenCalled(); + }); + + it("persists one trace per space for a shared workflow query id", async () => { + const answerTraces = createInMemoryAnswerTraceRepository({ maxSteps: 10, maxTraces: 10 }); + const answerTraceRecorder = createAnswerTraceRecorder({ + now: () => "2026-09-04T08:00:00.000Z", + repository: answerTraces, + }); + const overview = { + appendActivity: vi.fn(async (input) => input as never), + } satisfies Pick; + const first = workflowTelemetryApp(retrievalResult("fast"), { + answerTraceRecorder, + overview, + spaceId: SPACE_ID, + }); + const second = workflowTelemetryApp(retrievalResult("fast"), { + answerTraceRecorder, + grantId: "10000000-0000-4000-8000-000000000012", + overview, + spaceId: SECOND_SPACE_ID, + }); + + const responses = await Promise.all( + [ + { app: first.app, spaceId: SPACE_ID }, + { app: second.app, spaceId: SECOND_SPACE_ID }, + ].map(({ app, spaceId }) => + app.request(`/knowledge-spaces/${spaceId}/retrieval-tests`, { + body: JSON.stringify({ query: "shared workflow query", queryId: WORKFLOW_QUERY_ID }), + headers: { "content-type": "application/json" }, + method: "POST", + }), + ), + ); + + expect(responses.map((response) => response.status)).toEqual([200, 200]); + const traceIds = overview.appendActivity.mock.calls.map( + ([activity]) => activity.resource.id as string, + ); + expect(new Set(traceIds).size).toBe(2); + const traces = await Promise.all([ + answerTraces.get({ id: traceIds[0] as string, knowledgeSpaceId: SPACE_ID }), + answerTraces.get({ id: traceIds[1] as string, knowledgeSpaceId: SECOND_SPACE_ID }), + ]); + expect(traces.map((trace) => trace?.knowledgeSpaceId)).toEqual([SPACE_ID, SECOND_SPACE_ID]); + for (const trace of traces) { + expect(trace?.steps[0]?.metadata).toMatchObject({ workflowQueryId: WORKFLOW_QUERY_ID }); + } + }); + + it("keeps pure-image workflow requests on text-only spaces out of Overview", async () => { + const { answerTraceRecorder, app, overview } = workflowTelemetryApp(retrievalResult("fast"), { + modelInputModalityResolver: { resolve: async () => ["text"] }, + }); + + const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, { + body: JSON.stringify({ + queryId: WORKFLOW_QUERY_ID, + queryImages: [{ uploadFileId: "00000000-0000-4000-8000-000000000001" }], + }), + headers: { "content-type": "application/json" }, + method: "POST", + }); + + expect(response.status).toBe(200); + expect(overview.appendActivity).not.toHaveBeenCalled(); + expect(answerTraceRecorder.record).not.toHaveBeenCalled(); + }); + it("resolves workflow-granted images and freezes each space's model modalities", async () => { const image = { body: new Uint8Array([1, 2, 3]), @@ -775,6 +942,115 @@ function researchMetrics() { }; } +function emptyRetrievalResult( + metrics: { readonly scoreThresholdFilteredCandidates?: number } = {}, +): RetrievalTestResult { + return { + ...retrievalResult("fast"), + items: [], + metrics: { + ...retrievalResult("fast").metrics, + ...metrics, + }, + }; +} + +function workflowTelemetryApp( + result: RetrievalTestResult, + options: { + readonly answerTraceRecorder?: AnswerTraceRecorder; + readonly callerKind?: DifyCapabilityV2SanitizedGrant["callerKind"]; + readonly grantId?: string; + readonly modelInputModalityResolver?: Parameters< + typeof registerRetrievalTestHandlers + >[0]["modelInputModalityResolver"]; + readonly overview?: Pick; + readonly spaceId?: string; + } = {}, +) { + const spaceId = options.spaceId ?? SPACE_ID; + const app = new OpenAPIHono(); + const overview = + options.overview ?? + ({ + appendActivity: vi.fn(async (input) => input as never), + } satisfies Pick); + const answerTraceRecorder = + options.answerTraceRecorder ?? + ({ + record: vi.fn(async (input) => input as never), + } satisfies AnswerTraceRecorder); + const grant: DifyCapabilityV2SanitizedGrant = { + action: "queries.retrieval_test", + actor: "dify-app:app-1", + authzRevision: { + credential_revision: null, + external_access_epoch: 1, + membership_epoch: 1, + space_acl_epoch: 1, + }, + azp: "app-1", + callerKind: options.callerKind ?? "workflow", + capVersion: 2, + contentPolicyRevision: 1, + contentScopeIds: [`knowledge-space:${spaceId}`], + controlSpaceId: "control-space-1", + expiresAt: 9_999_999_999, + grantId: options.grantId ?? "10000000-0000-4000-8000-000000000011", + issuedAt: 1, + jtiHash: "hash", + namespaceId: "tenant-1", + notBefore: 1, + resource: { id: spaceId, parent_id: null, type: "knowledge_space" }, + subject: "dify-app:app-1", + traceId: "workflow-run-1", + }; + app.use("*", async (context, next) => { + context.set("subject", { + scopes: [], + subjectId: "dify-app:app-1", + tenantId: "tenant-1", + }); + context.set("capabilityV2Grant", grant); + context.set("traceId", "transport-trace-1"); + await next(); + }); + registerRetrievalTestHandlers({ + answerTraceRecorder, + app, + executor: { execute: async () => result }, + ...(options.modelInputModalityResolver + ? { modelInputModalityResolver: options.modelInputModalityResolver } + : {}), + overview, + retrievalExecutionLeases: { + acquire: async () => ({ + assertActive: async () => undefined, + release: async () => undefined, + signal: new AbortController().signal, + }), + }, + runtimeSnapshotResolver: { + assertReady: async () => undefined, + resolve: async () => ({ + ...runtimeSnapshot(), + projectionSnapshot: { + ...runtimeSnapshot().projectionSnapshot, + knowledgeSpaceId: spaceId, + }, + retrievalProfile: { + ...retrievalProfile, + scoreThreshold: { enabled: true, stage: "mode-final", value: 0.7 }, + }, + }), + }, + spaces: { + get: async () => ({ id: spaceId, tenantId: "tenant-1" }) as never, + }, + }); + return { answerTraceRecorder, app, overview }; +} + function capability( kind: "embedding" | "reasoning" | "rerank", selection: typeof embeddingSelection | typeof reasoningSelection | typeof rerankSelection, diff --git a/knowledge-fs/packages/api/src/retrieval-test-handlers.ts b/knowledge-fs/packages/api/src/retrieval-test-handlers.ts index 00e10cc14a0..91d2808e244 100644 --- a/knowledge-fs/packages/api/src/retrieval-test-handlers.ts +++ b/knowledge-fs/packages/api/src/retrieval-test-handlers.ts @@ -1,8 +1,15 @@ import type { OpenAPIHono } from "@hono/zod-openapi"; import { validateKnowledgeSpaceRetrievalProfileForMode } from "@knowledge/core"; +import type { AnswerTraceRecorder } from "./answer-trace-recorder"; import { currentCandidateGrants } from "./candidate-content-authorization"; +import { classifyQueryOutcome } from "./failed-query-recorder"; import type { KnowledgeGatewayEnv } from "./gateway-openapi-contracts"; +import { queryRetrievalProfileMetadata } from "./gateway-sse-responses"; +import { + type KnowledgeSpaceOverviewRepository, + deterministicKnowledgeSpaceActivityId, +} from "./knowledge-space-overview"; import type { KnowledgeSpaceRepository } from "./knowledge-space-repository"; import { ModelCapabilitySnapshotSchema, @@ -16,6 +23,7 @@ import { QueryImageResolutionError, type QueryImageResolutionReference, type QueryImageResolver, + queryImageMetadata, queryImageResolutionReferencesFromHeader, } from "./query-images"; import { @@ -34,9 +42,11 @@ import { RetrievalTestResponseSchema, runRetrievalTestRoute } from "./retrieval- const RETRIEVAL_TEST_UNAVAILABLE = "Published retrieval test is unavailable"; export interface RegisterRetrievalTestHandlersOptions { + readonly answerTraceRecorder?: AnswerTraceRecorder | undefined; readonly app: OpenAPIHono; readonly executor?: RetrievalTestExecutor | undefined; readonly modelInputModalityResolver?: ModelInputModalityResolver | undefined; + readonly overview?: Pick | undefined; readonly queryImageResolver?: QueryImageResolver | undefined; readonly retrievalExecutionLeases?: RetrievalExecutionLeaseCoordinator | undefined; readonly runtimeSnapshotResolver?: PublishedKnowledgeSpaceRuntimeSnapshotResolver | undefined; @@ -44,9 +54,11 @@ export interface RegisterRetrievalTestHandlersOptions { } export function registerRetrievalTestHandlers({ + answerTraceRecorder, app, executor, modelInputModalityResolver, + overview, queryImageResolver, retrievalExecutionLeases, runtimeSnapshotResolver, @@ -56,6 +68,8 @@ export function registerRetrievalTestHandlers({ const subject = context.get("subject"); const knowledgeSpaceId = context.req.valid("param").id; const body = context.req.valid("json"); + const capabilityGrant = context.get("capabilityV2Grant"); + const workflowQueryId = capabilityGrant?.callerKind === "workflow" ? body.queryId : undefined; const space = await spaces.get({ id: knowledgeSpaceId, tenantId: subject.tenantId }); if (!space) { return context.json({ error: "Knowledge space not found" }, 404); @@ -202,6 +216,39 @@ export function registerRetrievalTestHandlers({ } } + const workflowAnswerTraceId = + workflowQueryId && (body.query || resolvedQueryImages.length > 0) + ? deterministicKnowledgeSpaceActivityId( + "workflow.answer-trace", + subject.tenantId, + knowledgeSpaceId, + workflowQueryId, + ) + : undefined; + if (workflowAnswerTraceId && answerTraceRecorder && overview) { + const occurredAt = new Date().toISOString(); + await overview.appendActivity({ + action: "query.requested", + actor: { id: subject.subjectId, type: "member" }, + details: { + mode, + ...(body.query ? { question: body.query } : {}), + }, + id: deterministicKnowledgeSpaceActivityId( + "query.requested", + subject.tenantId, + knowledgeSpaceId, + workflowAnswerTraceId, + ), + knowledgeSpaceId, + occurredAt, + requiredPermissionScope: [], + resource: { id: workflowAnswerTraceId, type: "query" }, + result: "success", + tenantId: subject.tenantId, + }); + } + const result = await executor.execute({ ...(runtimeSnapshot.embeddingProfile ? { embeddingProfile: runtimeSnapshot.embeddingProfile } @@ -227,6 +274,52 @@ export function registerRetrievalTestHandlers({ traceId, }); await executionLease.assertActive(); + if (workflowAnswerTraceId && capabilityGrant && answerTraceRecorder && overview) { + const finishReason = + result.items.length > 0 ? "retrieval-evidence" : "no-retrieval-evidence"; + const outcomeMetadata = { + finishReason, + ...(result.items[0]?.score !== undefined ? { topScore: result.items[0].score } : {}), + metrics: { + scoreThresholdFilteredCandidates: + result.metrics.scoreThresholdFilteredCandidates ?? 0, + }, + retrievalProfile: queryRetrievalProfileMetadata(runtimeSnapshot.retrievalProfile), + source: "workflow", + workflowQueryId, + }; + const classification = classifyQueryOutcome({ + finishReason, + metadata: outcomeMetadata, + ...(runtimeSnapshot.retrievalProfile.scoreThreshold.enabled + ? { + lowConfidenceScoreFloor: runtimeSnapshot.retrievalProfile.scoreThreshold.value, + } + : {}), + }); + await answerTraceRecorder.record({ + capabilityGrantId: capabilityGrant.grantId, + knowledgeSpaceId, + mode, + query: body.query, + ...(resolvedQueryImages.length > 0 + ? { queryImages: resolvedQueryImages.map(queryImageMetadata) } + : {}), + steps: [ + { + metadata: { + ...outcomeMetadata, + queryOutcome: classification.outcome, + ...(classification.trigger ? { failedQueryTrigger: classification.trigger } : {}), + }, + name: "query.generate", + status: "ok", + }, + ], + tenantId: subject.tenantId, + traceId: workflowAnswerTraceId, + }); + } const embeddingCapabilityStatus = "verified" as const; const rerankCapabilityStatus: "disabled" | "verified" = runtimeSnapshot.retrievalProfile .rerank.enabled diff --git a/knowledge-fs/packages/api/src/retrieval-test-routes.ts b/knowledge-fs/packages/api/src/retrieval-test-routes.ts index 1dc041e5253..76c4e3fabd7 100644 --- a/knowledge-fs/packages/api/src/retrieval-test-routes.ts +++ b/knowledge-fs/packages/api/src/retrieval-test-routes.ts @@ -107,6 +107,7 @@ export const RetrievalTestRequestSchema = z includeText: z.boolean().default(false), mode: KnowledgeSpaceRetrievalModeSchema.optional(), query: RetrievalQuerySchema.default(""), + queryId: z.string().uuid().optional(), queryImages: QueryImageReferencesSchema.default([]), }) .strict()