mirror of
https://github.com/langgenius/dify.git
synced 2026-09-05 08:48:10 +08:00
fix(knowledge-fs): track workflow retrieval outcomes
This commit is contained in:
Stephen Zhou
parent
7fba0afc91
commit
876aab708e
@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
@ -379,6 +380,7 @@ class KnowledgeRetrievalV2Node(Node[KnowledgeRetrievalV2NodeData]):
|
||||
service = self._service()
|
||||
payload = KnowledgeFSRetrievalTestPayload(
|
||||
query=query,
|
||||
query_id=uuid.uuid4(),
|
||||
queryImages=list(query_images),
|
||||
mode=self._node_data.mode,
|
||||
include_text=True,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"schemaVersion": 5,
|
||||
"subtreeTree": "1d3eabf30c7ef2aef0383595be622670c2ec56a8",
|
||||
"openapiSha256": "5f9bee6a3593d8bd05308c7a57ab3d02ef451133cde9347a40543fae934a9a59",
|
||||
"subtreeTree": "ca8a11624a77a972ea79c9dddd2817f887588cf5",
|
||||
"openapiSha256": "aab7dfcb8163af398c5e308b7e48f2d7225898d7cedecfaaf677b804fa5572d2",
|
||||
"capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109",
|
||||
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
|
||||
"productOperationManifestSha256": "5d1241a83bcca12ebbd848928dd3cdda0d2ecaeb5f5336e955eb24a8c8db175b",
|
||||
|
||||
@ -2993,6 +2993,11 @@ class KnowledgeFSRetrievalMetadataFilters(BaseModel):
|
||||
|
||||
class KnowledgeFSRetrievalTestPayload(BaseModel):
|
||||
query: str = Field(default="", max_length=16_000)
|
||||
query_id: UUID | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("query_id", "queryId"),
|
||||
serialization_alias="queryId",
|
||||
)
|
||||
query_images: list[KnowledgeFSRetrievalQueryImageReference] = Field(
|
||||
default_factory=list,
|
||||
max_length=4,
|
||||
|
||||
@ -411,6 +411,9 @@ def test_multi_space_retrieval_uses_one_system_reranker_and_returns_mixed_metric
|
||||
assert rerank_manager.default_calls
|
||||
assert rerank_manager.explicit_calls == []
|
||||
assert all(call["payload"].include_text is True for call in service.calls) # type: ignore[attr-defined]
|
||||
query_ids = {call["payload"].query_id for call in service.calls} # type: ignore[attr-defined]
|
||||
assert len(query_ids) == 1
|
||||
assert query_ids.pop().version == 4 # type: ignore[union-attr]
|
||||
assert all(
|
||||
call["payload"].filters.document_types == ["handbook"] # type: ignore[attr-defined]
|
||||
for call in service.calls
|
||||
@ -1113,8 +1116,9 @@ def test_threshold_filtered_results_are_not_recorded_as_a_raw_retrieval_miss(
|
||||
"enqueue_workflow_failed_retrieval_capture",
|
||||
lambda **kwargs: dispatched.append(kwargs),
|
||||
)
|
||||
service = RecordingCapabilityService({"space-a": _response(mode="fast", score=0.91, space="a", text="A")})
|
||||
result = _node(
|
||||
service=RecordingCapabilityService({"space-a": _response(mode="fast", score=0.91, space="a", text="A")}),
|
||||
service=service,
|
||||
spaces=["space-a"],
|
||||
rerank_model_manager=RecordingRerankModelManager(RecordingRerankModel({"A": 0.42})),
|
||||
node_data_overrides={"score_threshold": 0.8},
|
||||
@ -1123,6 +1127,9 @@ def test_threshold_filtered_results_are_not_recorded_as_a_raw_retrieval_miss(
|
||||
assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED
|
||||
assert result.outputs["result"].value == []
|
||||
assert result.outputs["metrics"].value["workflow_rerank"]["output_count"] == 0
|
||||
assert "scoreThreshold" not in service.calls[0]["payload"].model_dump( # type: ignore[attr-defined]
|
||||
mode="json", by_alias=True
|
||||
)
|
||||
assert dispatched == []
|
||||
|
||||
|
||||
|
||||
@ -328,6 +328,7 @@ def test_run_retrieval_issues_read_capability_and_calls_bounded_product_operatio
|
||||
"includeText": True,
|
||||
"mode": "fast",
|
||||
"query": "camera",
|
||||
"queryId": "10000000-0000-4000-8000-000000000010",
|
||||
"queryImages": [
|
||||
{
|
||||
"accessGrant": "short-lived-grant",
|
||||
@ -347,6 +348,7 @@ def test_run_retrieval_issues_read_capability_and_calls_bounded_product_operatio
|
||||
"includeText": True,
|
||||
"mode": "fast",
|
||||
"query": "camera",
|
||||
"queryId": "10000000-0000-4000-8000-000000000010",
|
||||
"queryImages": [{"uploadFileId": "00000000-0000-4000-8000-000000000001"}],
|
||||
}
|
||||
assert len(request.headers) == 1
|
||||
|
||||
@ -429,6 +429,7 @@ def test_connector_initial_source_requires_an_exact_credential_binding() -> None
|
||||
|
||||
|
||||
def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() -> None:
|
||||
query_id = "10000000-0000-4000-8000-000000000010"
|
||||
payload = KnowledgeFSRetrievalTestPayload.model_validate(
|
||||
{
|
||||
"filters": {
|
||||
@ -439,6 +440,7 @@ def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() ->
|
||||
"includeText": True,
|
||||
"mode": "deep",
|
||||
"query": " camera evidence ",
|
||||
"queryId": query_id,
|
||||
}
|
||||
)
|
||||
|
||||
@ -457,11 +459,14 @@ def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() ->
|
||||
"includeText": True,
|
||||
"mode": "deep",
|
||||
"query": "camera evidence",
|
||||
"queryId": query_id,
|
||||
}
|
||||
with pytest.raises(ValidationError):
|
||||
KnowledgeFSRetrievalTestPayload(query="camera", mode="auto") # type: ignore[arg-type]
|
||||
with pytest.raises(ValidationError):
|
||||
KnowledgeFSRetrievalMetadataFilters(tags=[f"tag-{index}" for index in range(101)])
|
||||
with pytest.raises(ValidationError):
|
||||
KnowledgeFSRetrievalTestPayload(query="camera", queryId="not-a-uuid")
|
||||
|
||||
|
||||
def test_retrieval_test_payload_accepts_transient_workflow_image_grants() -> None:
|
||||
|
||||
@ -0,0 +1,52 @@
|
||||
# Count Workflow retrievals in Knowledge Space Overview
|
||||
|
||||
## Why
|
||||
|
||||
The Knowledge Retrieval v2 node called the published retrieval-test operation directly. That path
|
||||
returned evidence to the Workflow but did not create the durable `query.requested` and AnswerTrace
|
||||
facts consumed by Knowledge Space Overview, so App/Workflow traffic was absent from query counts
|
||||
and outcome charts.
|
||||
|
||||
## What changed
|
||||
|
||||
- Each Knowledge Retrieval v2 node execution now creates one UUID workflow query identity and sends
|
||||
it to every selected KnowledgeFS space. KnowledgeFS derives a separate deterministic AnswerTrace
|
||||
ID for each space and keeps the workflow query ID in terminal metadata for cross-space linkage.
|
||||
- Capability v2 retrieval-test requests from a `workflow` caller persist a `query.requested`
|
||||
activity and a terminal `query.generate` AnswerTrace under that identity.
|
||||
- Outcome classification uses only the selected space's published retrieval profile and its
|
||||
mode-final score threshold. The Workflow node's optional post-retrieval threshold is not sent to
|
||||
KnowledgeFS and does not change the Overview outcome.
|
||||
- Interactive retrieval tests and legacy Workflow requests without the new business query identity
|
||||
remain excluded, preventing manual tests or whole-run transport trace collisions from inflating
|
||||
the dashboard.
|
||||
- Pure-image requests against text-only spaces stay on the existing retrieval path but are excluded
|
||||
from Overview because there is no traceable text or resolved image payload to persist.
|
||||
|
||||
## Verification
|
||||
|
||||
- Test-first regressions failed before the request contract and persistence path were implemented.
|
||||
- Focused retrieval handler suite: 17 passed, including a shared workflow query across two spaces
|
||||
using the real recorder and in-memory repository, plus the text-only pure-image compatibility case.
|
||||
- KnowledgeFS API full suite: 5,020 passed, 3 skipped.
|
||||
- Dify Knowledge Retrieval v2 node, KnowledgeFS DTO, and App execution capability: 124 passed.
|
||||
- KnowledgeFS contract generator: 36 passed; the reviewed subtree/OpenAPI lock was regenerated and
|
||||
checked with a temporary Git index.
|
||||
- `pnpm --filter @knowledge/api typecheck`, `pnpm openapi:export:test`, focused Biome, Ruff, Pyrefly,
|
||||
and `git diff --check` passed.
|
||||
- Browser acceptance ran the `KR v2 Reset verification` Workflow against control space
|
||||
`01a0556a-8e43-7e60-8be5-69fe8228f825`. The successful request persisted activity and terminal
|
||||
trace `2ecedddd-87c0-4d87-9231-74e81d4a77fd` as `answered`; the 24-hour Overview visibly moved
|
||||
from 6 to 7 queries and showed the `dify-app` request in Recent activity.
|
||||
- The App-threshold boundary has automated coverage: a Workflow threshold-filtered output does not
|
||||
send `scoreThreshold` to KnowledgeFS, while the KFS handler classifies from its published profile.
|
||||
A second browser attempt with a temporary App threshold of `0.99` reached KFS but the local model
|
||||
runtime returned unavailable before a terminal trace; the temporary App setting was restored.
|
||||
- Final runtime health and queue checks are recorded before handoff.
|
||||
|
||||
## Risks and follow-up
|
||||
|
||||
- Historical Workflow runs are not backfilled; the new query identity is available only after both
|
||||
Dify and KnowledgeFS contain this contract change.
|
||||
- Failed retrieval execution still leaves the durable request as unanswered, matching the existing
|
||||
Overview query lifecycle rather than fabricating a no-evidence terminal result.
|
||||
@ -1916,9 +1916,11 @@ export function createKnowledgeGateway({
|
||||
});
|
||||
|
||||
registerRetrievalTestHandlers({
|
||||
answerTraceRecorder,
|
||||
app,
|
||||
...(retrievalTestExecutor ? { executor: retrievalTestExecutor } : {}),
|
||||
...(modelInputModalityResolver ? { modelInputModalityResolver } : {}),
|
||||
overview: overviewRepository,
|
||||
...(queryImageResolver ? { queryImageResolver } : {}),
|
||||
...(retrievalExecutionLeases ? { retrievalExecutionLeases } : {}),
|
||||
...(runtimeSnapshotResolver ? { runtimeSnapshotResolver } : {}),
|
||||
|
||||
@ -1,10 +1,19 @@
|
||||
import { OpenAPIHono } from "@hono/zod-openapi";
|
||||
import { createNodePlatformAdapter } from "@knowledge/adapters/node";
|
||||
import type { KnowledgeSpaceRetrievalProfile } from "@knowledge/core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { type AnswerTraceRecorder, createAnswerTraceRecorder } from "./answer-trace-recorder";
|
||||
import { createInMemoryAnswerTraceRepository } from "./answer-trace-repository";
|
||||
import { createStaticAuthVerifier } from "./auth";
|
||||
import type { DifyCapabilityV2SanitizedGrant } from "./dify-capability-v2-grant";
|
||||
import type { KnowledgeGatewayEnv } from "./gateway-openapi-contracts";
|
||||
import type { QueryGenerator } from "./gateway-sse-responses";
|
||||
import { createKnowledgeGateway } from "./index";
|
||||
import {
|
||||
type KnowledgeSpaceOverviewRepository,
|
||||
deterministicKnowledgeSpaceActivityId,
|
||||
} from "./knowledge-space-overview";
|
||||
import { createInMemoryKnowledgeSpaceRepository } from "./knowledge-space-repository";
|
||||
import type { PublishedKnowledgeSpaceRuntimeSnapshot } from "./published-knowledge-space-runtime-snapshot";
|
||||
import { KNOWLEDGE_FS_QUERY_IMAGE_GRANTS_HEADER } from "./query-images";
|
||||
@ -14,6 +23,7 @@ import {
|
||||
type RetrievalTestResult,
|
||||
createRetrievalTestExecutor,
|
||||
} from "./retrieval-test";
|
||||
import { registerRetrievalTestHandlers } from "./retrieval-test-handlers";
|
||||
import {
|
||||
RetrievalTestMetricsSchema,
|
||||
RetrievalTestRequestSchema,
|
||||
@ -21,7 +31,9 @@ import {
|
||||
} from "./retrieval-test-routes";
|
||||
|
||||
const SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42";
|
||||
const SECOND_SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44";
|
||||
const TOKEN = "owner-token";
|
||||
const WORKFLOW_QUERY_ID = "10000000-0000-4000-8000-000000000010";
|
||||
const reasoningSelection = {
|
||||
model: "reasoning-1",
|
||||
pluginId: "plugin/reasoning",
|
||||
@ -47,6 +59,161 @@ const retrievalProfile: KnowledgeSpaceRetrievalProfile = {
|
||||
};
|
||||
|
||||
describe("retrieval test route", () => {
|
||||
it.each([
|
||||
{
|
||||
expectedOutcome: "answered",
|
||||
result: retrievalResult("fast"),
|
||||
},
|
||||
{
|
||||
expectedOutcome: "low-confidence",
|
||||
result: emptyRetrievalResult({ scoreThresholdFilteredCandidates: 2 }),
|
||||
},
|
||||
{
|
||||
expectedOutcome: "no-evidence",
|
||||
result: emptyRetrievalResult(),
|
||||
},
|
||||
] as const)(
|
||||
"records workflow retrieval as $expectedOutcome for Overview",
|
||||
async ({ expectedOutcome, result }) => {
|
||||
const { answerTraceRecorder, app, overview } = workflowTelemetryApp(result);
|
||||
const expectedTraceId = deterministicKnowledgeSpaceActivityId(
|
||||
"workflow.answer-trace",
|
||||
"tenant-1",
|
||||
SPACE_ID,
|
||||
WORKFLOW_QUERY_ID,
|
||||
);
|
||||
|
||||
const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, {
|
||||
body: JSON.stringify({ query: "workflow camera query", queryId: WORKFLOW_QUERY_ID }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(overview.appendActivity).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: "query.requested",
|
||||
details: expect.objectContaining({ mode: "fast" }),
|
||||
resource: { id: expectedTraceId, type: "query" },
|
||||
}),
|
||||
);
|
||||
expect(answerTraceRecorder.record).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
capabilityGrantId: "10000000-0000-4000-8000-000000000011",
|
||||
traceId: expectedTraceId,
|
||||
steps: [
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
queryOutcome: expectedOutcome,
|
||||
workflowQueryId: WORKFLOW_QUERY_ID,
|
||||
}),
|
||||
name: "query.generate",
|
||||
status: "ok",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps interactive retrieval tests out of Overview query traffic", async () => {
|
||||
const { answerTraceRecorder, app, overview } = workflowTelemetryApp(retrievalResult("fast"), {
|
||||
callerKind: "interactive",
|
||||
});
|
||||
|
||||
const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, {
|
||||
body: JSON.stringify({ query: "manual camera query", queryId: WORKFLOW_QUERY_ID }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(overview.appendActivity).not.toHaveBeenCalled();
|
||||
expect(answerTraceRecorder.record).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps legacy workflow requests without a business query id out of Overview", async () => {
|
||||
const { answerTraceRecorder, app, overview } = workflowTelemetryApp(retrievalResult("fast"));
|
||||
|
||||
const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, {
|
||||
body: JSON.stringify({ query: "legacy workflow camera query" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(overview.appendActivity).not.toHaveBeenCalled();
|
||||
expect(answerTraceRecorder.record).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("persists one trace per space for a shared workflow query id", async () => {
|
||||
const answerTraces = createInMemoryAnswerTraceRepository({ maxSteps: 10, maxTraces: 10 });
|
||||
const answerTraceRecorder = createAnswerTraceRecorder({
|
||||
now: () => "2026-09-04T08:00:00.000Z",
|
||||
repository: answerTraces,
|
||||
});
|
||||
const overview = {
|
||||
appendActivity: vi.fn(async (input) => input as never),
|
||||
} satisfies Pick<KnowledgeSpaceOverviewRepository, "appendActivity">;
|
||||
const first = workflowTelemetryApp(retrievalResult("fast"), {
|
||||
answerTraceRecorder,
|
||||
overview,
|
||||
spaceId: SPACE_ID,
|
||||
});
|
||||
const second = workflowTelemetryApp(retrievalResult("fast"), {
|
||||
answerTraceRecorder,
|
||||
grantId: "10000000-0000-4000-8000-000000000012",
|
||||
overview,
|
||||
spaceId: SECOND_SPACE_ID,
|
||||
});
|
||||
|
||||
const responses = await Promise.all(
|
||||
[
|
||||
{ app: first.app, spaceId: SPACE_ID },
|
||||
{ app: second.app, spaceId: SECOND_SPACE_ID },
|
||||
].map(({ app, spaceId }) =>
|
||||
app.request(`/knowledge-spaces/${spaceId}/retrieval-tests`, {
|
||||
body: JSON.stringify({ query: "shared workflow query", queryId: WORKFLOW_QUERY_ID }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
expect(responses.map((response) => response.status)).toEqual([200, 200]);
|
||||
const traceIds = overview.appendActivity.mock.calls.map(
|
||||
([activity]) => activity.resource.id as string,
|
||||
);
|
||||
expect(new Set(traceIds).size).toBe(2);
|
||||
const traces = await Promise.all([
|
||||
answerTraces.get({ id: traceIds[0] as string, knowledgeSpaceId: SPACE_ID }),
|
||||
answerTraces.get({ id: traceIds[1] as string, knowledgeSpaceId: SECOND_SPACE_ID }),
|
||||
]);
|
||||
expect(traces.map((trace) => trace?.knowledgeSpaceId)).toEqual([SPACE_ID, SECOND_SPACE_ID]);
|
||||
for (const trace of traces) {
|
||||
expect(trace?.steps[0]?.metadata).toMatchObject({ workflowQueryId: WORKFLOW_QUERY_ID });
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps pure-image workflow requests on text-only spaces out of Overview", async () => {
|
||||
const { answerTraceRecorder, app, overview } = workflowTelemetryApp(retrievalResult("fast"), {
|
||||
modelInputModalityResolver: { resolve: async () => ["text"] },
|
||||
});
|
||||
|
||||
const response = await app.request(`/knowledge-spaces/${SPACE_ID}/retrieval-tests`, {
|
||||
body: JSON.stringify({
|
||||
queryId: WORKFLOW_QUERY_ID,
|
||||
queryImages: [{ uploadFileId: "00000000-0000-4000-8000-000000000001" }],
|
||||
}),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(overview.appendActivity).not.toHaveBeenCalled();
|
||||
expect(answerTraceRecorder.record).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves workflow-granted images and freezes each space's model modalities", async () => {
|
||||
const image = {
|
||||
body: new Uint8Array([1, 2, 3]),
|
||||
@ -775,6 +942,115 @@ function researchMetrics() {
|
||||
};
|
||||
}
|
||||
|
||||
function emptyRetrievalResult(
|
||||
metrics: { readonly scoreThresholdFilteredCandidates?: number } = {},
|
||||
): RetrievalTestResult {
|
||||
return {
|
||||
...retrievalResult("fast"),
|
||||
items: [],
|
||||
metrics: {
|
||||
...retrievalResult("fast").metrics,
|
||||
...metrics,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function workflowTelemetryApp(
|
||||
result: RetrievalTestResult,
|
||||
options: {
|
||||
readonly answerTraceRecorder?: AnswerTraceRecorder;
|
||||
readonly callerKind?: DifyCapabilityV2SanitizedGrant["callerKind"];
|
||||
readonly grantId?: string;
|
||||
readonly modelInputModalityResolver?: Parameters<
|
||||
typeof registerRetrievalTestHandlers
|
||||
>[0]["modelInputModalityResolver"];
|
||||
readonly overview?: Pick<KnowledgeSpaceOverviewRepository, "appendActivity">;
|
||||
readonly spaceId?: string;
|
||||
} = {},
|
||||
) {
|
||||
const spaceId = options.spaceId ?? SPACE_ID;
|
||||
const app = new OpenAPIHono<KnowledgeGatewayEnv>();
|
||||
const overview =
|
||||
options.overview ??
|
||||
({
|
||||
appendActivity: vi.fn(async (input) => input as never),
|
||||
} satisfies Pick<KnowledgeSpaceOverviewRepository, "appendActivity">);
|
||||
const answerTraceRecorder =
|
||||
options.answerTraceRecorder ??
|
||||
({
|
||||
record: vi.fn(async (input) => input as never),
|
||||
} satisfies AnswerTraceRecorder);
|
||||
const grant: DifyCapabilityV2SanitizedGrant = {
|
||||
action: "queries.retrieval_test",
|
||||
actor: "dify-app:app-1",
|
||||
authzRevision: {
|
||||
credential_revision: null,
|
||||
external_access_epoch: 1,
|
||||
membership_epoch: 1,
|
||||
space_acl_epoch: 1,
|
||||
},
|
||||
azp: "app-1",
|
||||
callerKind: options.callerKind ?? "workflow",
|
||||
capVersion: 2,
|
||||
contentPolicyRevision: 1,
|
||||
contentScopeIds: [`knowledge-space:${spaceId}`],
|
||||
controlSpaceId: "control-space-1",
|
||||
expiresAt: 9_999_999_999,
|
||||
grantId: options.grantId ?? "10000000-0000-4000-8000-000000000011",
|
||||
issuedAt: 1,
|
||||
jtiHash: "hash",
|
||||
namespaceId: "tenant-1",
|
||||
notBefore: 1,
|
||||
resource: { id: spaceId, parent_id: null, type: "knowledge_space" },
|
||||
subject: "dify-app:app-1",
|
||||
traceId: "workflow-run-1",
|
||||
};
|
||||
app.use("*", async (context, next) => {
|
||||
context.set("subject", {
|
||||
scopes: [],
|
||||
subjectId: "dify-app:app-1",
|
||||
tenantId: "tenant-1",
|
||||
});
|
||||
context.set("capabilityV2Grant", grant);
|
||||
context.set("traceId", "transport-trace-1");
|
||||
await next();
|
||||
});
|
||||
registerRetrievalTestHandlers({
|
||||
answerTraceRecorder,
|
||||
app,
|
||||
executor: { execute: async () => result },
|
||||
...(options.modelInputModalityResolver
|
||||
? { modelInputModalityResolver: options.modelInputModalityResolver }
|
||||
: {}),
|
||||
overview,
|
||||
retrievalExecutionLeases: {
|
||||
acquire: async () => ({
|
||||
assertActive: async () => undefined,
|
||||
release: async () => undefined,
|
||||
signal: new AbortController().signal,
|
||||
}),
|
||||
},
|
||||
runtimeSnapshotResolver: {
|
||||
assertReady: async () => undefined,
|
||||
resolve: async () => ({
|
||||
...runtimeSnapshot(),
|
||||
projectionSnapshot: {
|
||||
...runtimeSnapshot().projectionSnapshot,
|
||||
knowledgeSpaceId: spaceId,
|
||||
},
|
||||
retrievalProfile: {
|
||||
...retrievalProfile,
|
||||
scoreThreshold: { enabled: true, stage: "mode-final", value: 0.7 },
|
||||
},
|
||||
}),
|
||||
},
|
||||
spaces: {
|
||||
get: async () => ({ id: spaceId, tenantId: "tenant-1" }) as never,
|
||||
},
|
||||
});
|
||||
return { answerTraceRecorder, app, overview };
|
||||
}
|
||||
|
||||
function capability(
|
||||
kind: "embedding" | "reasoning" | "rerank",
|
||||
selection: typeof embeddingSelection | typeof reasoningSelection | typeof rerankSelection,
|
||||
|
||||
@ -1,8 +1,15 @@
|
||||
import type { OpenAPIHono } from "@hono/zod-openapi";
|
||||
import { validateKnowledgeSpaceRetrievalProfileForMode } from "@knowledge/core";
|
||||
|
||||
import type { AnswerTraceRecorder } from "./answer-trace-recorder";
|
||||
import { currentCandidateGrants } from "./candidate-content-authorization";
|
||||
import { classifyQueryOutcome } from "./failed-query-recorder";
|
||||
import type { KnowledgeGatewayEnv } from "./gateway-openapi-contracts";
|
||||
import { queryRetrievalProfileMetadata } from "./gateway-sse-responses";
|
||||
import {
|
||||
type KnowledgeSpaceOverviewRepository,
|
||||
deterministicKnowledgeSpaceActivityId,
|
||||
} from "./knowledge-space-overview";
|
||||
import type { KnowledgeSpaceRepository } from "./knowledge-space-repository";
|
||||
import {
|
||||
ModelCapabilitySnapshotSchema,
|
||||
@ -16,6 +23,7 @@ import {
|
||||
QueryImageResolutionError,
|
||||
type QueryImageResolutionReference,
|
||||
type QueryImageResolver,
|
||||
queryImageMetadata,
|
||||
queryImageResolutionReferencesFromHeader,
|
||||
} from "./query-images";
|
||||
import {
|
||||
@ -34,9 +42,11 @@ import { RetrievalTestResponseSchema, runRetrievalTestRoute } from "./retrieval-
|
||||
const RETRIEVAL_TEST_UNAVAILABLE = "Published retrieval test is unavailable";
|
||||
|
||||
export interface RegisterRetrievalTestHandlersOptions {
|
||||
readonly answerTraceRecorder?: AnswerTraceRecorder | undefined;
|
||||
readonly app: OpenAPIHono<KnowledgeGatewayEnv>;
|
||||
readonly executor?: RetrievalTestExecutor | undefined;
|
||||
readonly modelInputModalityResolver?: ModelInputModalityResolver | undefined;
|
||||
readonly overview?: Pick<KnowledgeSpaceOverviewRepository, "appendActivity"> | undefined;
|
||||
readonly queryImageResolver?: QueryImageResolver | undefined;
|
||||
readonly retrievalExecutionLeases?: RetrievalExecutionLeaseCoordinator | undefined;
|
||||
readonly runtimeSnapshotResolver?: PublishedKnowledgeSpaceRuntimeSnapshotResolver | undefined;
|
||||
@ -44,9 +54,11 @@ export interface RegisterRetrievalTestHandlersOptions {
|
||||
}
|
||||
|
||||
export function registerRetrievalTestHandlers({
|
||||
answerTraceRecorder,
|
||||
app,
|
||||
executor,
|
||||
modelInputModalityResolver,
|
||||
overview,
|
||||
queryImageResolver,
|
||||
retrievalExecutionLeases,
|
||||
runtimeSnapshotResolver,
|
||||
@ -56,6 +68,8 @@ export function registerRetrievalTestHandlers({
|
||||
const subject = context.get("subject");
|
||||
const knowledgeSpaceId = context.req.valid("param").id;
|
||||
const body = context.req.valid("json");
|
||||
const capabilityGrant = context.get("capabilityV2Grant");
|
||||
const workflowQueryId = capabilityGrant?.callerKind === "workflow" ? body.queryId : undefined;
|
||||
const space = await spaces.get({ id: knowledgeSpaceId, tenantId: subject.tenantId });
|
||||
if (!space) {
|
||||
return context.json({ error: "Knowledge space not found" }, 404);
|
||||
@ -202,6 +216,39 @@ export function registerRetrievalTestHandlers({
|
||||
}
|
||||
}
|
||||
|
||||
const workflowAnswerTraceId =
|
||||
workflowQueryId && (body.query || resolvedQueryImages.length > 0)
|
||||
? deterministicKnowledgeSpaceActivityId(
|
||||
"workflow.answer-trace",
|
||||
subject.tenantId,
|
||||
knowledgeSpaceId,
|
||||
workflowQueryId,
|
||||
)
|
||||
: undefined;
|
||||
if (workflowAnswerTraceId && answerTraceRecorder && overview) {
|
||||
const occurredAt = new Date().toISOString();
|
||||
await overview.appendActivity({
|
||||
action: "query.requested",
|
||||
actor: { id: subject.subjectId, type: "member" },
|
||||
details: {
|
||||
mode,
|
||||
...(body.query ? { question: body.query } : {}),
|
||||
},
|
||||
id: deterministicKnowledgeSpaceActivityId(
|
||||
"query.requested",
|
||||
subject.tenantId,
|
||||
knowledgeSpaceId,
|
||||
workflowAnswerTraceId,
|
||||
),
|
||||
knowledgeSpaceId,
|
||||
occurredAt,
|
||||
requiredPermissionScope: [],
|
||||
resource: { id: workflowAnswerTraceId, type: "query" },
|
||||
result: "success",
|
||||
tenantId: subject.tenantId,
|
||||
});
|
||||
}
|
||||
|
||||
const result = await executor.execute({
|
||||
...(runtimeSnapshot.embeddingProfile
|
||||
? { embeddingProfile: runtimeSnapshot.embeddingProfile }
|
||||
@ -227,6 +274,52 @@ export function registerRetrievalTestHandlers({
|
||||
traceId,
|
||||
});
|
||||
await executionLease.assertActive();
|
||||
if (workflowAnswerTraceId && capabilityGrant && answerTraceRecorder && overview) {
|
||||
const finishReason =
|
||||
result.items.length > 0 ? "retrieval-evidence" : "no-retrieval-evidence";
|
||||
const outcomeMetadata = {
|
||||
finishReason,
|
||||
...(result.items[0]?.score !== undefined ? { topScore: result.items[0].score } : {}),
|
||||
metrics: {
|
||||
scoreThresholdFilteredCandidates:
|
||||
result.metrics.scoreThresholdFilteredCandidates ?? 0,
|
||||
},
|
||||
retrievalProfile: queryRetrievalProfileMetadata(runtimeSnapshot.retrievalProfile),
|
||||
source: "workflow",
|
||||
workflowQueryId,
|
||||
};
|
||||
const classification = classifyQueryOutcome({
|
||||
finishReason,
|
||||
metadata: outcomeMetadata,
|
||||
...(runtimeSnapshot.retrievalProfile.scoreThreshold.enabled
|
||||
? {
|
||||
lowConfidenceScoreFloor: runtimeSnapshot.retrievalProfile.scoreThreshold.value,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
await answerTraceRecorder.record({
|
||||
capabilityGrantId: capabilityGrant.grantId,
|
||||
knowledgeSpaceId,
|
||||
mode,
|
||||
query: body.query,
|
||||
...(resolvedQueryImages.length > 0
|
||||
? { queryImages: resolvedQueryImages.map(queryImageMetadata) }
|
||||
: {}),
|
||||
steps: [
|
||||
{
|
||||
metadata: {
|
||||
...outcomeMetadata,
|
||||
queryOutcome: classification.outcome,
|
||||
...(classification.trigger ? { failedQueryTrigger: classification.trigger } : {}),
|
||||
},
|
||||
name: "query.generate",
|
||||
status: "ok",
|
||||
},
|
||||
],
|
||||
tenantId: subject.tenantId,
|
||||
traceId: workflowAnswerTraceId,
|
||||
});
|
||||
}
|
||||
const embeddingCapabilityStatus = "verified" as const;
|
||||
const rerankCapabilityStatus: "disabled" | "verified" = runtimeSnapshot.retrievalProfile
|
||||
.rerank.enabled
|
||||
|
||||
@ -107,6 +107,7 @@ export const RetrievalTestRequestSchema = z
|
||||
includeText: z.boolean().default(false),
|
||||
mode: KnowledgeSpaceRetrievalModeSchema.optional(),
|
||||
query: RetrievalQuerySchema.default(""),
|
||||
queryId: z.string().uuid().optional(),
|
||||
queryImages: QueryImageReferencesSchema.default([]),
|
||||
})
|
||||
.strict()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user