From 1a7a90a307f021acc77449ff03c62aaaabc6f87f Mon Sep 17 00:00:00 2001 From: samzong Date: Tue, 25 Aug 2026 05:33:28 -0400 Subject: [PATCH] fix(knowledge_fs): let Research evaluation use the production Evidence V3 contract Signed-off-by: samzong --- .../src/research-evidence-reasoning.test.ts | 31 ++++++++++ .../api/src/research-evidence-reasoning.ts | 26 ++++---- .../src/research-evidence-retrieval.test.ts | 62 +++++++++++++++++++ .../api/src/research-evidence-retrieval.ts | 36 ++++++----- .../packages/api/src/retrieval-test.test.ts | 1 + .../packages/api/src/retrieval-test.ts | 5 +- 6 files changed, 134 insertions(+), 27 deletions(-) diff --git a/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts b/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts index 260c1e8df69..9892b6ff241 100644 --- a/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts +++ b/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts @@ -331,6 +331,37 @@ describe("Research evidence reasoning", () => { expect(generate).toHaveBeenCalledOnce(); }); + it("keeps a valid judgement when the provider adds prose fields or fills the token budget", async () => { + const reasoning = createResearchEvidenceReasoning({ + maxOutputTokens: 8_192, + providerFactory: () => ({ + generate: async () => ({ + metadata: { model: reasoningModel.model }, + model: reasoningModel.model, + text: JSON.stringify({ + coverage: 1, + coveredDimensions: ["materials"], + missingDimensions: [], + reasoning: "x".repeat(20_000), + sufficient: true, + supplementalQuery: null, + }), + }), + }), + timeoutMs: 1_000, + }); + + await expect( + reasoning.judge({ + evidence: [researchEvidenceItem()], + evidenceDimensions: ["materials"], + query: "Which materials make up the mark?", + reasoningModel, + tenantId: "tenant-1", + }), + ).resolves.toMatchObject({ modelCalled: true, sufficient: true }); + }); + it("normalizes a provider explanation in the boolean sufficient field", async () => { const reasoning = createResearchEvidenceReasoning({ maxOutputTokens: 128, diff --git a/knowledge-fs/packages/api/src/research-evidence-reasoning.ts b/knowledge-fs/packages/api/src/research-evidence-reasoning.ts index d74c0d6e6ed..0d553e09ef0 100644 --- a/knowledge-fs/packages/api/src/research-evidence-reasoning.ts +++ b/knowledge-fs/packages/api/src/research-evidence-reasoning.ts @@ -133,7 +133,7 @@ export function createResearchEvidenceReasoning({ maxEvidenceCharsPerItem = 1_200, maxEvidenceItems = 20, maxOutputTokens, - maxResponseChars = 16_000, + maxResponseChars = maxOutputTokens * 8, modelRequestGate, providerFactory, timeoutMs, @@ -443,13 +443,12 @@ function parseQueryPlan(text: string): z.infer { } if (value && typeof value === "object" && !Array.isArray(value)) { const record = value as Record; - // Dify's structured-output compatibility layer represents JSON Schema booleans as strings - // for providers that do not accept native boolean fields. Normalize that transport detail - // before applying the strict domain schema, just as the evidence judge does below. - const useGraph = normalizeUseGraphValue(record.useGraph); - if (useGraph !== undefined) { - value = { ...record, useGraph }; - } + value = { + evidenceDimensions: record.evidenceDimensions, + intent: record.intent, + subqueries: record.subqueries, + useGraph: normalizeUseGraphValue(record.useGraph) ?? record.useGraph, + }; } try { return QueryPlanSchema.parse(value); @@ -473,10 +472,13 @@ function parseEvidenceJudgement(text: string): z.infer; - const sufficient = normalizeBooleanValue(record.sufficient); - if (sufficient !== undefined) { - value = { ...record, sufficient }; - } + value = { + coverage: record.coverage, + coveredDimensions: record.coveredDimensions, + missingDimensions: record.missingDimensions, + sufficient: normalizeBooleanValue(record.sufficient) ?? record.sufficient, + supplementalQuery: record.supplementalQuery, + }; } try { return EvidenceJudgementSchema.parse(value); diff --git a/knowledge-fs/packages/api/src/research-evidence-retrieval.test.ts b/knowledge-fs/packages/api/src/research-evidence-retrieval.test.ts index 283200fa24e..74ee2594ca2 100644 --- a/knowledge-fs/packages/api/src/research-evidence-retrieval.test.ts +++ b/knowledge-fs/packages/api/src/research-evidence-retrieval.test.ts @@ -146,6 +146,68 @@ describe("Research evidence retrieval V3", () => { }); }); + it("reports the score-threshold stage when the frozen profile enables it", async () => { + const retrieve = vi.fn(async (input: RetrieveHybridInput) => ({ + items: [ + item(`keep-${slug(input.query)}`, input.query), + item(`drop-${slug(input.query)}`, input.query), + ], + metrics: { + denseCandidates: 2, + denseMs: 1, + ftsCandidates: 2, + ftsMs: 1, + fusedCandidates: 2, + fusionMs: 1, + totalMs: 3, + }, + })); + const rerank = vi.fn(async (input: Parameters[0]) => ({ + items: input.documents.map((document, index) => ({ + document: { ...document, metadata: { ...(document.metadata ?? {}) } }, + index, + score: index === 0 ? 0.91 : 0.11, + })), + metadata: { model: input.model, provider: "static" as const }, + model: input.model, + })); + const retriever = createResearchEvidenceRetrieval({ + planner: createRetrievalPlanner({ maxTopK: 100 }), + queryVectorizer: { vectorize: async () => [] }, + reasoning: { + judge: async () => ({ + coverage: 1, + coveredDimensions: ["renewal"], + missingDimensions: [], + sufficient: true, + }), + plan: async () => ({ + evidenceDimensions: ["renewal"], + intent: "lookup" as const, + modelCalled: true, + subqueries: [], + useGraph: false, + }), + }, + rerankerFactory: () => ({ kind: "static", models: async () => [], rerank }), + retriever: { retrieve }, + }); + + const result = await retriever.retrieve({ + ...researchInput(), + retrievalProfile: { + ...retrievalProfile(), + scoreThreshold: { enabled: true, stage: "rerank", value: 0.8 }, + }, + }); + + expect(result.metrics).toMatchObject({ + researchStrategyVersion: "research-evidence-v3", + scoreThresholdFilteredCandidates: 1, + }); + expect(result.items).toHaveLength(1); + }); + it("keeps the strongest query-specific rerank score for evidence shared across intents", async () => { const shared = item( "shared-node", diff --git a/knowledge-fs/packages/api/src/research-evidence-retrieval.ts b/knowledge-fs/packages/api/src/research-evidence-retrieval.ts index 13fe0804c4b..3938890965f 100644 --- a/knowledge-fs/packages/api/src/research-evidence-retrieval.ts +++ b/knowledge-fs/packages/api/src/research-evidence-retrieval.ts @@ -188,27 +188,29 @@ export function createResearchEvidenceRetrieval({ rerankerModel: undefined, }); if (!rerankRuntime) throw new Error("Research retrieval reranker is unavailable"); + const scoreThreshold = rerankRuntime.scoreThreshold; let rerankMs = restored?.result?.metrics?.rerankMs ?? 0; let rerankCandidates = restored?.result?.metrics?.rerankCandidates ?? 0; + let scoreThresholdFilteredCandidates = + restored?.result?.metrics?.scoreThresholdFilteredCandidates ?? 0; const rerankLists = async (lists: readonly ResearchQueryRerankList[]) => { const rerankStartedAt = now(); rerankCandidates += lists.reduce((total, list) => total + list.items.length, 0); try { return await Promise.all( - lists.map(async (list) => ({ - ...list, - items: thresholdItems( - await rerankHybridRetrievalItems({ - items: list.items, - limit: list.items.length, - model: rerankRuntime.model, - query: list.query, - reranker: rerankRuntime.provider, - tenantId, - }), - rerankRuntime.scoreThreshold, - ), - })), + lists.map(async (list) => { + const rerankedItems = await rerankHybridRetrievalItems({ + items: list.items, + limit: list.items.length, + model: rerankRuntime.model, + query: list.query, + reranker: rerankRuntime.provider, + tenantId, + }); + const thresholded = thresholdItems(rerankedItems, scoreThreshold); + scoreThresholdFilteredCandidates += rerankedItems.length - thresholded.length; + return { ...list, items: thresholded }; + }), ); } finally { // Calls in one intent batch run concurrently, so this is user-visible wall time rather @@ -416,6 +418,7 @@ export function createResearchEvidenceRetrieval({ rerankCandidates, rerankMs, rounds: snapshot.rounds, + ...(scoreThreshold === undefined ? {} : { scoreThresholdFilteredCandidates }), sufficiencyReached: judgement.sufficient, supplementalSearches, totalMs: Math.max(0, now() - startedAt), @@ -789,6 +792,7 @@ function combineResearchMetrics({ rerankCandidates, rerankMs, rounds, + scoreThresholdFilteredCandidates, sufficiencyReached, supplementalSearches, totalMs, @@ -802,6 +806,7 @@ function combineResearchMetrics({ readonly rerankCandidates: number; readonly rerankMs: number; readonly rounds: number; + readonly scoreThresholdFilteredCandidates?: number | undefined; readonly sufficiencyReached: boolean; readonly supplementalSearches: number; readonly totalMs: number; @@ -826,6 +831,9 @@ function combineResearchMetrics({ researchStrategyVersion: "research-evidence-v3", researchSufficiencyReached: sufficiencyReached, researchSupplementalSearches: supplementalSearches, + ...(scoreThresholdFilteredCandidates === undefined + ? {} + : { scoreThresholdFilteredCandidates }), totalMs, }; } diff --git a/knowledge-fs/packages/api/src/retrieval-test.test.ts b/knowledge-fs/packages/api/src/retrieval-test.test.ts index d4fe463169b..0233ac95341 100644 --- a/knowledge-fs/packages/api/src/retrieval-test.test.ts +++ b/knowledge-fs/packages/api/src/retrieval-test.test.ts @@ -231,6 +231,7 @@ describe("createRetrievalTestExecutor", () => { expect(calls).toHaveLength(1); expect(calls[0]).toMatchObject({ denseProjectionModel: embeddingProfile.vectorSpaceId, + embeddingProfile, mode: "research", queryVector: [0.1, 0.2, 0.3], }); diff --git a/knowledge-fs/packages/api/src/retrieval-test.ts b/knowledge-fs/packages/api/src/retrieval-test.ts index 2811d795804..72aee1ede1d 100644 --- a/knowledge-fs/packages/api/src/retrieval-test.ts +++ b/knowledge-fs/packages/api/src/retrieval-test.ts @@ -214,7 +214,10 @@ export function createRetrievalTestExecutor({ const embeddingMs = Math.max(0, Date.now() - embeddingStartedAt); const retrieval = await retriever.retrieve({ ...(input.embeddingProfile - ? { denseProjectionModel: input.embeddingProfile.vectorSpaceId } + ? { + denseProjectionModel: input.embeddingProfile.vectorSpaceId, + embeddingProfile: input.embeddingProfile, + } : {}), knowledgeSpaceId: input.knowledgeSpaceId, ...(input.filters === undefined