opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-03 06:53:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(knowledge_fs): let Research evaluation use the production Evidence V3 contract

Browse Source 
Signed-off-by: samzong <samzong.lu@gmail.com>
This commit is contained in:
samzong 2026-08-25 05:33:28 -04:00
parent 4768283500
commit 1a7a90a307
No known key found for this signature in database
GPG Key ID: 207ED79A41A78FA6
6 changed files with 134 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts
View File

@ -331,6 +331,37 @@ describe("Research evidence reasoning", () => {
expect(generate).toHaveBeenCalledOnce();
});
it("keeps a valid judgement when the provider adds prose fields or fills the token budget", async () => {
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 8_192,
providerFactory: () => ({
generate: async () => ({
metadata: { model: reasoningModel.model },
model: reasoningModel.model,
text: JSON.stringify({
coverage: 1,
coveredDimensions: ["materials"],
missingDimensions: [],
reasoning: "x".repeat(20_000),
sufficient: true,
supplementalQuery: null,
}),
}),
}),
timeoutMs: 1_000,
});
await expect(
reasoning.judge({
evidence: [researchEvidenceItem()],
evidenceDimensions: ["materials"],
query: "Which materials make up the mark?",
reasoningModel,
tenantId: "tenant-1",
}),
).resolves.toMatchObject({ modelCalled: true, sufficient: true });
});
it("normalizes a provider explanation in the boolean sufficient field", async () => {
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 128,

26
knowledge-fs/packages/api/src/research-evidence-reasoning.ts
View File

@ -133,7 +133,7 @@ export function createResearchEvidenceReasoning({
maxEvidenceCharsPerItem = 1_200,
maxEvidenceItems = 20,
maxOutputTokens,
maxResponseChars = 16_000,
maxResponseChars = maxOutputTokens * 8,
modelRequestGate,
providerFactory,
timeoutMs,
@ -443,13 +443,12 @@ function parseQueryPlan(text: string): z.infer<typeof QueryPlanSchema> {
}
if (value && typeof value === "object" && !Array.isArray(value)) {
const record = value as Record<string, unknown>;
// Dify's structured-output compatibility layer represents JSON Schema booleans as strings
// for providers that do not accept native boolean fields. Normalize that transport detail
// before applying the strict domain schema, just as the evidence judge does below.
const useGraph = normalizeUseGraphValue(record.useGraph);
if (useGraph !== undefined) {
value = { ...record, useGraph };
}
value = {
evidenceDimensions: record.evidenceDimensions,
intent: record.intent,
subqueries: record.subqueries,
useGraph: normalizeUseGraphValue(record.useGraph) ?? record.useGraph,
};
}
try {
return QueryPlanSchema.parse(value);
@ -473,10 +472,13 @@ function parseEvidenceJudgement(text: string): z.infer<typeof EvidenceJudgementS
}
if (value && typeof value === "object" && !Array.isArray(value)) {
const record = value as Record<string, unknown>;
const sufficient = normalizeBooleanValue(record.sufficient);
if (sufficient !== undefined) {
value = { ...record, sufficient };
}
value = {
coverage: record.coverage,
coveredDimensions: record.coveredDimensions,
missingDimensions: record.missingDimensions,
sufficient: normalizeBooleanValue(record.sufficient) ?? record.sufficient,
supplementalQuery: record.supplementalQuery,
};
}
try {
return EvidenceJudgementSchema.parse(value);

62
knowledge-fs/packages/api/src/research-evidence-retrieval.test.ts
View File

@ -146,6 +146,68 @@ describe("Research evidence retrieval V3", () => {
});
});
it("reports the score-threshold stage when the frozen profile enables it", async () => {
const retrieve = vi.fn(async (input: RetrieveHybridInput) => ({
items: [
item(`keep-${slug(input.query)}`, input.query),
item(`drop-${slug(input.query)}`, input.query),
],
metrics: {
denseCandidates: 2,
denseMs: 1,
ftsCandidates: 2,
ftsMs: 1,
fusedCandidates: 2,
fusionMs: 1,
totalMs: 3,
},
}));
const rerank = vi.fn(async (input: Parameters<RerankerProvider["rerank"]>[0]) => ({
items: input.documents.map((document, index) => ({
document: { ...document, metadata: { ...(document.metadata ?? {}) } },
index,
score: index === 0 ? 0.91 : 0.11,
})),
metadata: { model: input.model, provider: "static" as const },
model: input.model,
}));
const retriever = createResearchEvidenceRetrieval({
planner: createRetrievalPlanner({ maxTopK: 100 }),
queryVectorizer: { vectorize: async () => [] },
reasoning: {
judge: async () => ({
coverage: 1,
coveredDimensions: ["renewal"],
missingDimensions: [],
sufficient: true,
}),
plan: async () => ({
evidenceDimensions: ["renewal"],
intent: "lookup" as const,
modelCalled: true,
subqueries: [],
useGraph: false,
}),
},
rerankerFactory: () => ({ kind: "static", models: async () => [], rerank }),
retriever: { retrieve },
});
const result = await retriever.retrieve({
...researchInput(),
retrievalProfile: {
...retrievalProfile(),
scoreThreshold: { enabled: true, stage: "rerank", value: 0.8 },
},
});
expect(result.metrics).toMatchObject({
researchStrategyVersion: "research-evidence-v3",
scoreThresholdFilteredCandidates: 1,
});
expect(result.items).toHaveLength(1);
});
it("keeps the strongest query-specific rerank score for evidence shared across intents", async () => {
const shared = item(
"shared-node",

36
knowledge-fs/packages/api/src/research-evidence-retrieval.ts
View File

@ -188,27 +188,29 @@ export function createResearchEvidenceRetrieval({
rerankerModel: undefined,
});
if (!rerankRuntime) throw new Error("Research retrieval reranker is unavailable");
const scoreThreshold = rerankRuntime.scoreThreshold;
let rerankMs = restored?.result?.metrics?.rerankMs ?? 0;
let rerankCandidates = restored?.result?.metrics?.rerankCandidates ?? 0;
let scoreThresholdFilteredCandidates =
restored?.result?.metrics?.scoreThresholdFilteredCandidates ?? 0;
const rerankLists = async (lists: readonly ResearchQueryRerankList[]) => {
const rerankStartedAt = now();
rerankCandidates += lists.reduce((total, list) => total + list.items.length, 0);
try {
return await Promise.all(
lists.map(async (list) => ({
...list,
items: thresholdItems(
await rerankHybridRetrievalItems({
items: list.items,
limit: list.items.length,
model: rerankRuntime.model,
query: list.query,
reranker: rerankRuntime.provider,
tenantId,
}),
rerankRuntime.scoreThreshold,
),
})),
lists.map(async (list) => {
const rerankedItems = await rerankHybridRetrievalItems({
items: list.items,
limit: list.items.length,
model: rerankRuntime.model,
query: list.query,
reranker: rerankRuntime.provider,
tenantId,
});
const thresholded = thresholdItems(rerankedItems, scoreThreshold);
scoreThresholdFilteredCandidates += rerankedItems.length - thresholded.length;
return { ...list, items: thresholded };
}),
);
} finally {
// Calls in one intent batch run concurrently, so this is user-visible wall time rather
@ -416,6 +418,7 @@ export function createResearchEvidenceRetrieval({
rerankCandidates,
rerankMs,
rounds: snapshot.rounds,
...(scoreThreshold === undefined ? {} : { scoreThresholdFilteredCandidates }),
sufficiencyReached: judgement.sufficient,
supplementalSearches,
totalMs: Math.max(0, now() - startedAt),
@ -789,6 +792,7 @@ function combineResearchMetrics({
rerankCandidates,
rerankMs,
rounds,
scoreThresholdFilteredCandidates,
sufficiencyReached,
supplementalSearches,
totalMs,
@ -802,6 +806,7 @@ function combineResearchMetrics({
readonly rerankCandidates: number;
readonly rerankMs: number;
readonly rounds: number;
readonly scoreThresholdFilteredCandidates?: number | undefined;
readonly sufficiencyReached: boolean;
readonly supplementalSearches: number;
readonly totalMs: number;
@ -826,6 +831,9 @@ function combineResearchMetrics({
researchStrategyVersion: "research-evidence-v3",
researchSufficiencyReached: sufficiencyReached,
researchSupplementalSearches: supplementalSearches,
...(scoreThresholdFilteredCandidates === undefined
? {}
: { scoreThresholdFilteredCandidates }),
totalMs,
};
}

1
knowledge-fs/packages/api/src/retrieval-test.test.ts
View File

@ -231,6 +231,7 @@ describe("createRetrievalTestExecutor", () => {
expect(calls).toHaveLength(1);
expect(calls[0]).toMatchObject({
denseProjectionModel: embeddingProfile.vectorSpaceId,
embeddingProfile,
mode: "research",
queryVector: [0.1, 0.2, 0.3],
});

5
knowledge-fs/packages/api/src/retrieval-test.ts
View File

@ -214,7 +214,10 @@ export function createRetrievalTestExecutor({
const embeddingMs = Math.max(0, Date.now() - embeddingStartedAt);
const retrieval = await retriever.retrieve({
...(input.embeddingProfile
? { denseProjectionModel: input.embeddingProfile.vectorSpaceId }
? {
denseProjectionModel: input.embeddingProfile.vectorSpaceId,
embeddingProfile: input.embeddingProfile,
}
: {}),
knowledgeSpaceId: input.knowledgeSpaceId,
...(input.filters === undefined