opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-08 02:43:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(knowledge-fs): recover truncated research reasoning

Browse Source
This commit is contained in:
Jyong 2026-08-19 00:11:16 -04:00
parent b3ddc3cdf2
commit 0d39b59875
13 changed files with 556 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/knowledge-fs-contract.lock.json
View File

@ -1,6 +1,6 @@
{
"schemaVersion": 5,
"subtreeTree": "6637d57839e9043d320738af1a7afd94fa6470c2",
"subtreeTree": "a42203597dd3974fa051194f31b6965327e394ba",
"openapiSha256": "37c8bdd6a6e7696aae3b336b0de215577ecda45535b1c2eb02d7a337b7399a95",
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",

5
docker/envs/core-services/knowledge-fs.env.example
View File

@ -49,6 +49,11 @@ KNOWLEDGE_DIRECT_UPLOAD_ENABLED=on
KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_BYTES=33554432
# Enable Research task SSE only when the durable task progress repository is ready.
KNOWLEDGE_DIRECT_STREAM_ENABLED=off
# Research structured reasoning normally uses the smaller budget. Only a provider-confirmed
# truncated response receives one in-place retry with the larger recovery budget.
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS=1024
KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS=2048
KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS=60000
# Optional image-query retrieval. This remains off unless the existing visual-embedding provider,
# model/plugin selection, visual index, and query mode are configured for the deployment.

59
knowledge-fs/.harness/changes/2026-08-19-research-reasoning-truncation-recovery.md Normal file
View File

@ -0,0 +1,59 @@
# Research reasoning truncation recovery
Date: 2026-08-19
## What changed
- Removed the hard-coded 512-token ceiling from Research planning and evidence judgement.
- Added dedicated, operator-configurable normal and recovery output budgets. The defaults are
1,024 and 2,048 tokens respectively.
- Raised the dedicated Research reasoning deadline from 30 to 60 seconds so the larger bounded
recovery is not constrained by the old short judgement deadline. Other model call types retain
their existing timeout policy.
- Propagated the model provider's terminal `finishReason` into Research reasoning and also inspect
provider-reported completion-token usage.
- When an otherwise invalid structured response is proven to have reached an output limit, retry
that one buffered reasoning call once with the larger recovery budget. Ordinary malformed JSON
and schema violations are not retried.
- Account for the initial and recovery calls independently through the existing Research model
observer and retain the shared model-request concurrency gate for both physical calls.
- Added a distinct terminal `RESEARCH_EVIDENCE_REASONING_TRUNCATED` error when the bounded recovery
is also truncated. Both deterministic contract errors stop after one durable task attempt.
- Added the three Research reasoning bounds to local Compose, the Dify service env example, and the
Kubernetes integration baseline.
## Why
The failed Research task retrieved and reranked the correct evidence (top score `0.9978258`) and
the model endpoint returned HTTP 200. The evidence judgement then consumed exactly 512 completion
tokens, matching the old hard ceiling, and returned an incomplete JSON object. Strict parsing
reported `RESEARCH_EVIDENCE_REASONING_INVALID`, so the user saw a failed Research request even
though retrieval itself had succeeded.
The fix distinguishes output truncation from a genuine response-contract violation. It provides
one bounded recovery opportunity without restoring broad retries or multiplying every Research
model call.
## Correctness and cost invariants
- A valid response is never repeated, even when its token usage reaches the configured bound.
- Recovery requires either an explicit output-limit finish reason or completion-token usage at the
requested maximum.
- At most one recovery call is made per plan or judgement invocation.
- A non-truncated invalid response remains terminal and is never retried.
- Each physical call reserves and reconciles its own token budget, and each reacquires the shared
model gate.
- Provider/network and timeout errors retain their existing retry classification.
## Verification
- Focused Research reasoning and durable task runtime: 70 tests passed.
- API-app Research runtime configuration: 4 tests passed.
- Deployment Compose/Kubernetes contract: 12 tests passed.
- Complete `@knowledge/api` suite: 416 files passed, 1 skipped; 4,601 tests passed, 3 skipped.
- Complete `@knowledge/api-app` suite: 46 files and 262 tests passed.
- `@knowledge/api` and `@knowledge/api-app` typechecks passed.
- Full `pnpm --dir knowledge-fs check` and `pnpm --dir knowledge-fs build` passed.
- Focused Biome checks and `git diff --check` passed. The repository-wide lint command remains
blocked by existing formatting diagnostics in generated OpenAPI/Capability artifacts; none of
the changed source or test files is involved.

7
knowledge-fs/apps/api/src/index.ts
View File

@ -94,6 +94,7 @@ import {
createApiDatabaseRepositories,
} from "./repository-options";
import { createApiRerankerOptions } from "./reranker-options";
import { createApiResearchEvidenceReasoningOptions } from "./research-evidence-reasoning-options";
import {
assertApiResearchTaskDurability,
createApiResearchTaskRuntime,
@ -177,6 +178,7 @@ const semanticEntityExtractionOptions = createApiSemanticEntityExtractionOptions
modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate,
});
const profileReasoningCapability = createApiProfileReasoningCapability();
const researchEvidenceReasoningOptions = createApiResearchEvidenceReasoningOptions();
const pageIndexSemanticTreeSearch = createPageIndexSemanticTreeSearch({
batchSize: 5,
maxConcurrentBatches: 4,
@ -511,10 +513,11 @@ const embeddingResolver =
})
: undefined;
const researchEvidenceReasoning = createResearchEvidenceReasoning({
maxOutputTokens: Math.min(profileReasoningCapability.maxOutputTokens, 512),
maxOutputTokens: researchEvidenceReasoningOptions.maxOutputTokens,
modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate,
providerFactory: profileReasoningCapability.providerFactory,
timeoutMs: 30_000,
recoveryMaxOutputTokens: researchEvidenceReasoningOptions.recoveryMaxOutputTokens,
timeoutMs: researchEvidenceReasoningOptions.timeoutMs,
});
const documentCompilationRuntime = createApiDocumentCompilationRuntime({
adapter,

58
knowledge-fs/apps/api/src/research-evidence-reasoning-options.test.ts Normal file
View File

@ -0,0 +1,58 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { createApiResearchEvidenceReasoningOptions } from "./research-evidence-reasoning-options";
describe("Research evidence reasoning options", () => {
it("uses a larger, bounded recovery budget by default", () => {
expect(createApiResearchEvidenceReasoningOptions({})).toEqual({
maxOutputTokens: 1_024,
recoveryMaxOutputTokens: 2_048,
timeoutMs: 60_000,
});
});
it("accepts explicit positive runtime bounds", () => {
expect(
createApiResearchEvidenceReasoningOptions({
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "1536",
KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "3072",
KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: "45000",
}),
).toEqual({
maxOutputTokens: 1_536,
recoveryMaxOutputTokens: 3_072,
timeoutMs: 45_000,
});
});
it("rejects invalid and inverted bounds", () => {
expect(() =>
createApiResearchEvidenceReasoningOptions({
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "0",
}),
).toThrow("KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS must be a positive integer");
expect(() =>
createApiResearchEvidenceReasoningOptions({
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "2048",
KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "1024",
}),
).toThrow(
"KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS must be at least KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS",
);
});
it("wires the dedicated Research budgets instead of the old 512-token answer cap", () => {
const indexSource = readFileSync(new URL("./index.ts", import.meta.url), "utf8");
expect(indexSource).toContain("createApiResearchEvidenceReasoningOptions");
expect(indexSource).toContain(
"maxOutputTokens: researchEvidenceReasoningOptions.maxOutputTokens",
);
expect(indexSource).toContain(
"recoveryMaxOutputTokens: researchEvidenceReasoningOptions.recoveryMaxOutputTokens",
);
expect(indexSource).not.toContain("Math.min(profileReasoningCapability.maxOutputTokens, 512)");
});
});

46
knowledge-fs/apps/api/src/research-evidence-reasoning-options.ts Normal file
View File

@ -0,0 +1,46 @@
import { positiveIntegerEnv } from "./generation-provider";
export interface ApiResearchEvidenceReasoningEnv {
readonly KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS?: string | undefined;
readonly KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS?: string | undefined;
readonly KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS?: string | undefined;
}
export interface ApiResearchEvidenceReasoningOptions {
readonly maxOutputTokens: number;
readonly recoveryMaxOutputTokens: number;
readonly timeoutMs: number;
}
/**
* Research judgement uses a small normal budget and one larger recovery budget. The second call
* is made only when the provider proves that the first structured response was truncated.
*/
export function createApiResearchEvidenceReasoningOptions(
env: ApiResearchEvidenceReasoningEnv = process.env,
): ApiResearchEvidenceReasoningOptions {
const maxOutputTokens = positiveIntegerEnv(
env.KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS,
1_024,
"KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS",
);
const recoveryMaxOutputTokens = positiveIntegerEnv(
env.KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS,
2_048,
"KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS",
);
if (recoveryMaxOutputTokens < maxOutputTokens) {
throw new Error(
"KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS must be at least KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS",
);
}
return {
maxOutputTokens,
recoveryMaxOutputTokens,
timeoutMs: positiveIntegerEnv(
env.KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS,
60_000,
"KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS",
),
};
}

3
knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml
View File

@ -22,6 +22,9 @@ data:
KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED: "false"
KNOWLEDGE_DIRECT_UPLOAD_ENABLED: "off"
KNOWLEDGE_DIRECT_STREAM_ENABLED: "off"
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "1024"
KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "2048"
KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: "60000"
DURABLE_DELETION_ENABLED: "off"
DIFY_INNER_API_URL: http://api:5001
DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS: "60000"

3
knowledge-fs/infra/local/.env.example
View File

@ -26,6 +26,9 @@ DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES=8388608
DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS=60000
DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES=8388608
DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS=60000
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS=1024
KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS=2048
KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS=60000
# Rollout/cutover gate only; it never enables a standalone runtime.
KNOWLEDGE_INTEGRATED_MODE_ENABLED=false

3
knowledge-fs/infra/local/compose.yaml
View File

@ -44,6 +44,9 @@ services:
DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES:-8388608}
DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS: ${DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS:-60000}
KNOWLEDGE_INTEGRATED_MODE_ENABLED: ${KNOWLEDGE_INTEGRATED_MODE_ENABLED:-false}
KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS:-1024}
KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS:-2048}
KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: ${KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS:-60000}
KNOWLEDGE_PDF_RASTERIZER: ${KNOWLEDGE_PDF_RASTERIZER:-poppler}
KNOWLEDGE_PDF_RASTERIZER_DPI: ${KNOWLEDGE_PDF_RASTERIZER_DPI:-144}
KNOWLEDGE_PDF_RASTERIZER_THUMBNAIL_DPI: ${KNOWLEDGE_PDF_RASTERIZER_THUMBNAIL_DPI:-48}

203
knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts
View File

@ -66,6 +66,49 @@ describe("Research evidence reasoning", () => {
});
});
it("recovers a provider-truncated complex query plan once", async () => {
const generate = vi
.fn()
.mockResolvedValueOnce({
finishReason: "length",
metadata: { model: reasoningModel.model, usage: { completionTokens: 256 } },
model: reasoningModel.model,
text: '{"intent":"comparison","subqueries":["renewal terms"',
})
.mockResolvedValueOnce({
finishReason: "stop",
metadata: { model: reasoningModel.model, usage: { completionTokens: 96 } },
model: reasoningModel.model,
text: JSON.stringify({
evidenceDimensions: ["renewal", "termination"],
intent: "comparison",
subqueries: ["renewal terms", "termination terms"],
useGraph: false,
}),
});
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 256,
providerFactory: () => ({ generate }),
recoveryMaxOutputTokens: 1_024,
timeoutMs: 1_000,
});
await expect(
reasoning.plan({
query: "比较续约条款和终止条款，并说明两者风险",
reasoningModel,
tenantId: "tenant-1",
traceId: "trace-plan",
}),
).resolves.toMatchObject({
evidenceDimensions: ["renewal", "termination"],
intent: "comparison",
modelCalled: true,
subqueries: ["renewal terms", "termination terms"],
});
expect(generate.mock.calls.map(([input]) => input.maxOutputTokens)).toEqual([256, 1_024]);
});
it("judges the evidence set once and emits only a focused supplemental query", async () => {
const generate = vi.fn(async (_input: unknown) => ({
metadata: { model: reasoningModel.model },
@ -193,6 +236,141 @@ describe("Research evidence reasoning", () => {
});
});
it.each([
{
finishReason: "length",
label: "an explicit provider length finish reason",
metadata: { model: reasoningModel.model, usage: { completionTokens: 480 } },
},
{
finishReason: undefined,
label: "usage that reaches the requested output-token bound",
metadata: { model: reasoningModel.model, usage: { completionTokens: 512 } },
},
])("recovers one truncated judgement detected from $label", async (firstResponse) => {
const before = vi.fn();
const after = vi.fn();
const generate = vi
.fn()
.mockResolvedValueOnce({
...(firstResponse.finishReason ? { finishReason: firstResponse.finishReason } : {}),
metadata: firstResponse.metadata,
model: reasoningModel.model,
text: '{"coverage":0.8,"coveredDimensions":["timeline"]',
})
.mockResolvedValueOnce({
finishReason: "stop",
metadata: {
model: reasoningModel.model,
usage: { completionTokens: 180 },
},
model: reasoningModel.model,
text: JSON.stringify({
coverage: 1,
coveredDimensions: ["timeline"],
missingDimensions: [],
sufficient: true,
supplementalQuery: null,
}),
});
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 512,
providerFactory: () => ({ generate }),
recoveryMaxOutputTokens: 2_048,
timeoutMs: 1_000,
});
await expect(
reasoning.judge({
evidence: [researchEvidenceItem()],
evidenceDimensions: ["timeline"],
query: "Apple，1985 到底发生了什么",
reasoningModel,
researchModelCallObserver: { after, before },
tenantId: "tenant-1",
traceId: "trace-1",
}),
).resolves.toEqual({
coverage: 1,
coveredDimensions: ["timeline"],
missingDimensions: [],
modelCalled: true,
sufficient: true,
});
expect(generate).toHaveBeenCalledTimes(2);
expect(generate.mock.calls.map(([input]) => input.maxOutputTokens)).toEqual([512, 2_048]);
expect(before.mock.calls.map(([input]) => input.callId)).toEqual([
"research-judge:trace-1:1",
"research-judge:trace-1:1:recovery",
]);
expect(after.mock.calls.map(([input]) => input.status)).toEqual(["succeeded", "succeeded"]);
});
it("does not retry a non-truncated judgement contract violation", async () => {
const generate = vi.fn(async () => ({
finishReason: "stop",
metadata: {
model: reasoningModel.model,
usage: { completionTokens: 120 },
},
model: reasoningModel.model,
text: "not-json",
}));
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 512,
providerFactory: () => ({ generate }),
recoveryMaxOutputTokens: 2_048,
timeoutMs: 1_000,
});
await expect(
reasoning.judge({
evidence: [researchEvidenceItem()],
evidenceDimensions: ["timeline"],
query: "Apple，1985 到底发生了什么",
reasoningModel,
tenantId: "tenant-1",
}),
).rejects.toMatchObject({
code: "RESEARCH_EVIDENCE_REASONING_INVALID",
retryable: false,
});
expect(generate).toHaveBeenCalledOnce();
});
it("fails with a distinct terminal code when the bounded recovery is also truncated", async () => {
const generate = vi.fn(async (input: { readonly maxOutputTokens?: number | undefined }) => ({
finishReason: "length",
metadata: {
model: reasoningModel.model,
usage: { completionTokens: input.maxOutputTokens },
},
model: reasoningModel.model,
text: '{"coverage":0.8',
}));
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 512,
providerFactory: () => ({ generate }),
recoveryMaxOutputTokens: 2_048,
timeoutMs: 1_000,
});
await expect(
reasoning.judge({
evidence: [researchEvidenceItem()],
evidenceDimensions: ["timeline"],
query: "Apple，1985 到底发生了什么",
reasoningModel,
tenantId: "tenant-1",
}),
).rejects.toMatchObject({
code: "RESEARCH_EVIDENCE_REASONING_TRUNCATED",
retryable: false,
});
expect(generate.mock.calls.map(([input]) => input.maxOutputTokens)).toEqual([512, 2_048]);
});
it("reports successful and failed provider calls through the model observer", async () => {
const before = vi.fn();
const after = vi.fn();
@ -312,6 +490,15 @@ describe("Research evidence reasoning", () => {
}),
).toThrow("maxOutputTokens must be at least 1");
expect(() =>
createResearchEvidenceReasoning({
maxOutputTokens: 128,
providerFactory: vi.fn(),
recoveryMaxOutputTokens: 64,
timeoutMs: 1_000,
}),
).toThrow("recoveryMaxOutputTokens must be at least maxOutputTokens");
const reasoning = createResearchEvidenceReasoning({
maxOutputTokens: 128,
providerFactory: vi.fn(),
@ -373,3 +560,19 @@ describe("Research evidence reasoning", () => {
});
});
});
function researchEvidenceItem() {
return {
citation: {
artifactHash: "a".repeat(64),
documentAssetId: "doc-1",
documentVersion: 1,
sectionPath: ["Apple, 1985"],
},
metadata: { text: "Apple's board removed Steve Jobs from operational control in 1985." },
nodeId: "node-1",
projectionIds: ["projection-1"],
score: 0.99,
sources: ["dense" as const],
};
}

125
knowledge-fs/packages/api/src/research-evidence-reasoning.ts
View File

@ -7,6 +7,7 @@ import {
estimateResearchModelPromptTokens,
notifyResearchModelCallAfter,
notifyResearchModelCallBefore,
parseResearchModelUsage,
} from "./research-model-usage";
import type { HybridRetrievalItem } from "./retrieval-fusion";
import { evidenceTextFromHybridItem } from "./retrieval-rerank";
@ -59,6 +60,8 @@ export interface ResearchEvidenceReasoningOptions {
readonly providerFactory: (
selection: KnowledgeSpaceModelSelection,
) => ResearchEvidenceReasoningProvider;
/** One larger, in-place retry is allowed only when the provider proves output truncation. */
readonly recoveryMaxOutputTokens?: number | undefined;
readonly timeoutMs: number;
}
@ -75,6 +78,7 @@ export interface ResearchEvidenceReasoningProvider {
readonly temperature?: number | undefined;
readonly tenantId?: string | undefined;
}): Promise<{
readonly finishReason?: string | undefined;
readonly metadata?: unknown;
readonly model: string;
readonly text: string;
@ -101,15 +105,23 @@ const EvidenceJudgementSchema = z
.strict();
export class ResearchEvidenceReasoningContractError extends Error {
readonly code = "RESEARCH_EVIDENCE_REASONING_INVALID";
readonly code: "RESEARCH_EVIDENCE_REASONING_INVALID" | "RESEARCH_EVIDENCE_REASONING_TRUNCATED";
readonly retryable: boolean;
constructor(
message: string,
options: { readonly cause?: unknown; readonly retryable?: boolean | undefined } = {},
options: {
readonly cause?: unknown;
readonly code?:
| "RESEARCH_EVIDENCE_REASONING_INVALID"
| "RESEARCH_EVIDENCE_REASONING_TRUNCATED"
| undefined;
readonly retryable?: boolean | undefined;
} = {},
) {
super(message, options.cause === undefined ? undefined : { cause: options.cause });
this.name = "ResearchEvidenceReasoningContractError";
this.code = options.code ?? "RESEARCH_EVIDENCE_REASONING_INVALID";
this.retryable = options.retryable ?? false;
}
}
@ -121,6 +133,7 @@ export function createResearchEvidenceReasoning({
maxResponseChars = 16_000,
modelRequestGate,
providerFactory,
recoveryMaxOutputTokens = maxOutputTokens,
timeoutMs,
}: ResearchEvidenceReasoningOptions): ResearchEvidenceReasoning {
for (const [label, value] of Object.entries({
@ -128,15 +141,22 @@ export function createResearchEvidenceReasoning({
maxEvidenceItems,
maxOutputTokens,
maxResponseChars,
recoveryMaxOutputTokens,
timeoutMs,
})) {
if (!Number.isSafeInteger(value) || value < 1) {
throw new Error(`Research evidence reasoning ${label} must be at least 1`);
}
}
if (recoveryMaxOutputTokens < maxOutputTokens) {
throw new Error(
"Research evidence reasoning recoveryMaxOutputTokens must be at least maxOutputTokens",
);
}
const generate = async ({
callId,
callMaxOutputTokens,
messages,
observer,
reasoningModel,
@ -145,6 +165,7 @@ export function createResearchEvidenceReasoning({
tenantId,
}: {
readonly callId: string;
readonly callMaxOutputTokens: number;
readonly messages: readonly { readonly content: string; readonly role: "system" | "user" }[];
readonly observer?: ResearchModelCallObserver | undefined;
readonly reasoningModel: KnowledgeSpaceModelSelection;
@ -155,7 +176,7 @@ export function createResearchEvidenceReasoning({
const modelCall = {
callId,
estimatedPromptTokens: estimateResearchModelPromptTokens({ messages, schema }),
maxOutputTokens,
maxOutputTokens: callMaxOutputTokens,
model: reasoningModel.model,
provider: reasoningModel.provider,
step,
@ -174,7 +195,7 @@ export function createResearchEvidenceReasoning({
const provider = providerFactory(reasoningModel);
const operation = () =>
provider.generate({
maxOutputTokens,
maxOutputTokens: callMaxOutputTokens,
messages,
model: reasoningModel.model,
signal: controller.signal,
@ -212,7 +233,65 @@ export function createResearchEvidenceReasoning({
metadata: result.metadata,
status: "succeeded",
});
return result.text;
return result;
};
const generateStructured = async <T>({
callId,
messages,
observer,
parse,
reasoningModel,
schema,
step,
tenantId,
}: {
readonly callId: string;
readonly messages: readonly { readonly content: string; readonly role: "system" | "user" }[];
readonly observer?: ResearchModelCallObserver | undefined;
readonly parse: (text: string) => T;
readonly reasoningModel: KnowledgeSpaceModelSelection;
readonly schema: Readonly<Record<string, unknown>>;
readonly step: "research.judge" | "research.plan";
readonly tenantId: string;
}): Promise<T> => {
const initial = await generate({
callId,
callMaxOutputTokens: maxOutputTokens,
messages,
observer,
reasoningModel,
schema,
step,
tenantId,
});
try {
return parse(initial.text);
} catch (error) {
if (!responseWasTruncated(initial, maxOutputTokens)) throw error;
if (recoveryMaxOutputTokens === maxOutputTokens) {
throw truncatedResponseError(step, error);
}
}
const recovery = await generate({
callId: `${callId}:recovery`,
callMaxOutputTokens: recoveryMaxOutputTokens,
messages,
observer,
reasoningModel,
schema,
step,
tenantId,
});
try {
return parse(recovery.text);
} catch (error) {
if (responseWasTruncated(recovery, recoveryMaxOutputTokens)) {
throw truncatedResponseError(step, error);
}
throw error;
}
};
return {
@ -222,7 +301,7 @@ export function createResearchEvidenceReasoning({
if (!local.requiresModel) {
return { ...local.plan, modelCalled: false };
}
const text = await generate({
const parsed = await generateStructured({
callId: `research-plan:${input.traceId ?? "interactive"}`,
messages: [
{
@ -233,12 +312,12 @@ export function createResearchEvidenceReasoning({
{ content: query, role: "user" },
],
observer: input.researchModelCallObserver,
parse: (text) => parseJson(text, QueryPlanSchema, "research.plan"),
reasoningModel: input.reasoningModel,
schema: zodJsonSchema(QueryPlanSchema),
step: "research.plan",
tenantId: requiredText(input.tenantId, "tenantId"),
});
const parsed = parseJson(text, QueryPlanSchema, "research.plan");
return {
...parsed,
evidenceDimensions: uniqueStrings(parsed.evidenceDimensions),
@ -266,7 +345,7 @@ export function createResearchEvidenceReasoning({
sectionPath: item.citation.sectionPath,
text: truncate(evidenceTextFromHybridItem(item), maxEvidenceCharsPerItem),
}));
const text = await generate({
const parsed = await generateStructured({
callId: `research-judge:${input.traceId ?? "interactive"}:${evidence.length}`,
messages: [
{
@ -284,12 +363,12 @@ export function createResearchEvidenceReasoning({
},
],
observer: input.researchModelCallObserver,
parse: parseEvidenceJudgement,
reasoningModel: input.reasoningModel,
schema: zodJsonSchema(EvidenceJudgementSchema),
step: "research.judge",
tenantId: requiredText(input.tenantId, "tenantId"),
});
const parsed = parseEvidenceJudgement(text);
return {
coverage: parsed.coverage,
coveredDimensions: uniqueStrings(parsed.coveredDimensions),
@ -399,6 +478,34 @@ function parseEvidenceJudgement(text: string): z.infer<typeof EvidenceJudgementS
}
}
function responseWasTruncated(
response: {
readonly finishReason?: string | undefined;
readonly metadata?: unknown;
},
maxOutputTokens: number,
): boolean {
const finishReason = response.finishReason?.trim().toLocaleLowerCase();
if (
finishReason &&
/^(?:length|max[_ -]?(?:output[_ -]?)?tokens?|token[_ -]?limit|incomplete)$/u.test(finishReason)
) {
return true;
}
const usage = parseResearchModelUsage(response.metadata);
return usage?.completionTokens !== undefined && usage.completionTokens >= maxOutputTokens;
}
function truncatedResponseError(step: "research.judge" | "research.plan", cause: unknown) {
return new ResearchEvidenceReasoningContractError(
`${step} response remained truncated after bounded recovery`,
{
cause,
code: "RESEARCH_EVIDENCE_REASONING_TRUNCATED",
},
);
}
function normalizeSufficientValue(value: unknown): boolean | undefined {
if (typeof value === "boolean") return value;
if (typeof value !== "string") return undefined;

51
knowledge-fs/packages/api/src/research-task-runtime.test.ts
View File

@ -1472,32 +1472,35 @@ describe("research task production runtime", () => {
});
});
it("fails an explicitly non-retryable execution error without spending every attempt", async () => {
const repository = new MemoryDurableRepository(baseJob());
const runtime = createResearchTaskRuntime({
...runtimeOptions(repository),
generator: {
stream: async function* () {
yield traceStep("query.retrieve");
throw Object.assign(new Error("research.judge returned invalid structured JSON"), {
code: "RESEARCH_EVIDENCE_REASONING_INVALID",
retryable: false,
});
it.each(["RESEARCH_EVIDENCE_REASONING_INVALID", "RESEARCH_EVIDENCE_REASONING_TRUNCATED"])(
"fails an explicitly non-retryable %s error without spending every attempt",
async (code) => {
const repository = new MemoryDurableRepository(baseJob());
const runtime = createResearchTaskRuntime({
...runtimeOptions(repository),
generator: {
stream: async function* () {
yield traceStep("query.retrieve");
throw Object.assign(new Error("research.judge returned invalid structured JSON"), {
code,
retryable: false,
});
},
},
},
});
});
await expect(runtime.tick()).resolves.toMatchObject({
failed: 1,
leased: 1,
retryScheduled: 0,
});
expect(repository.job).toMatchObject({
error: "RESEARCH_EVIDENCE_REASONING_INVALID",
executionAttempts: 1,
stage: "failed",
});
});
await expect(runtime.tick()).resolves.toMatchObject({
failed: 1,
leased: 1,
retryScheduled: 0,
});
expect(repository.job).toMatchObject({
error: code,
executionAttempts: 1,
stage: "failed",
});
},
);
it("reports progress publication failures without rolling back a completed task", async () => {
const repository = new MemoryDurableRepository(baseJob());

27
knowledge-fs/scripts/compose-apps.test.mjs
View File

@ -173,6 +173,18 @@ test("app compose profile uses local middleware and the required Dify dependency
/^ {6}DIFY_INNER_API_URL: \$\{DIFY_INNER_API_URL:-http:\/\/host\.docker\.internal:5001\}$/m,
);
assert.match(compose, /^ {6}DIFY_INNER_API_KEY: \$\{DIFY_INNER_API_KEY:-\}$/m);
assert.match(
compose,
/^ {6}KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: \$\{KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS:-1024\}$/m,
);
assert.match(
compose,
/^ {6}KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: \$\{KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS:-2048\}$/m,
);
assert.match(
compose,
/^ {6}KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: \$\{KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS:-60000\}$/m,
);
assert.doesNotMatch(compose, /^ {6}(?:MINIO|R2|OPENAI|ANTHROPIC|COHERE|GEMINI|VOYAGE)_/m);
});
@ -310,6 +322,9 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", (
"KNOWLEDGE_DIRECT_UPLOAD_ENABLED",
"KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_BYTES",
"KNOWLEDGE_DIRECT_STREAM_ENABLED",
"KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS",
"KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS",
"KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS",
"KNOWLEDGE_QUERY_IMAGE_RETRIEVAL_ENABLED",
"KNOWLEDGE_QUERY_IMAGE_EXPANSION_TIMEOUT_MS",
"UNSTRUCTURED_API_URL",
@ -327,6 +342,12 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", (
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_PDF_RASTERIZER_MAX_ASSETS=500$/m);
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_PDF_RASTERIZER_MAX_CONCURRENCY=2$/m);
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false$/m);
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS=1024$/m);
assert.match(
difyKnowledgeFsEnv,
/^KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS=2048$/m,
);
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS=60000$/m);
assert.doesNotMatch(difyKnowledgeFsEnv, /^MINIO_/m);
});
@ -342,6 +363,12 @@ test("deployment examples keep Dify KnowledgeFS rollout capabilities disabled",
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED: "false"$/m);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_DIRECT_UPLOAD_ENABLED: "off"$/m);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_DIRECT_STREAM_ENABLED: "off"$/m);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "1024"$/m);
assert.match(
kubernetesBaseline,
/^ {2}KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "2048"$/m,
);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: "60000"$/m);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_PDF_RASTERIZER: poppler$/m);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_PDF_RASTERIZER_DPI: "144"$/m);
assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_PDF_RASTERIZER_THUMBNAIL_DPI: "48"$/m);