From 0d39b598752b173c7f411a874ca8fdf07f3c993f Mon Sep 17 00:00:00 2001 From: Jyong Date: Wed, 19 Aug 2026 00:11:16 -0400 Subject: [PATCH] fix(knowledge-fs): recover truncated research reasoning --- api/knowledge-fs-contract.lock.json | 2 +- .../core-services/knowledge-fs.env.example | 5 + ...-research-reasoning-truncation-recovery.md | 59 +++++ knowledge-fs/apps/api/src/index.ts | 7 +- ...esearch-evidence-reasoning-options.test.ts | 58 +++++ .../research-evidence-reasoning-options.ts | 46 ++++ .../kubernetes/dify-integration-baseline.yaml | 3 + knowledge-fs/infra/local/.env.example | 3 + knowledge-fs/infra/local/compose.yaml | 3 + .../src/research-evidence-reasoning.test.ts | 203 ++++++++++++++++++ .../api/src/research-evidence-reasoning.ts | 125 ++++++++++- .../api/src/research-task-runtime.test.ts | 51 ++--- knowledge-fs/scripts/compose-apps.test.mjs | 27 +++ 13 files changed, 556 insertions(+), 36 deletions(-) create mode 100644 knowledge-fs/.harness/changes/2026-08-19-research-reasoning-truncation-recovery.md create mode 100644 knowledge-fs/apps/api/src/research-evidence-reasoning-options.test.ts create mode 100644 knowledge-fs/apps/api/src/research-evidence-reasoning-options.ts diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index e7e038c888e..43113f2bb2a 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "6637d57839e9043d320738af1a7afd94fa6470c2", + "subtreeTree": "a42203597dd3974fa051194f31b6965327e394ba", "openapiSha256": "37c8bdd6a6e7696aae3b336b0de215577ecda45535b1c2eb02d7a337b7399a95", "capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", diff --git a/docker/envs/core-services/knowledge-fs.env.example b/docker/envs/core-services/knowledge-fs.env.example index f1a13a4a013..051100dfe92 100644 --- a/docker/envs/core-services/knowledge-fs.env.example +++ b/docker/envs/core-services/knowledge-fs.env.example @@ -49,6 +49,11 @@ KNOWLEDGE_DIRECT_UPLOAD_ENABLED=on KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_BYTES=33554432 # Enable Research task SSE only when the durable task progress repository is ready. KNOWLEDGE_DIRECT_STREAM_ENABLED=off +# Research structured reasoning normally uses the smaller budget. Only a provider-confirmed +# truncated response receives one in-place retry with the larger recovery budget. +KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS=1024 +KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS=2048 +KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS=60000 # Optional image-query retrieval. This remains off unless the existing visual-embedding provider, # model/plugin selection, visual index, and query mode are configured for the deployment. diff --git a/knowledge-fs/.harness/changes/2026-08-19-research-reasoning-truncation-recovery.md b/knowledge-fs/.harness/changes/2026-08-19-research-reasoning-truncation-recovery.md new file mode 100644 index 00000000000..fd38e5be116 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-08-19-research-reasoning-truncation-recovery.md @@ -0,0 +1,59 @@ +# Research reasoning truncation recovery + +Date: 2026-08-19 + +## What changed + +- Removed the hard-coded 512-token ceiling from Research planning and evidence judgement. +- Added dedicated, operator-configurable normal and recovery output budgets. The defaults are + 1,024 and 2,048 tokens respectively. +- Raised the dedicated Research reasoning deadline from 30 to 60 seconds so the larger bounded + recovery is not constrained by the old short judgement deadline. Other model call types retain + their existing timeout policy. +- Propagated the model provider's terminal `finishReason` into Research reasoning and also inspect + provider-reported completion-token usage. +- When an otherwise invalid structured response is proven to have reached an output limit, retry + that one buffered reasoning call once with the larger recovery budget. Ordinary malformed JSON + and schema violations are not retried. +- Account for the initial and recovery calls independently through the existing Research model + observer and retain the shared model-request concurrency gate for both physical calls. +- Added a distinct terminal `RESEARCH_EVIDENCE_REASONING_TRUNCATED` error when the bounded recovery + is also truncated. Both deterministic contract errors stop after one durable task attempt. +- Added the three Research reasoning bounds to local Compose, the Dify service env example, and the + Kubernetes integration baseline. + +## Why + +The failed Research task retrieved and reranked the correct evidence (top score `0.9978258`) and +the model endpoint returned HTTP 200. The evidence judgement then consumed exactly 512 completion +tokens, matching the old hard ceiling, and returned an incomplete JSON object. Strict parsing +reported `RESEARCH_EVIDENCE_REASONING_INVALID`, so the user saw a failed Research request even +though retrieval itself had succeeded. + +The fix distinguishes output truncation from a genuine response-contract violation. It provides +one bounded recovery opportunity without restoring broad retries or multiplying every Research +model call. + +## Correctness and cost invariants + +- A valid response is never repeated, even when its token usage reaches the configured bound. +- Recovery requires either an explicit output-limit finish reason or completion-token usage at the + requested maximum. +- At most one recovery call is made per plan or judgement invocation. +- A non-truncated invalid response remains terminal and is never retried. +- Each physical call reserves and reconciles its own token budget, and each reacquires the shared + model gate. +- Provider/network and timeout errors retain their existing retry classification. + +## Verification + +- Focused Research reasoning and durable task runtime: 70 tests passed. +- API-app Research runtime configuration: 4 tests passed. +- Deployment Compose/Kubernetes contract: 12 tests passed. +- Complete `@knowledge/api` suite: 416 files passed, 1 skipped; 4,601 tests passed, 3 skipped. +- Complete `@knowledge/api-app` suite: 46 files and 262 tests passed. +- `@knowledge/api` and `@knowledge/api-app` typechecks passed. +- Full `pnpm --dir knowledge-fs check` and `pnpm --dir knowledge-fs build` passed. +- Focused Biome checks and `git diff --check` passed. The repository-wide lint command remains + blocked by existing formatting diagnostics in generated OpenAPI/Capability artifacts; none of + the changed source or test files is involved. diff --git a/knowledge-fs/apps/api/src/index.ts b/knowledge-fs/apps/api/src/index.ts index 8fc27052559..da37e933333 100644 --- a/knowledge-fs/apps/api/src/index.ts +++ b/knowledge-fs/apps/api/src/index.ts @@ -94,6 +94,7 @@ import { createApiDatabaseRepositories, } from "./repository-options"; import { createApiRerankerOptions } from "./reranker-options"; +import { createApiResearchEvidenceReasoningOptions } from "./research-evidence-reasoning-options"; import { assertApiResearchTaskDurability, createApiResearchTaskRuntime, @@ -177,6 +178,7 @@ const semanticEntityExtractionOptions = createApiSemanticEntityExtractionOptions modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate, }); const profileReasoningCapability = createApiProfileReasoningCapability(); +const researchEvidenceReasoningOptions = createApiResearchEvidenceReasoningOptions(); const pageIndexSemanticTreeSearch = createPageIndexSemanticTreeSearch({ batchSize: 5, maxConcurrentBatches: 4, @@ -511,10 +513,11 @@ const embeddingResolver = }) : undefined; const researchEvidenceReasoning = createResearchEvidenceReasoning({ - maxOutputTokens: Math.min(profileReasoningCapability.maxOutputTokens, 512), + maxOutputTokens: researchEvidenceReasoningOptions.maxOutputTokens, modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate, providerFactory: profileReasoningCapability.providerFactory, - timeoutMs: 30_000, + recoveryMaxOutputTokens: researchEvidenceReasoningOptions.recoveryMaxOutputTokens, + timeoutMs: researchEvidenceReasoningOptions.timeoutMs, }); const documentCompilationRuntime = createApiDocumentCompilationRuntime({ adapter, diff --git a/knowledge-fs/apps/api/src/research-evidence-reasoning-options.test.ts b/knowledge-fs/apps/api/src/research-evidence-reasoning-options.test.ts new file mode 100644 index 00000000000..c596a315919 --- /dev/null +++ b/knowledge-fs/apps/api/src/research-evidence-reasoning-options.test.ts @@ -0,0 +1,58 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { createApiResearchEvidenceReasoningOptions } from "./research-evidence-reasoning-options"; + +describe("Research evidence reasoning options", () => { + it("uses a larger, bounded recovery budget by default", () => { + expect(createApiResearchEvidenceReasoningOptions({})).toEqual({ + maxOutputTokens: 1_024, + recoveryMaxOutputTokens: 2_048, + timeoutMs: 60_000, + }); + }); + + it("accepts explicit positive runtime bounds", () => { + expect( + createApiResearchEvidenceReasoningOptions({ + KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "1536", + KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "3072", + KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: "45000", + }), + ).toEqual({ + maxOutputTokens: 1_536, + recoveryMaxOutputTokens: 3_072, + timeoutMs: 45_000, + }); + }); + + it("rejects invalid and inverted bounds", () => { + expect(() => + createApiResearchEvidenceReasoningOptions({ + KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "0", + }), + ).toThrow("KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS must be a positive integer"); + expect(() => + createApiResearchEvidenceReasoningOptions({ + KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "2048", + KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "1024", + }), + ).toThrow( + "KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS must be at least KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS", + ); + }); + + it("wires the dedicated Research budgets instead of the old 512-token answer cap", () => { + const indexSource = readFileSync(new URL("./index.ts", import.meta.url), "utf8"); + + expect(indexSource).toContain("createApiResearchEvidenceReasoningOptions"); + expect(indexSource).toContain( + "maxOutputTokens: researchEvidenceReasoningOptions.maxOutputTokens", + ); + expect(indexSource).toContain( + "recoveryMaxOutputTokens: researchEvidenceReasoningOptions.recoveryMaxOutputTokens", + ); + expect(indexSource).not.toContain("Math.min(profileReasoningCapability.maxOutputTokens, 512)"); + }); +}); diff --git a/knowledge-fs/apps/api/src/research-evidence-reasoning-options.ts b/knowledge-fs/apps/api/src/research-evidence-reasoning-options.ts new file mode 100644 index 00000000000..4bcbc229557 --- /dev/null +++ b/knowledge-fs/apps/api/src/research-evidence-reasoning-options.ts @@ -0,0 +1,46 @@ +import { positiveIntegerEnv } from "./generation-provider"; + +export interface ApiResearchEvidenceReasoningEnv { + readonly KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS?: string | undefined; + readonly KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS?: string | undefined; + readonly KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS?: string | undefined; +} + +export interface ApiResearchEvidenceReasoningOptions { + readonly maxOutputTokens: number; + readonly recoveryMaxOutputTokens: number; + readonly timeoutMs: number; +} + +/** + * Research judgement uses a small normal budget and one larger recovery budget. The second call + * is made only when the provider proves that the first structured response was truncated. + */ +export function createApiResearchEvidenceReasoningOptions( + env: ApiResearchEvidenceReasoningEnv = process.env, +): ApiResearchEvidenceReasoningOptions { + const maxOutputTokens = positiveIntegerEnv( + env.KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS, + 1_024, + "KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS", + ); + const recoveryMaxOutputTokens = positiveIntegerEnv( + env.KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS, + 2_048, + "KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS", + ); + if (recoveryMaxOutputTokens < maxOutputTokens) { + throw new Error( + "KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS must be at least KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS", + ); + } + return { + maxOutputTokens, + recoveryMaxOutputTokens, + timeoutMs: positiveIntegerEnv( + env.KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS, + 60_000, + "KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS", + ), + }; +} diff --git a/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml b/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml index fe642b2d3de..ccd984f3653 100644 --- a/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml +++ b/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml @@ -22,6 +22,9 @@ data: KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED: "false" KNOWLEDGE_DIRECT_UPLOAD_ENABLED: "off" KNOWLEDGE_DIRECT_STREAM_ENABLED: "off" + KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "1024" + KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "2048" + KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: "60000" DURABLE_DELETION_ENABLED: "off" DIFY_INNER_API_URL: http://api:5001 DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS: "60000" diff --git a/knowledge-fs/infra/local/.env.example b/knowledge-fs/infra/local/.env.example index d4a41a66222..304b3ce2385 100644 --- a/knowledge-fs/infra/local/.env.example +++ b/knowledge-fs/infra/local/.env.example @@ -26,6 +26,9 @@ DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES=8388608 DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS=60000 DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES=8388608 DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS=60000 +KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS=1024 +KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS=2048 +KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS=60000 # Rollout/cutover gate only; it never enables a standalone runtime. KNOWLEDGE_INTEGRATED_MODE_ENABLED=false diff --git a/knowledge-fs/infra/local/compose.yaml b/knowledge-fs/infra/local/compose.yaml index 042f204c1a5..7f46ce0d166 100644 --- a/knowledge-fs/infra/local/compose.yaml +++ b/knowledge-fs/infra/local/compose.yaml @@ -44,6 +44,9 @@ services: DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES:-8388608} DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS: ${DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS:-60000} KNOWLEDGE_INTEGRATED_MODE_ENABLED: ${KNOWLEDGE_INTEGRATED_MODE_ENABLED:-false} + KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS:-1024} + KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS:-2048} + KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: ${KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS:-60000} KNOWLEDGE_PDF_RASTERIZER: ${KNOWLEDGE_PDF_RASTERIZER:-poppler} KNOWLEDGE_PDF_RASTERIZER_DPI: ${KNOWLEDGE_PDF_RASTERIZER_DPI:-144} KNOWLEDGE_PDF_RASTERIZER_THUMBNAIL_DPI: ${KNOWLEDGE_PDF_RASTERIZER_THUMBNAIL_DPI:-48} diff --git a/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts b/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts index 6788739332e..c5daaaf7751 100644 --- a/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts +++ b/knowledge-fs/packages/api/src/research-evidence-reasoning.test.ts @@ -66,6 +66,49 @@ describe("Research evidence reasoning", () => { }); }); + it("recovers a provider-truncated complex query plan once", async () => { + const generate = vi + .fn() + .mockResolvedValueOnce({ + finishReason: "length", + metadata: { model: reasoningModel.model, usage: { completionTokens: 256 } }, + model: reasoningModel.model, + text: '{"intent":"comparison","subqueries":["renewal terms"', + }) + .mockResolvedValueOnce({ + finishReason: "stop", + metadata: { model: reasoningModel.model, usage: { completionTokens: 96 } }, + model: reasoningModel.model, + text: JSON.stringify({ + evidenceDimensions: ["renewal", "termination"], + intent: "comparison", + subqueries: ["renewal terms", "termination terms"], + useGraph: false, + }), + }); + const reasoning = createResearchEvidenceReasoning({ + maxOutputTokens: 256, + providerFactory: () => ({ generate }), + recoveryMaxOutputTokens: 1_024, + timeoutMs: 1_000, + }); + + await expect( + reasoning.plan({ + query: "比较续约条款和终止条款，并说明两者风险", + reasoningModel, + tenantId: "tenant-1", + traceId: "trace-plan", + }), + ).resolves.toMatchObject({ + evidenceDimensions: ["renewal", "termination"], + intent: "comparison", + modelCalled: true, + subqueries: ["renewal terms", "termination terms"], + }); + expect(generate.mock.calls.map(([input]) => input.maxOutputTokens)).toEqual([256, 1_024]); + }); + it("judges the evidence set once and emits only a focused supplemental query", async () => { const generate = vi.fn(async (_input: unknown) => ({ metadata: { model: reasoningModel.model }, @@ -193,6 +236,141 @@ describe("Research evidence reasoning", () => { }); }); + it.each([ + { + finishReason: "length", + label: "an explicit provider length finish reason", + metadata: { model: reasoningModel.model, usage: { completionTokens: 480 } }, + }, + { + finishReason: undefined, + label: "usage that reaches the requested output-token bound", + metadata: { model: reasoningModel.model, usage: { completionTokens: 512 } }, + }, + ])("recovers one truncated judgement detected from $label", async (firstResponse) => { + const before = vi.fn(); + const after = vi.fn(); + const generate = vi + .fn() + .mockResolvedValueOnce({ + ...(firstResponse.finishReason ? { finishReason: firstResponse.finishReason } : {}), + metadata: firstResponse.metadata, + model: reasoningModel.model, + text: '{"coverage":0.8,"coveredDimensions":["timeline"]', + }) + .mockResolvedValueOnce({ + finishReason: "stop", + metadata: { + model: reasoningModel.model, + usage: { completionTokens: 180 }, + }, + model: reasoningModel.model, + text: JSON.stringify({ + coverage: 1, + coveredDimensions: ["timeline"], + missingDimensions: [], + sufficient: true, + supplementalQuery: null, + }), + }); + const reasoning = createResearchEvidenceReasoning({ + maxOutputTokens: 512, + providerFactory: () => ({ generate }), + recoveryMaxOutputTokens: 2_048, + timeoutMs: 1_000, + }); + + await expect( + reasoning.judge({ + evidence: [researchEvidenceItem()], + evidenceDimensions: ["timeline"], + query: "Apple，1985 到底发生了什么", + reasoningModel, + researchModelCallObserver: { after, before }, + tenantId: "tenant-1", + traceId: "trace-1", + }), + ).resolves.toEqual({ + coverage: 1, + coveredDimensions: ["timeline"], + missingDimensions: [], + modelCalled: true, + sufficient: true, + }); + + expect(generate).toHaveBeenCalledTimes(2); + expect(generate.mock.calls.map(([input]) => input.maxOutputTokens)).toEqual([512, 2_048]); + expect(before.mock.calls.map(([input]) => input.callId)).toEqual([ + "research-judge:trace-1:1", + "research-judge:trace-1:1:recovery", + ]); + expect(after.mock.calls.map(([input]) => input.status)).toEqual(["succeeded", "succeeded"]); + }); + + it("does not retry a non-truncated judgement contract violation", async () => { + const generate = vi.fn(async () => ({ + finishReason: "stop", + metadata: { + model: reasoningModel.model, + usage: { completionTokens: 120 }, + }, + model: reasoningModel.model, + text: "not-json", + })); + const reasoning = createResearchEvidenceReasoning({ + maxOutputTokens: 512, + providerFactory: () => ({ generate }), + recoveryMaxOutputTokens: 2_048, + timeoutMs: 1_000, + }); + + await expect( + reasoning.judge({ + evidence: [researchEvidenceItem()], + evidenceDimensions: ["timeline"], + query: "Apple，1985 到底发生了什么", + reasoningModel, + tenantId: "tenant-1", + }), + ).rejects.toMatchObject({ + code: "RESEARCH_EVIDENCE_REASONING_INVALID", + retryable: false, + }); + expect(generate).toHaveBeenCalledOnce(); + }); + + it("fails with a distinct terminal code when the bounded recovery is also truncated", async () => { + const generate = vi.fn(async (input: { readonly maxOutputTokens?: number | undefined }) => ({ + finishReason: "length", + metadata: { + model: reasoningModel.model, + usage: { completionTokens: input.maxOutputTokens }, + }, + model: reasoningModel.model, + text: '{"coverage":0.8', + })); + const reasoning = createResearchEvidenceReasoning({ + maxOutputTokens: 512, + providerFactory: () => ({ generate }), + recoveryMaxOutputTokens: 2_048, + timeoutMs: 1_000, + }); + + await expect( + reasoning.judge({ + evidence: [researchEvidenceItem()], + evidenceDimensions: ["timeline"], + query: "Apple，1985 到底发生了什么", + reasoningModel, + tenantId: "tenant-1", + }), + ).rejects.toMatchObject({ + code: "RESEARCH_EVIDENCE_REASONING_TRUNCATED", + retryable: false, + }); + expect(generate.mock.calls.map(([input]) => input.maxOutputTokens)).toEqual([512, 2_048]); + }); + it("reports successful and failed provider calls through the model observer", async () => { const before = vi.fn(); const after = vi.fn(); @@ -312,6 +490,15 @@ describe("Research evidence reasoning", () => { }), ).toThrow("maxOutputTokens must be at least 1"); + expect(() => + createResearchEvidenceReasoning({ + maxOutputTokens: 128, + providerFactory: vi.fn(), + recoveryMaxOutputTokens: 64, + timeoutMs: 1_000, + }), + ).toThrow("recoveryMaxOutputTokens must be at least maxOutputTokens"); + const reasoning = createResearchEvidenceReasoning({ maxOutputTokens: 128, providerFactory: vi.fn(), @@ -373,3 +560,19 @@ describe("Research evidence reasoning", () => { }); }); }); + +function researchEvidenceItem() { + return { + citation: { + artifactHash: "a".repeat(64), + documentAssetId: "doc-1", + documentVersion: 1, + sectionPath: ["Apple, 1985"], + }, + metadata: { text: "Apple's board removed Steve Jobs from operational control in 1985." }, + nodeId: "node-1", + projectionIds: ["projection-1"], + score: 0.99, + sources: ["dense" as const], + }; +} diff --git a/knowledge-fs/packages/api/src/research-evidence-reasoning.ts b/knowledge-fs/packages/api/src/research-evidence-reasoning.ts index 4102f010ede..408fe8e36a4 100644 --- a/knowledge-fs/packages/api/src/research-evidence-reasoning.ts +++ b/knowledge-fs/packages/api/src/research-evidence-reasoning.ts @@ -7,6 +7,7 @@ import { estimateResearchModelPromptTokens, notifyResearchModelCallAfter, notifyResearchModelCallBefore, + parseResearchModelUsage, } from "./research-model-usage"; import type { HybridRetrievalItem } from "./retrieval-fusion"; import { evidenceTextFromHybridItem } from "./retrieval-rerank"; @@ -59,6 +60,8 @@ export interface ResearchEvidenceReasoningOptions { readonly providerFactory: ( selection: KnowledgeSpaceModelSelection, ) => ResearchEvidenceReasoningProvider; + /** One larger, in-place retry is allowed only when the provider proves output truncation. */ + readonly recoveryMaxOutputTokens?: number | undefined; readonly timeoutMs: number; } @@ -75,6 +78,7 @@ export interface ResearchEvidenceReasoningProvider { readonly temperature?: number | undefined; readonly tenantId?: string | undefined; }): Promise<{ + readonly finishReason?: string | undefined; readonly metadata?: unknown; readonly model: string; readonly text: string; @@ -101,15 +105,23 @@ const EvidenceJudgementSchema = z .strict(); export class ResearchEvidenceReasoningContractError extends Error { - readonly code = "RESEARCH_EVIDENCE_REASONING_INVALID"; + readonly code: "RESEARCH_EVIDENCE_REASONING_INVALID" | "RESEARCH_EVIDENCE_REASONING_TRUNCATED"; readonly retryable: boolean; constructor( message: string, - options: { readonly cause?: unknown; readonly retryable?: boolean | undefined } = {}, + options: { + readonly cause?: unknown; + readonly code?: + | "RESEARCH_EVIDENCE_REASONING_INVALID" + | "RESEARCH_EVIDENCE_REASONING_TRUNCATED" + | undefined; + readonly retryable?: boolean | undefined; + } = {}, ) { super(message, options.cause === undefined ? undefined : { cause: options.cause }); this.name = "ResearchEvidenceReasoningContractError"; + this.code = options.code ?? "RESEARCH_EVIDENCE_REASONING_INVALID"; this.retryable = options.retryable ?? false; } } @@ -121,6 +133,7 @@ export function createResearchEvidenceReasoning({ maxResponseChars = 16_000, modelRequestGate, providerFactory, + recoveryMaxOutputTokens = maxOutputTokens, timeoutMs, }: ResearchEvidenceReasoningOptions): ResearchEvidenceReasoning { for (const [label, value] of Object.entries({ @@ -128,15 +141,22 @@ export function createResearchEvidenceReasoning({ maxEvidenceItems, maxOutputTokens, maxResponseChars, + recoveryMaxOutputTokens, timeoutMs, })) { if (!Number.isSafeInteger(value) || value < 1) { throw new Error(`Research evidence reasoning ${label} must be at least 1`); } } + if (recoveryMaxOutputTokens < maxOutputTokens) { + throw new Error( + "Research evidence reasoning recoveryMaxOutputTokens must be at least maxOutputTokens", + ); + } const generate = async ({ callId, + callMaxOutputTokens, messages, observer, reasoningModel, @@ -145,6 +165,7 @@ export function createResearchEvidenceReasoning({ tenantId, }: { readonly callId: string; + readonly callMaxOutputTokens: number; readonly messages: readonly { readonly content: string; readonly role: "system" | "user" }[]; readonly observer?: ResearchModelCallObserver | undefined; readonly reasoningModel: KnowledgeSpaceModelSelection; @@ -155,7 +176,7 @@ export function createResearchEvidenceReasoning({ const modelCall = { callId, estimatedPromptTokens: estimateResearchModelPromptTokens({ messages, schema }), - maxOutputTokens, + maxOutputTokens: callMaxOutputTokens, model: reasoningModel.model, provider: reasoningModel.provider, step, @@ -174,7 +195,7 @@ export function createResearchEvidenceReasoning({ const provider = providerFactory(reasoningModel); const operation = () => provider.generate({ - maxOutputTokens, + maxOutputTokens: callMaxOutputTokens, messages, model: reasoningModel.model, signal: controller.signal, @@ -212,7 +233,65 @@ export function createResearchEvidenceReasoning({ metadata: result.metadata, status: "succeeded", }); - return result.text; + return result; + }; + + const generateStructured = async ({ + callId, + messages, + observer, + parse, + reasoningModel, + schema, + step, + tenantId, + }: { + readonly callId: string; + readonly messages: readonly { readonly content: string; readonly role: "system" | "user" }[]; + readonly observer?: ResearchModelCallObserver | undefined; + readonly parse: (text: string) => T; + readonly reasoningModel: KnowledgeSpaceModelSelection; + readonly schema: Readonly>; + readonly step: "research.judge" | "research.plan"; + readonly tenantId: string; + }): Promise => { + const initial = await generate({ + callId, + callMaxOutputTokens: maxOutputTokens, + messages, + observer, + reasoningModel, + schema, + step, + tenantId, + }); + try { + return parse(initial.text); + } catch (error) { + if (!responseWasTruncated(initial, maxOutputTokens)) throw error; + if (recoveryMaxOutputTokens === maxOutputTokens) { + throw truncatedResponseError(step, error); + } + } + + const recovery = await generate({ + callId: `${callId}:recovery`, + callMaxOutputTokens: recoveryMaxOutputTokens, + messages, + observer, + reasoningModel, + schema, + step, + tenantId, + }); + try { + return parse(recovery.text); + } catch (error) { + if (responseWasTruncated(recovery, recoveryMaxOutputTokens)) { + throw truncatedResponseError(step, error); + } + throw error; + } }; return { @@ -222,7 +301,7 @@ export function createResearchEvidenceReasoning({ if (!local.requiresModel) { return { ...local.plan, modelCalled: false }; } - const text = await generate({ + const parsed = await generateStructured({ callId: `research-plan:${input.traceId ?? "interactive"}`, messages: [ { @@ -233,12 +312,12 @@ export function createResearchEvidenceReasoning({ { content: query, role: "user" }, ], observer: input.researchModelCallObserver, + parse: (text) => parseJson(text, QueryPlanSchema, "research.plan"), reasoningModel: input.reasoningModel, schema: zodJsonSchema(QueryPlanSchema), step: "research.plan", tenantId: requiredText(input.tenantId, "tenantId"), }); - const parsed = parseJson(text, QueryPlanSchema, "research.plan"); return { ...parsed, evidenceDimensions: uniqueStrings(parsed.evidenceDimensions), @@ -266,7 +345,7 @@ export function createResearchEvidenceReasoning({ sectionPath: item.citation.sectionPath, text: truncate(evidenceTextFromHybridItem(item), maxEvidenceCharsPerItem), })); - const text = await generate({ + const parsed = await generateStructured({ callId: `research-judge:${input.traceId ?? "interactive"}:${evidence.length}`, messages: [ { @@ -284,12 +363,12 @@ export function createResearchEvidenceReasoning({ }, ], observer: input.researchModelCallObserver, + parse: parseEvidenceJudgement, reasoningModel: input.reasoningModel, schema: zodJsonSchema(EvidenceJudgementSchema), step: "research.judge", tenantId: requiredText(input.tenantId, "tenantId"), }); - const parsed = parseEvidenceJudgement(text); return { coverage: parsed.coverage, coveredDimensions: uniqueStrings(parsed.coveredDimensions), @@ -399,6 +478,34 @@ function parseEvidenceJudgement(text: string): z.infer= maxOutputTokens; +} + +function truncatedResponseError(step: "research.judge" | "research.plan", cause: unknown) { + return new ResearchEvidenceReasoningContractError( + `${step} response remained truncated after bounded recovery`, + { + cause, + code: "RESEARCH_EVIDENCE_REASONING_TRUNCATED", + }, + ); +} + function normalizeSufficientValue(value: unknown): boolean | undefined { if (typeof value === "boolean") return value; if (typeof value !== "string") return undefined; diff --git a/knowledge-fs/packages/api/src/research-task-runtime.test.ts b/knowledge-fs/packages/api/src/research-task-runtime.test.ts index 5dc844f3706..ab0f9f5fd12 100644 --- a/knowledge-fs/packages/api/src/research-task-runtime.test.ts +++ b/knowledge-fs/packages/api/src/research-task-runtime.test.ts @@ -1472,32 +1472,35 @@ describe("research task production runtime", () => { }); }); - it("fails an explicitly non-retryable execution error without spending every attempt", async () => { - const repository = new MemoryDurableRepository(baseJob()); - const runtime = createResearchTaskRuntime({ - ...runtimeOptions(repository), - generator: { - stream: async function* () { - yield traceStep("query.retrieve"); - throw Object.assign(new Error("research.judge returned invalid structured JSON"), { - code: "RESEARCH_EVIDENCE_REASONING_INVALID", - retryable: false, - }); + it.each(["RESEARCH_EVIDENCE_REASONING_INVALID", "RESEARCH_EVIDENCE_REASONING_TRUNCATED"])( + "fails an explicitly non-retryable %s error without spending every attempt", + async (code) => { + const repository = new MemoryDurableRepository(baseJob()); + const runtime = createResearchTaskRuntime({ + ...runtimeOptions(repository), + generator: { + stream: async function* () { + yield traceStep("query.retrieve"); + throw Object.assign(new Error("research.judge returned invalid structured JSON"), { + code, + retryable: false, + }); + }, }, - }, - }); + }); - await expect(runtime.tick()).resolves.toMatchObject({ - failed: 1, - leased: 1, - retryScheduled: 0, - }); - expect(repository.job).toMatchObject({ - error: "RESEARCH_EVIDENCE_REASONING_INVALID", - executionAttempts: 1, - stage: "failed", - }); - }); + await expect(runtime.tick()).resolves.toMatchObject({ + failed: 1, + leased: 1, + retryScheduled: 0, + }); + expect(repository.job).toMatchObject({ + error: code, + executionAttempts: 1, + stage: "failed", + }); + }, + ); it("reports progress publication failures without rolling back a completed task", async () => { const repository = new MemoryDurableRepository(baseJob()); diff --git a/knowledge-fs/scripts/compose-apps.test.mjs b/knowledge-fs/scripts/compose-apps.test.mjs index 7d90c310f13..e526b13ee0b 100644 --- a/knowledge-fs/scripts/compose-apps.test.mjs +++ b/knowledge-fs/scripts/compose-apps.test.mjs @@ -173,6 +173,18 @@ test("app compose profile uses local middleware and the required Dify dependency /^ {6}DIFY_INNER_API_URL: \$\{DIFY_INNER_API_URL:-http:\/\/host\.docker\.internal:5001\}$/m, ); assert.match(compose, /^ {6}DIFY_INNER_API_KEY: \$\{DIFY_INNER_API_KEY:-\}$/m); + assert.match( + compose, + /^ {6}KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: \$\{KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS:-1024\}$/m, + ); + assert.match( + compose, + /^ {6}KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: \$\{KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS:-2048\}$/m, + ); + assert.match( + compose, + /^ {6}KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: \$\{KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS:-60000\}$/m, + ); assert.doesNotMatch(compose, /^ {6}(?:MINIO|R2|OPENAI|ANTHROPIC|COHERE|GEMINI|VOYAGE)_/m); }); @@ -310,6 +322,9 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", ( "KNOWLEDGE_DIRECT_UPLOAD_ENABLED", "KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_BYTES", "KNOWLEDGE_DIRECT_STREAM_ENABLED", + "KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS", + "KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS", + "KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS", "KNOWLEDGE_QUERY_IMAGE_RETRIEVAL_ENABLED", "KNOWLEDGE_QUERY_IMAGE_EXPANSION_TIMEOUT_MS", "UNSTRUCTURED_API_URL", @@ -327,6 +342,12 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", ( assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_PDF_RASTERIZER_MAX_ASSETS=500$/m); assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_PDF_RASTERIZER_MAX_CONCURRENCY=2$/m); assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false$/m); + assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS=1024$/m); + assert.match( + difyKnowledgeFsEnv, + /^KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS=2048$/m, + ); + assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS=60000$/m); assert.doesNotMatch(difyKnowledgeFsEnv, /^MINIO_/m); }); @@ -342,6 +363,12 @@ test("deployment examples keep Dify KnowledgeFS rollout capabilities disabled", assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED: "false"$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_DIRECT_UPLOAD_ENABLED: "off"$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_DIRECT_STREAM_ENABLED: "off"$/m); + assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_RESEARCH_REASONING_MAX_OUTPUT_TOKENS: "1024"$/m); + assert.match( + kubernetesBaseline, + /^ {2}KNOWLEDGE_RESEARCH_REASONING_RECOVERY_MAX_OUTPUT_TOKENS: "2048"$/m, + ); + assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_RESEARCH_REASONING_TIMEOUT_MS: "60000"$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_PDF_RASTERIZER: poppler$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_PDF_RASTERIZER_DPI: "144"$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_PDF_RASTERIZER_THUMBNAIL_DPI: "48"$/m);