diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 90359644465..61de62202dc 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "2badcef0264f09fd0e0d33577866b8cbb4278a8b", + "subtreeTree": "fdd9cde96eb839ce06c8230c3e4a38e2e6d87369", "openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796", "capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", diff --git a/knowledge-fs/.harness/changes/2026-08-26-document-compilation-memory-bounds.md b/knowledge-fs/.harness/changes/2026-08-26-document-compilation-memory-bounds.md new file mode 100644 index 00000000000..692791e4f5d --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-08-26-document-compilation-memory-bounds.md @@ -0,0 +1,71 @@ +# Document compilation memory bounds + +## What changed + +- Replaced per-fragment parser-metadata cloning with a shared, format-independent source-element + reference during both LLM semantic chunking and deterministic chunking. +- Added one bounded metadata projection for final knowledge nodes. Fragments retain compact source + references and coordinates, while large OCR, table, and HTML payloads are not duplicated onto + every derived node. Complete elements retain compatible rich metadata up to a 256 KiB budget. +- Replaced repeated prefix slicing and UTF-8 re-encoding in semantic unit materialization with a + monotonic byte cursor. +- Stopped retaining Unstructured's raw `text_as_html` field after it has been normalized to the + existing `textAsHtml` / table representation, and advanced the default parser policy to + `unstructured@6` so old and new artifacts cannot share a cache identity. +- Made outline character counting, outline fallback summaries, and LLM summary inputs stream over + bounded prefixes instead of materializing complete multi-megabyte sections before truncation. +- Added low-heap child-process regressions for semantic and deterministic chunking using the size + and structure of the reported 2,000-row spreadsheet artifact. + +## Why + +The spreadsheet parsed successfully, but its first table element contained roughly 631,000 text +characters and 798,000 HTML characters. Semantic preflight split that element into 526 atomic units +and cloned the complete table metadata into every unit. The resulting amplification consumed about +2.60 GiB of heap before the first model call, exceeding the service's roughly 2.20 GiB V8 heap +limit. This was an in-process memory amplification bug, not an XLSX multi-sheet timeout. + +The affected code is shared by every parser format. A large HTML table, OCR-heavy PDF, or Office +document could therefore trigger the same failure even when its uploaded file was small. + +## Safety boundaries + +- Fragment metadata has a 16 KiB serialized budget; complete-element metadata has a 256 KiB + serialized budget. Omitted fields are recorded deterministically in `sourceMetadataProjection`. +- Parser metadata remains available on the canonical parse artifact; the projection only controls + repeated knowledge-node copies and does not discard source evidence. +- Compact multimodal references (`assetRef`), bounding boxes, captions, and titles remain eligible + on fragments, preserving image display, citation location, and retrieval provenance. +- Semantic prompt contents, chunk text, offsets, window sizes, and model-call behavior are unchanged. + +## Measured regression + +On the production-sized synthetic artifact (631,113 text characters and 797,526 HTML characters): + +- semantic preflight produced 526 atomic units and 132 windows under a 128 MiB V8 heap; +- deterministic chunking produced 526 nodes under the same 128 MiB heap; +- the semantic preflight process reported about 32 MiB heap in use after completion. + +These are local regression measurements, not projected production latency or throughput numbers. + +## Verification + +- `pnpm --dir knowledge-fs --filter @knowledge/api exec vitest run --reporter=dot` — 4,650 passed, + 3 skipped. +- `pnpm --dir knowledge-fs --filter @knowledge/core test:coverage` — 61 passed; package coverage + remained above 90% in every dimension. +- `pnpm --dir knowledge-fs --filter @knowledge/compute test:coverage` — 25 passed; package coverage + remained above 90% in every dimension. +- `pnpm --dir knowledge-fs --filter @knowledge/parsers test:coverage` — 63 passed; package coverage + remained above 90% in every dimension. +- `pnpm --dir knowledge-fs --filter @knowledge/api-app test` — 262 passed. +- KnowledgeFS Core, Compute, Parser, and API typechecks passed. +- `pnpm --dir knowledge-fs lint:backend` passed. + +## Remaining operational note + +The pipeline still intentionally holds one canonical parse artifact and bounded model windows in +memory. Existing upload-size, parser-response, element-count, node-count, model-window, PDF raster, +image-byte, and concurrency limits remain the admission boundaries for unusually large documents. +This change removes the unbounded multiplication by fragment count; it does not claim constant +memory independent of the admitted document size. diff --git a/knowledge-fs/packages/api/src/document-outline-builder.test.ts b/knowledge-fs/packages/api/src/document-outline-builder.test.ts index 355a6c95d94..d34827570f4 100644 --- a/knowledge-fs/packages/api/src/document-outline-builder.test.ts +++ b/knowledge-fs/packages/api/src/document-outline-builder.test.ts @@ -179,6 +179,38 @@ describe("document outline builder", () => { }); }); + it("builds a bounded summary for a large table without joining the complete payload", () => { + const builder = createDocumentOutlineBuilder({ + generateId: sequenceIds([ + "018f0d60-7a49-7cc2-9c1b-5b36f18f2c50", + "018f0d60-7a49-7cc2-9c1b-5b36f18f2c51", + ]), + maxElements: 20, + maxNodes: 10, + maxSummaryChars: 80, + now: () => createdAt, + }); + const text = `表头\n${"字段值 ".repeat(300_000)}`; + + const outline = builder.build({ + knowledgeSpaceId, + parseArtifact: parseArtifact([ + { + id: "large-table", + sectionPath: ["知识库"], + text, + type: "table", + }, + ]), + }); + + expect(outline.nodes[0]?.summary).toHaveLength(80); + expect(outline.nodes[0]?.summary?.endsWith("...")).toBe(true); + expect(outline.nodes[0]?.metadata).toMatchObject({ + canonicalCharacterCount: text.trim().length, + }); + }); + it("uses the same normalized UTF-8 byte coordinates as artifact segments and chunking", () => { const builder = createDocumentOutlineBuilder({ generateId: sequenceIds([ diff --git a/knowledge-fs/packages/api/src/document-outline-builder.ts b/knowledge-fs/packages/api/src/document-outline-builder.ts index 67de3506235..00bfdd8d15d 100644 --- a/knowledge-fs/packages/api/src/document-outline-builder.ts +++ b/knowledge-fs/packages/api/src/document-outline-builder.ts @@ -352,7 +352,7 @@ function createOutlineDraft({ } function applySpanToDraft(draft: OutlineNodeDraft, span: ElementSpan): void { - draft.characterCount += Array.from(span.text).length; + draft.characterCount += countCodePoints(span.text); draft.startOffset = draft.startOffset === undefined ? span.startOffset @@ -555,13 +555,13 @@ function summarizeOutlineTexts({ readonly texts: readonly string[]; readonly title: string; }): string { - const ownText = texts.join(" ").replaceAll(/\s+/gu, " ").trim(); - const childSummary = children - .map((child) => child.summary) - .filter((summary): summary is string => Boolean(summary?.trim())) - .join(" ") - .replaceAll(/\s+/gu, " ") - .trim(); + const ownText = normalizedTextPrefix(texts, maxSummaryChars + 1); + const childSummary = ownText + ? "" + : normalizedTextPrefix( + children.flatMap((child) => (child.summary?.trim() ? [child.summary] : [])), + maxSummaryChars + 1, + ); const summary = ownText || childSummary || title; if (summary.length <= maxSummaryChars) { @@ -575,6 +575,39 @@ function summarizeOutlineTexts({ return `${summary.slice(0, maxSummaryChars - 3)}...`; } +function normalizedTextPrefix(texts: readonly string[], maxChars: number): string { + let result = ""; + let pendingSpace = false; + const whitespace = /\s/u; + + outer: for (const text of texts) { + if (result.length > 0) { + pendingSpace = true; + } + for (const character of text) { + if (whitespace.test(character)) { + if (result.length > 0) pendingSpace = true; + continue; + } + if (pendingSpace) { + if (result.length > 0) result += " "; + pendingSpace = false; + if (result.length >= maxChars) break outer; + } + result += character; + if (result.length >= maxChars) break outer; + } + } + + return result; +} + +function countCodePoints(text: string): number { + let count = 0; + for (const _character of text) count += 1; + return count; +} + function outlineSectionKey(sectionPath: readonly string[]): string { return sectionPath.join(sectionPathSeparator); } diff --git a/knowledge-fs/packages/api/src/document-outline-summary-enhancer.test.ts b/knowledge-fs/packages/api/src/document-outline-summary-enhancer.test.ts index e8ea598590a..6c77d4e4e11 100644 --- a/knowledge-fs/packages/api/src/document-outline-summary-enhancer.test.ts +++ b/knowledge-fs/packages/api/src/document-outline-summary-enhancer.test.ts @@ -112,6 +112,38 @@ describe("document outline summary enhancer", () => { ).toThrow("Document outline summary maxSummaryChars must be at least 1"); }); + it("materializes only the admitted prefix of a large section", async () => { + const synthetic = largeOutline(1); + const calls: Parameters[0][] = []; + const artifact: ParseArtifact = { + ...synthetic.artifact, + elements: [ + { + ...(synthetic.artifact.elements[0] as ParseArtifact["elements"][number]), + text: "长文本".repeat(500_000), + }, + ], + }; + const enhancer = createDocumentOutlineSummaryEnhancer({ + maxInputChars: 80, + maxSummaryChars: 40, + model: "outline-summary-model", + promptVersion: "document-outline-summary-v1", + provider: { + summarize: async (input) => { + calls.push(input); + return { summary: "bounded" }; + }, + }, + }); + + await enhancer.enhance({ outline: synthetic.outline, parseArtifact: artifact }); + + expect(calls).toHaveLength(1); + expect(calls[0]?.text).toHaveLength(80); + expect(calls[0]?.text.endsWith("...")).toBe(true); + }); + it("bounds provider concurrency across independent outline branches", async () => { let active = 0; let maxActive = 0; diff --git a/knowledge-fs/packages/api/src/document-outline-summary-enhancer.ts b/knowledge-fs/packages/api/src/document-outline-summary-enhancer.ts index f74477d331c..8f887e7bc03 100644 --- a/knowledge-fs/packages/api/src/document-outline-summary-enhancer.ts +++ b/knowledge-fs/packages/api/src/document-outline-summary-enhancer.ts @@ -628,7 +628,7 @@ function summaryInput({ promptVersion, sectionPath: [...node.sectionPath], ...(signal ? { signal } : {}), - text: truncateText(sectionText(artifact, node), maxInputChars), + text: sectionText(artifact, node, maxInputChars), title: node.title, ...(traceId ? { traceId } : {}), }; @@ -804,12 +804,37 @@ function applySummaryResults({ }); } -function sectionText(artifact: ParseArtifact, node: DocumentOutlineNode): string { - return artifact.elements - .filter((element) => elementSectionStartsWith(element.sectionPath, node.sectionPath)) - .map((element) => element.text?.trim() ?? "") - .filter(Boolean) - .join("\n\n"); +function sectionText(artifact: ParseArtifact, node: DocumentOutlineNode, maxChars: number): string { + // Build only the admitted prefix. Joining an entire large table/HTML section before truncating + // multiplies the document size by the number of ancestor outline nodes. + let text = ""; + let selectedElements = 0; + const materializationLimit = maxChars + 1; + + for (const element of artifact.elements) { + if (!elementSectionStartsWith(element.sectionPath, node.sectionPath)) { + continue; + } + const elementText = element.text?.trim() ?? ""; + if (!elementText) { + continue; + } + if (selectedElements > 0) { + const separator = "\n\n".slice(0, materializationLimit - text.length); + text += separator; + } + const remaining = materializationLimit - text.length; + if (remaining <= 0) { + break; + } + text += elementText.slice(0, remaining); + selectedElements += 1; + if (elementText.length > remaining || text.length >= materializationLimit) { + break; + } + } + + return truncateText(text, maxChars); } function elementSectionStartsWith( diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts new file mode 100644 index 00000000000..9103b7c2652 --- /dev/null +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts @@ -0,0 +1,49 @@ +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +describe("LLM semantic chunker memory admission", () => { + it("preflights a production-sized spreadsheet artifact under a 128 MiB V8 heap", () => { + const coreUrl = new URL("../../core/src/index.ts", import.meta.url).href; + const chunkerUrl = new URL("./llm-semantic-chunker.ts", import.meta.url).href; + const script = ` + import { ParseArtifactSchema } from ${JSON.stringify(coreUrl)}; + import { preflightLlmSemanticWindows } from ${JSON.stringify(chunkerUrl)}; + const text = "行数据字段值。".repeat(90_159); + const html = \`\${"x".repeat(797_511)}
\`; + const parseArtifact = ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "structured", + createdAt: "2026-08-26T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42", + elements: [{ + id: "sheet-1", + metadata: { table: { html }, textAsHtml: html, text_as_html: html }, + sectionPath: ["知识库"], + text, + type: "table", + }], + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + metadata: {}, + parser: "unstructured", + version: 1, + }); + const result = preflightLlmSemanticWindows({ parseArtifact }); + console.log(JSON.stringify({ ...result, heapUsed: process.memoryUsage().heapUsed })); + `; + const completed = spawnSync( + process.execPath, + ["--max-old-space-size=128", "--import", "tsx", "--input-type=module", "-e", script], + { encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 15_000 }, + ); + + expect(completed.status, completed.stderr).toBe(0); + const result = JSON.parse(completed.stdout.trim()) as { + heapUsed: number; + maximumWindowCount: number; + unitCount: number; + }; + expect(result).toMatchObject({ maximumWindowCount: 132, unitCount: 526 }); + expect(result.heapUsed).toBeLessThan(128 * 1024 * 1024); + }); +}); diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts index 2433e7b5da6..2ceaed62e95 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts @@ -371,6 +371,59 @@ describe("LLM semantic chunker", () => { expect(nodes[1]?.endOffset).toBe(new TextEncoder().encode(text).byteLength); }); + it("does not amplify large parser metadata across fragmented table units", async () => { + const largeTableHtml = `${"".repeat(24_000)}
value
`; + const text = Array.from( + { length: 240 }, + (_, index) => `第 ${index + 1} 行 | 字段 ${index + 1} | 这是用于检索的表格内容`, + ).join("\n"); + const parseArtifact = artifact([ + { + id: "large-spreadsheet-table", + metadata: { + assetRef: { objectKey: "assets/spreadsheet-table.png" }, + table: { html: largeTableHtml }, + textAsHtml: largeTableHtml, + title: "知识库", + }, + sectionPath: ["知识库"], + text, + type: "table", + }, + ]); + const config = { maxChunkChars: 80, maxWindowChars: 320 } as const; + const preflight = preflightLlmSemanticWindows({ config, parseArtifact }); + const provider = new ScriptedProvider([echoEachUnit]); + const nodes = await createLlmSemanticChunker({ + ...config, + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + expect(preflight.unitCount).toBeGreaterThan(20); + expect(nodes).toHaveLength(preflight.unitCount); + expect(nodes.map((node) => node.text).join("")).toBe(text); + expect(nodes.at(-1)?.endOffset).toBe(new TextEncoder().encode(text).byteLength); + for (const node of nodes) { + expect(node.metadata).toMatchObject({ + assetRef: { objectKey: "assets/spreadsheet-table.png" }, + sourceMetadataProjection: { + completeElement: false, + omitted: [ + { field: "table", reason: "fragmented-source-element" }, + { field: "textAsHtml", reason: "fragmented-source-element" }, + ], + }, + title: "知识库", + }); + expect(node.metadata).not.toHaveProperty("table"); + expect(node.metadata).not.toHaveProperty("textAsHtml"); + } + }); + it("records legacy overlap as unapplied provenance and keeps semantic chunks contiguous", async () => { const chunker = createLlmSemanticChunker({ reasoningProviderFactory: () => new ScriptedProvider([echoEachUnit]), diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts index 7bd4ef38cdb..83481c9b88d 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts @@ -9,6 +9,7 @@ import { type KnowledgeSpaceRetrievalProfile, type ParseArtifact, emptyImageElementIndexText, + knowledgeNodeSourceMetadataWithProjection, stableJson, } from "@knowledge/core"; import { @@ -243,7 +244,7 @@ interface EffectiveChunkConfig { interface MaterializedElement { readonly elementId: string; readonly elementIndex: number; - readonly elementMetadata: Record; + readonly elementMetadata: Readonly>; readonly elementType: ParseArtifact["elements"][number]["type"]; readonly endCodeUnit: number; readonly endOffset: number; @@ -256,7 +257,6 @@ interface MaterializedElement { interface AtomicUnit { readonly elementId: string; - readonly elementMetadata: Record; readonly elementType: ParseArtifact["elements"][number]["type"]; readonly endCodeUnit: number; readonly endOffset: number; @@ -265,6 +265,8 @@ interface AtomicUnit { readonly isolationKey?: string | undefined; readonly pageNumber?: number | undefined; readonly sectionPath: readonly string[]; + /** Shared source element reference; never clone parser metadata per atomic unit. */ + readonly sourceElement: MaterializedElement; readonly startCodeUnit: number; readonly startOffset: number; readonly text: string; @@ -1692,7 +1694,7 @@ function materializeElements(parseArtifact: ParseArtifact): { elements.push({ elementId: element.id, elementIndex, - elementMetadata: cloneJsonObject(element.metadata), + elementMetadata: element.metadata, elementType: element.type, endCodeUnit: canonicalText.length, endOffset: span.endOffset, @@ -1720,17 +1722,26 @@ function materializeAtomicUnits( for (const element of elements) { const sentenceRanges = semanticRanges(element); let atomicIndex = 0; + let byteCursorCodeUnit = 0; + let byteCursorOffset = element.startOffset; for (const range of sentenceRanges) { const sentence = element.text.slice(range.start, range.end); for (const hardRange of graphemeRanges(sentence, maxChunkChars)) { const localStart = range.start + hardRange.start; const localEnd = range.start + hardRange.end; const text = element.text.slice(localStart, localEnd); - const startOffset = element.startOffset + utf8ByteLength(element.text.slice(0, localStart)); - const endOffset = element.startOffset + utf8ByteLength(element.text.slice(0, localEnd)); + if (localStart < byteCursorCodeUnit) { + throw new Error("LLM semantic chunking atomic units must preserve source order"); + } + if (localStart > byteCursorCodeUnit) { + byteCursorOffset += utf8ByteLength(element.text.slice(byteCursorCodeUnit, localStart)); + } + const startOffset = byteCursorOffset; + const endOffset = startOffset + utf8ByteLength(text); + byteCursorCodeUnit = localEnd; + byteCursorOffset = endOffset; units.push({ elementId: element.elementId, - elementMetadata: cloneJsonObject(element.elementMetadata), elementType: element.elementType, endCodeUnit: element.startCodeUnit + localEnd, endOffset, @@ -1742,7 +1753,8 @@ function materializeAtomicUnits( ? { isolationKey: `${element.elementType}:${element.elementId}` } : {}), ...(element.pageNumber === undefined ? {} : { pageNumber: element.pageNumber }), - sectionPath: [...element.sectionPath], + sectionPath: element.sectionPath, + sourceElement: element, startCodeUnit: element.startCodeUnit + localStart, startOffset, text, @@ -2701,7 +2713,16 @@ function materializeKnowledgeNode({ textNormalization: DOCUMENT_ELEMENT_TEXT_NORMALIZATION, }; if (uniqueStrings(chunk.units.map((unit) => unit.elementId)).length === 1) { - mergeSingleElementMetadata(metadata, first.elementMetadata); + const completeElement = + first.sourceElement === last.sourceElement && + first.startCodeUnit === first.sourceElement.startCodeUnit && + last.endCodeUnit === first.sourceElement.endCodeUnit; + Object.assign( + metadata, + knowledgeNodeSourceMetadataWithProjection(first.sourceElement.elementMetadata, { + completeElement, + }), + ); } const pageNumber = commonPageNumber(chunk.units); @@ -2791,25 +2812,6 @@ function commonPageNumber(units: readonly AtomicUnit[]): number | undefined { return units.every((unit) => unit.pageNumber === pageNumber) ? pageNumber : undefined; } -function mergeSingleElementMetadata( - target: Record, - source: Readonly>, -): void { - for (const key of [ - "assetRef", - "boundingBox", - "caption", - "ocrText", - "table", - "textAsHtml", - "title", - ]) { - if (Object.hasOwn(source, key)) { - target[key] = JSON.parse(JSON.stringify(source[key])) as unknown; - } - } -} - function validatePositiveInteger(name: string, value: number): void { if (!Number.isInteger(value) || value < 1) { throw new Error(`LLM semantic chunking ${name} must be at least 1`); diff --git a/knowledge-fs/packages/compute/src/compute-memory.test.ts b/knowledge-fs/packages/compute/src/compute-memory.test.ts new file mode 100644 index 00000000000..ce03e9989cf --- /dev/null +++ b/knowledge-fs/packages/compute/src/compute-memory.test.ts @@ -0,0 +1,57 @@ +import { spawnSync } from "node:child_process"; + +import { describe, expect, it } from "vitest"; + +describe("deterministic chunker memory admission", () => { + it("chunks a production-sized spreadsheet artifact under a 128 MiB V8 heap", () => { + const coreUrl = new URL("../../core/src/index.ts", import.meta.url).href; + const computeUrl = new URL("./index.ts", import.meta.url).href; + const script = ` + import { ParseArtifactSchema } from ${JSON.stringify(coreUrl)}; + import { createTypeScriptComputeRuntime } from ${JSON.stringify(computeUrl)}; + const text = "行数据字段值。".repeat(90_159); + const html = \`\${"x".repeat(797_511)}
\`; + const parseArtifact = ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "structured", + createdAt: "2026-08-26T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42", + elements: [{ + id: "sheet-1", + metadata: { table: { html }, textAsHtml: html, text_as_html: html }, + sectionPath: ["知识库"], + text, + type: "table", + }], + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + metadata: {}, + parser: "unstructured", + version: 1, + }); + const nodes = createTypeScriptComputeRuntime().chunkParseArtifact({ + config: { overlapChars: 0 }, + knowledgeSpaceId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + parseArtifact, + }); + console.log(JSON.stringify({ + heapUsed: process.memoryUsage().heapUsed, + lastEndOffset: nodes.at(-1)?.endOffset, + nodeCount: nodes.length, + })); + `; + const completed = spawnSync( + process.execPath, + ["--max-old-space-size=128", "--import", "tsx", "--input-type=module", "-e", script], + { encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 15_000 }, + ); + + expect(completed.status, completed.stderr).toBe(0); + const result = JSON.parse(completed.stdout.trim()) as { + heapUsed: number; + lastEndOffset: number; + nodeCount: number; + }; + expect(result).toMatchObject({ lastEndOffset: 1_893_339, nodeCount: 526 }); + expect(result.heapUsed).toBeLessThan(128 * 1024 * 1024); + }); +}); diff --git a/knowledge-fs/packages/compute/src/compute.test.ts b/knowledge-fs/packages/compute/src/compute.test.ts index 4ebfcf8a162..4556d8d9842 100644 --- a/knowledge-fs/packages/compute/src/compute.test.ts +++ b/knowledge-fs/packages/compute/src/compute.test.ts @@ -293,6 +293,47 @@ describe("createTypeScriptComputeRuntime", () => { expect(nodes[2]?.metadata).toMatchObject({ table: { columns: 2 }, textAsHtml: "" }); }); + it("does not copy large table metadata onto every deterministic fragment", () => { + const tableHtml = `
${"".repeat(16_000)}
value
`; + const text = "表格字段".repeat(40); + const nodes = runtime.chunkParseArtifact({ + config: { maxChunkChars: 16, overlapChars: 0 }, + knowledgeSpaceId, + parseArtifact: artifact([ + { + id: "large-table", + metadata: { + assetRef: { objectKey: "space/large-table.png" }, + table: { html: tableHtml }, + textAsHtml: tableHtml, + title: "知识库", + }, + sectionPath: ["知识库"], + text, + type: "table", + }, + ]), + }); + + expect(nodes).toHaveLength(10); + expect(nodes.map((node) => node.text).join("")).toBe(text); + for (const node of nodes) { + expect(node.metadata).toMatchObject({ + assetRef: { objectKey: "space/large-table.png" }, + sourceMetadataProjection: { + completeElement: false, + omitted: [ + { field: "table", reason: "fragmented-source-element" }, + { field: "textAsHtml", reason: "fragmented-source-element" }, + ], + }, + title: "知识库", + }); + expect(node.metadata).not.toHaveProperty("table"); + expect(node.metadata).not.toHaveProperty("textAsHtml"); + } + }); + it("returns independent output objects and enforces all configured bounds", () => { const input = { config: { maxChunkChars: 120, overlapChars: 0 }, diff --git a/knowledge-fs/packages/compute/src/index.ts b/knowledge-fs/packages/compute/src/index.ts index 8bf47d44284..60a86746367 100644 --- a/knowledge-fs/packages/compute/src/index.ts +++ b/knowledge-fs/packages/compute/src/index.ts @@ -6,6 +6,7 @@ import { type ParseArtifact, ParseArtifactSchema, emptyImageElementIndexText, + knowledgeNodeSourceMetadataWithProjection, } from "@knowledge/core"; import { isAlphabetic } from "unicode-segmenter/general"; import { @@ -275,9 +276,11 @@ interface TextSegment { elementType: string; endOffset: number; graphemeLength: number; - metadata: Record; + metadata: Readonly>; pageNumber?: number | undefined; sectionPath: string[]; + sourceEndOffset: number; + sourceStartOffset: number; startOffset: number; text: string; } @@ -452,7 +455,14 @@ function chunkParseArtifact(input: ChunkParseArtifactInput): KnowledgeNode[] { textNormalization: DOCUMENT_ELEMENT_TEXT_NORMALIZATION, }; if (nodeSegments.length === 1) { - mergeSingleSegmentMetadata(metadata, first.metadata); + Object.assign( + metadata, + knowledgeNodeSourceMetadataWithProjection(first.metadata, { + completeElement: + first.startOffset === first.sourceStartOffset && + first.endOffset === first.sourceEndOffset, + }), + ); } const pageNumber = commonPageNumber(nodeSegments); @@ -549,9 +559,11 @@ function materializeSegments(parseArtifact: ParseArtifact): TextSegment[] { elementType: element.type, endOffset, graphemeLength: countGraphemes(text), - metadata: jsonClone(element.metadata), + metadata: element.metadata, ...(element.pageNumber === undefined ? {} : { pageNumber: element.pageNumber }), sectionPath: [...element.sectionPath], + sourceEndOffset: endOffset, + sourceStartOffset: startOffset, startOffset, text, }); @@ -907,25 +919,6 @@ function toGraphemeSpan(boundaries: GraphemeBoundary[]): GraphemeSpan { }; } -function mergeSingleSegmentMetadata( - target: Record, - source: Record, -): void { - for (const key of [ - "assetRef", - "boundingBox", - "caption", - "ocrText", - "table", - "textAsHtml", - "title", - ]) { - if (Object.hasOwn(source, key)) { - target[key] = jsonClone(source[key]); - } - } -} - function commonPageNumber(segments: TextSegment[]): number | undefined { const first = (segments[0] as TextSegment).pageNumber; return segments.every((segment) => segment.pageNumber === first) ? first : undefined; diff --git a/knowledge-fs/packages/core/src/index.ts b/knowledge-fs/packages/core/src/index.ts index d8bd2d82bdf..c46ce1b38b2 100644 --- a/knowledge-fs/packages/core/src/index.ts +++ b/knowledge-fs/packages/core/src/index.ts @@ -3,3 +3,4 @@ export * from "./models"; export * from "./command-registry"; export * from "./json-utils"; export * from "./document-element-index-text"; +export * from "./source-element-metadata"; diff --git a/knowledge-fs/packages/core/src/source-element-metadata.test.ts b/knowledge-fs/packages/core/src/source-element-metadata.test.ts new file mode 100644 index 00000000000..84d3c05536b --- /dev/null +++ b/knowledge-fs/packages/core/src/source-element-metadata.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; + +import { + knowledgeNodeSourceMetadataWithProjection, + projectParseElementMetadataForKnowledgeNode, +} from "./source-element-metadata"; + +describe("knowledge node source metadata projection", () => { + it("keeps compact reference fields but omits amplified payloads from source fragments", () => { + const metadata = knowledgeNodeSourceMetadataWithProjection( + { + assetRef: { objectKey: "assets/table.png" }, + boundingBox: { height: 20, width: 40, x: 1, y: 2 }, + caption: "Quarterly metrics", + ocrText: "large OCR payload", + table: { html: "large table
" }, + textAsHtml: "large table
", + title: "Metrics", + }, + { completeElement: false }, + ); + + expect(metadata).toMatchObject({ + assetRef: { objectKey: "assets/table.png" }, + boundingBox: { height: 20, width: 40, x: 1, y: 2 }, + caption: "Quarterly metrics", + sourceMetadataProjection: { + completeElement: false, + omitted: [ + { field: "ocrText", reason: "fragmented-source-element" }, + { field: "table", reason: "fragmented-source-element" }, + { field: "textAsHtml", reason: "fragmented-source-element" }, + ], + }, + title: "Metrics", + }); + expect(metadata).not.toHaveProperty("ocrText"); + expect(metadata).not.toHaveProperty("table"); + expect(metadata).not.toHaveProperty("textAsHtml"); + }); + + it("retains complete element metadata within the byte budget and clones output values", () => { + const source = { + table: { rows: [{ metric: "ARR", value: 42 }] }, + textAsHtml: "
ARR
", + title: "Metrics", + }; + const projected = projectParseElementMetadataForKnowledgeNode(source, { + completeElement: true, + }); + + expect(projected.omissions).toEqual([]); + expect(projected.metadata).toEqual(source); + expect(projected.metadata.table).not.toBe(source.table); + }); + + it("omits oversized complete-element fields deterministically", () => { + const metadata = knowledgeNodeSourceMetadataWithProjection( + { + assetRef: { objectKey: "assets/chart.png" }, + table: { html: "x".repeat(1_000) }, + title: "Chart", + }, + { completeElement: true, maxBytes: 128 }, + ); + + expect(metadata).toMatchObject({ + assetRef: { objectKey: "assets/chart.png" }, + sourceMetadataProjection: { + completeElement: true, + maxBytes: 128, + omitted: [{ field: "table", reason: "size-limit" }], + }, + title: "Chart", + }); + expect(metadata).not.toHaveProperty("table"); + }); + + it("uses the smaller default budget for metadata repeated across fragments", () => { + const metadata = knowledgeNodeSourceMetadataWithProjection( + { + assetRef: { objectKey: "assets/chart.png" }, + caption: "x".repeat(20_000), + title: "Chart", + }, + { completeElement: false }, + ); + + expect(metadata).toMatchObject({ + assetRef: { objectKey: "assets/chart.png" }, + sourceMetadataProjection: { + completeElement: false, + maxBytes: 16 * 1024, + omitted: [{ field: "caption", reason: "size-limit" }], + }, + title: "Chart", + }); + }); + + it("rejects an invalid projection budget before serializing fields", () => { + expect(() => + projectParseElementMetadataForKnowledgeNode( + { table: { rows: 1 } }, + { + completeElement: true, + maxBytes: 0, + }, + ), + ).toThrow("maxBytes must be at least 1"); + }); +}); diff --git a/knowledge-fs/packages/core/src/source-element-metadata.ts b/knowledge-fs/packages/core/src/source-element-metadata.ts new file mode 100644 index 00000000000..2628de825ab --- /dev/null +++ b/knowledge-fs/packages/core/src/source-element-metadata.ts @@ -0,0 +1,120 @@ +export const MAX_KNOWLEDGE_NODE_SOURCE_METADATA_BYTES = 256 * 1024; +export const MAX_KNOWLEDGE_NODE_FRAGMENT_SOURCE_METADATA_BYTES = 16 * 1024; + +const FRAGMENT_SAFE_SOURCE_METADATA_KEYS = new Set(["assetRef", "boundingBox", "caption", "title"]); +const KNOWLEDGE_NODE_SOURCE_METADATA_KEYS = [ + "assetRef", + "boundingBox", + "caption", + "title", + "ocrText", + "table", + "textAsHtml", +] as const; + +export type SourceMetadataOmissionReason = "fragmented-source-element" | "size-limit"; + +export interface SourceMetadataOmission { + readonly field: string; + readonly reason: SourceMetadataOmissionReason; +} + +export interface KnowledgeNodeSourceMetadataProjection { + readonly metadata: Record; + readonly omissions: readonly SourceMetadataOmission[]; + readonly projectedBytes: number; +} + +/** + * Projects parser element metadata onto a knowledge node without amplifying large source payloads. + * + * A parser may attach the complete OCR text or HTML representation of a table to one source + * element. Copying that payload onto every chunk produced from the element turns a linear parse + * artifact into an O(chunks * metadata) allocation. Fragment nodes therefore retain only the + * compact location/reference fields. Complete elements may retain the richer fields, but only + * within a deterministic serialized-byte budget. + */ +export function projectParseElementMetadataForKnowledgeNode( + source: Readonly>, + { + completeElement, + maxBytes = completeElement + ? MAX_KNOWLEDGE_NODE_SOURCE_METADATA_BYTES + : MAX_KNOWLEDGE_NODE_FRAGMENT_SOURCE_METADATA_BYTES, + }: { + readonly completeElement: boolean; + readonly maxBytes?: number | undefined; + }, +): KnowledgeNodeSourceMetadataProjection { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) { + throw new Error("Knowledge node source metadata maxBytes must be at least 1"); + } + + const metadata: Record = {}; + const omissions: SourceMetadataOmission[] = []; + let projectedBytes = 2; // Opening and closing braces of the projected JSON object. + let projectedFields = 0; + + for (const field of KNOWLEDGE_NODE_SOURCE_METADATA_KEYS) { + if (!Object.hasOwn(source, field) || source[field] === undefined) { + continue; + } + if (!completeElement && !FRAGMENT_SAFE_SOURCE_METADATA_KEYS.has(field)) { + omissions.push({ field, reason: "fragmented-source-element" }); + continue; + } + + const serialized = JSON.stringify({ [field]: source[field] }); + if (serialized === undefined || serialized === "{}") { + continue; + } + const entryBytes = utf8ByteLength(serialized) - 2 + (projectedFields === 0 ? 0 : 1); + if (projectedBytes + entryBytes > maxBytes) { + omissions.push({ field, reason: "size-limit" }); + continue; + } + + Object.assign(metadata, JSON.parse(serialized) as Record); + projectedBytes += entryBytes; + projectedFields += 1; + } + + return { metadata, omissions, projectedBytes }; +} + +/** Adds compact, deterministic provenance only when source metadata had to be omitted. */ +export function knowledgeNodeSourceMetadataWithProjection( + source: Readonly>, + options: { + readonly completeElement: boolean; + readonly maxBytes?: number | undefined; + }, +): Record { + const projection = projectParseElementMetadataForKnowledgeNode(source, options); + if (projection.omissions.length === 0) { + return projection.metadata; + } + + return { + ...projection.metadata, + sourceMetadataProjection: { + completeElement: options.completeElement, + maxBytes: + options.maxBytes ?? + (options.completeElement + ? MAX_KNOWLEDGE_NODE_SOURCE_METADATA_BYTES + : MAX_KNOWLEDGE_NODE_FRAGMENT_SOURCE_METADATA_BYTES), + omitted: projection.omissions.map(({ field, reason }) => ({ field, reason })), + projectedBytes: projection.projectedBytes, + }, + }; +} + +function utf8ByteLength(value: string): number { + let length = 0; + for (const character of value) { + const codePoint = character.codePointAt(0) as number; + length += codePoint <= 0x7f ? 1 : codePoint <= 0x7ff ? 2 : codePoint <= 0xffff ? 3 : 4; + } + return length; +} diff --git a/knowledge-fs/packages/parsers/src/index.ts b/knowledge-fs/packages/parsers/src/index.ts index ad29251aec4..7dfaf17f7d5 100644 --- a/knowledge-fs/packages/parsers/src/index.ts +++ b/knowledge-fs/packages/parsers/src/index.ts @@ -318,7 +318,7 @@ export function createUnstructuredParserClient({ requestGate.run(async () => { const deadline = createUnstructuredRequestDeadline(input.signal, requestTimeoutMs); try { - const parserVersion = options.parserVersion ?? "unstructured@5"; + const parserVersion = options.parserVersion ?? "unstructured@6"; const partitionStrategy = unstructuredPartitionStrategy(input); const providerImageBlockTypes = unstructuredProviderImageBlockTypes(input); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); @@ -1683,9 +1683,13 @@ function unstructuredParseElementMetadata({ }): Record { // `image_base64` can be several megabytes. Move it into the short-lived assetRef URI consumed // by the multimodal extractor instead of retaining a second copy in ParseElement metadata. + // `text_as_html` is normalized below. Keeping the provider spelling as well would retain the + // same potentially multi-megabyte table HTML three times (`text_as_html`, `textAsHtml`, and + // `table.html`) in every parse artifact. const { image_base64: _imageBase64, page_number: _pageNumber, + text_as_html: _textAsHtml, ...metadataWithoutInlineImage } = metadata; const parsed = cloneMetadata(metadataWithoutInlineImage); diff --git a/knowledge-fs/packages/parsers/src/parser.test.ts b/knowledge-fs/packages/parsers/src/parser.test.ts index 65e3b815502..8666ed238c6 100644 --- a/knowledge-fs/packages/parsers/src/parser.test.ts +++ b/knowledge-fs/packages/parsers/src/parser.test.ts @@ -1049,7 +1049,7 @@ describe("parser adapters", () => { metadata: { filename: "report.pdf", mimeType: "application/pdf", - parserVersion: "unstructured@5", + parserVersion: "unstructured@6", }, parser: "unstructured", version: 1, @@ -1099,7 +1099,6 @@ describe("parser adapters", () => { metadata: { table: { html: "
ARR
" }, textAsHtml: "
ARR
", - text_as_html: "
ARR
", unstructuredType: "Table", }, pageNumber: 3, @@ -1139,7 +1138,7 @@ describe("parser adapters", () => { version: 1, }), ).resolves.toMatchObject({ - metadata: { parserVersion: "unstructured@5" }, + metadata: { parserVersion: "unstructured@6" }, parser: "unstructured", }); },