mirror of
https://github.com/langgenius/dify.git
synced 2026-09-03 23:47:16 +08:00
fix(knowledge-fs): bound document compilation memory
This commit is contained in:
Jyong
parent
e0fe418b3a
commit
c195854260
@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 5,
|
||||
"subtreeTree": "2badcef0264f09fd0e0d33577866b8cbb4278a8b",
|
||||
"subtreeTree": "fdd9cde96eb839ce06c8230c3e4a38e2e6d87369",
|
||||
"openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796",
|
||||
"capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109",
|
||||
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
|
||||
|
||||
@ -0,0 +1,71 @@
|
||||
# Document compilation memory bounds
|
||||
|
||||
## What changed
|
||||
|
||||
- Replaced per-fragment parser-metadata cloning with a shared, format-independent source-element
|
||||
reference during both LLM semantic chunking and deterministic chunking.
|
||||
- Added one bounded metadata projection for final knowledge nodes. Fragments retain compact source
|
||||
references and coordinates, while large OCR, table, and HTML payloads are not duplicated onto
|
||||
every derived node. Complete elements retain compatible rich metadata up to a 256 KiB budget.
|
||||
- Replaced repeated prefix slicing and UTF-8 re-encoding in semantic unit materialization with a
|
||||
monotonic byte cursor.
|
||||
- Stopped retaining Unstructured's raw `text_as_html` field after it has been normalized to the
|
||||
existing `textAsHtml` / table representation, and advanced the default parser policy to
|
||||
`unstructured@6` so old and new artifacts cannot share a cache identity.
|
||||
- Made outline character counting, outline fallback summaries, and LLM summary inputs stream over
|
||||
bounded prefixes instead of materializing complete multi-megabyte sections before truncation.
|
||||
- Added low-heap child-process regressions for semantic and deterministic chunking using the size
|
||||
and structure of the reported 2,000-row spreadsheet artifact.
|
||||
|
||||
## Why
|
||||
|
||||
The spreadsheet parsed successfully, but its first table element contained roughly 631,000 text
|
||||
characters and 798,000 HTML characters. Semantic preflight split that element into 526 atomic units
|
||||
and cloned the complete table metadata into every unit. The resulting amplification consumed about
|
||||
2.60 GiB of heap before the first model call, exceeding the service's roughly 2.20 GiB V8 heap
|
||||
limit. This was an in-process memory amplification bug, not an XLSX multi-sheet timeout.
|
||||
|
||||
The affected code is shared by every parser format. A large HTML table, OCR-heavy PDF, or Office
|
||||
document could therefore trigger the same failure even when its uploaded file was small.
|
||||
|
||||
## Safety boundaries
|
||||
|
||||
- Fragment metadata has a 16 KiB serialized budget; complete-element metadata has a 256 KiB
|
||||
serialized budget. Omitted fields are recorded deterministically in `sourceMetadataProjection`.
|
||||
- Parser metadata remains available on the canonical parse artifact; the projection only controls
|
||||
repeated knowledge-node copies and does not discard source evidence.
|
||||
- Compact multimodal references (`assetRef`), bounding boxes, captions, and titles remain eligible
|
||||
on fragments, preserving image display, citation location, and retrieval provenance.
|
||||
- Semantic prompt contents, chunk text, offsets, window sizes, and model-call behavior are unchanged.
|
||||
|
||||
## Measured regression
|
||||
|
||||
On the production-sized synthetic artifact (631,113 text characters and 797,526 HTML characters):
|
||||
|
||||
- semantic preflight produced 526 atomic units and 132 windows under a 128 MiB V8 heap;
|
||||
- deterministic chunking produced 526 nodes under the same 128 MiB heap;
|
||||
- the semantic preflight process reported about 32 MiB heap in use after completion.
|
||||
|
||||
These are local regression measurements, not projected production latency or throughput numbers.
|
||||
|
||||
## Verification
|
||||
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/api exec vitest run --reporter=dot` — 4,650 passed,
|
||||
3 skipped.
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/core test:coverage` — 61 passed; package coverage
|
||||
remained above 90% in every dimension.
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/compute test:coverage` — 25 passed; package coverage
|
||||
remained above 90% in every dimension.
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/parsers test:coverage` — 63 passed; package coverage
|
||||
remained above 90% in every dimension.
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/api-app test` — 262 passed.
|
||||
- KnowledgeFS Core, Compute, Parser, and API typechecks passed.
|
||||
- `pnpm --dir knowledge-fs lint:backend` passed.
|
||||
|
||||
## Remaining operational note
|
||||
|
||||
The pipeline still intentionally holds one canonical parse artifact and bounded model windows in
|
||||
memory. Existing upload-size, parser-response, element-count, node-count, model-window, PDF raster,
|
||||
image-byte, and concurrency limits remain the admission boundaries for unusually large documents.
|
||||
This change removes the unbounded multiplication by fragment count; it does not claim constant
|
||||
memory independent of the admitted document size.
|
||||
@ -179,6 +179,38 @@ describe("document outline builder", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("builds a bounded summary for a large table without joining the complete payload", () => {
|
||||
const builder = createDocumentOutlineBuilder({
|
||||
generateId: sequenceIds([
|
||||
"018f0d60-7a49-7cc2-9c1b-5b36f18f2c50",
|
||||
"018f0d60-7a49-7cc2-9c1b-5b36f18f2c51",
|
||||
]),
|
||||
maxElements: 20,
|
||||
maxNodes: 10,
|
||||
maxSummaryChars: 80,
|
||||
now: () => createdAt,
|
||||
});
|
||||
const text = `表头\n${"字段值 ".repeat(300_000)}`;
|
||||
|
||||
const outline = builder.build({
|
||||
knowledgeSpaceId,
|
||||
parseArtifact: parseArtifact([
|
||||
{
|
||||
id: "large-table",
|
||||
sectionPath: ["知识库"],
|
||||
text,
|
||||
type: "table",
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
expect(outline.nodes[0]?.summary).toHaveLength(80);
|
||||
expect(outline.nodes[0]?.summary?.endsWith("...")).toBe(true);
|
||||
expect(outline.nodes[0]?.metadata).toMatchObject({
|
||||
canonicalCharacterCount: text.trim().length,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the same normalized UTF-8 byte coordinates as artifact segments and chunking", () => {
|
||||
const builder = createDocumentOutlineBuilder({
|
||||
generateId: sequenceIds([
|
||||
|
||||
@ -352,7 +352,7 @@ function createOutlineDraft({
|
||||
}
|
||||
|
||||
function applySpanToDraft(draft: OutlineNodeDraft, span: ElementSpan): void {
|
||||
draft.characterCount += Array.from(span.text).length;
|
||||
draft.characterCount += countCodePoints(span.text);
|
||||
draft.startOffset =
|
||||
draft.startOffset === undefined
|
||||
? span.startOffset
|
||||
@ -555,13 +555,13 @@ function summarizeOutlineTexts({
|
||||
readonly texts: readonly string[];
|
||||
readonly title: string;
|
||||
}): string {
|
||||
const ownText = texts.join(" ").replaceAll(/\s+/gu, " ").trim();
|
||||
const childSummary = children
|
||||
.map((child) => child.summary)
|
||||
.filter((summary): summary is string => Boolean(summary?.trim()))
|
||||
.join(" ")
|
||||
.replaceAll(/\s+/gu, " ")
|
||||
.trim();
|
||||
const ownText = normalizedTextPrefix(texts, maxSummaryChars + 1);
|
||||
const childSummary = ownText
|
||||
? ""
|
||||
: normalizedTextPrefix(
|
||||
children.flatMap((child) => (child.summary?.trim() ? [child.summary] : [])),
|
||||
maxSummaryChars + 1,
|
||||
);
|
||||
const summary = ownText || childSummary || title;
|
||||
|
||||
if (summary.length <= maxSummaryChars) {
|
||||
@ -575,6 +575,39 @@ function summarizeOutlineTexts({
|
||||
return `${summary.slice(0, maxSummaryChars - 3)}...`;
|
||||
}
|
||||
|
||||
function normalizedTextPrefix(texts: readonly string[], maxChars: number): string {
|
||||
let result = "";
|
||||
let pendingSpace = false;
|
||||
const whitespace = /\s/u;
|
||||
|
||||
outer: for (const text of texts) {
|
||||
if (result.length > 0) {
|
||||
pendingSpace = true;
|
||||
}
|
||||
for (const character of text) {
|
||||
if (whitespace.test(character)) {
|
||||
if (result.length > 0) pendingSpace = true;
|
||||
continue;
|
||||
}
|
||||
if (pendingSpace) {
|
||||
if (result.length > 0) result += " ";
|
||||
pendingSpace = false;
|
||||
if (result.length >= maxChars) break outer;
|
||||
}
|
||||
result += character;
|
||||
if (result.length >= maxChars) break outer;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function countCodePoints(text: string): number {
|
||||
let count = 0;
|
||||
for (const _character of text) count += 1;
|
||||
return count;
|
||||
}
|
||||
|
||||
function outlineSectionKey(sectionPath: readonly string[]): string {
|
||||
return sectionPath.join(sectionPathSeparator);
|
||||
}
|
||||
|
||||
@ -112,6 +112,38 @@ describe("document outline summary enhancer", () => {
|
||||
).toThrow("Document outline summary maxSummaryChars must be at least 1");
|
||||
});
|
||||
|
||||
it("materializes only the admitted prefix of a large section", async () => {
|
||||
const synthetic = largeOutline(1);
|
||||
const calls: Parameters<DocumentOutlineSummaryProvider["summarize"]>[0][] = [];
|
||||
const artifact: ParseArtifact = {
|
||||
...synthetic.artifact,
|
||||
elements: [
|
||||
{
|
||||
...(synthetic.artifact.elements[0] as ParseArtifact["elements"][number]),
|
||||
text: "长文本".repeat(500_000),
|
||||
},
|
||||
],
|
||||
};
|
||||
const enhancer = createDocumentOutlineSummaryEnhancer({
|
||||
maxInputChars: 80,
|
||||
maxSummaryChars: 40,
|
||||
model: "outline-summary-model",
|
||||
promptVersion: "document-outline-summary-v1",
|
||||
provider: {
|
||||
summarize: async (input) => {
|
||||
calls.push(input);
|
||||
return { summary: "bounded" };
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await enhancer.enhance({ outline: synthetic.outline, parseArtifact: artifact });
|
||||
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]?.text).toHaveLength(80);
|
||||
expect(calls[0]?.text.endsWith("...")).toBe(true);
|
||||
});
|
||||
|
||||
it("bounds provider concurrency across independent outline branches", async () => {
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
|
||||
@ -628,7 +628,7 @@ function summaryInput({
|
||||
promptVersion,
|
||||
sectionPath: [...node.sectionPath],
|
||||
...(signal ? { signal } : {}),
|
||||
text: truncateText(sectionText(artifact, node), maxInputChars),
|
||||
text: sectionText(artifact, node, maxInputChars),
|
||||
title: node.title,
|
||||
...(traceId ? { traceId } : {}),
|
||||
};
|
||||
@ -804,12 +804,37 @@ function applySummaryResults({
|
||||
});
|
||||
}
|
||||
|
||||
function sectionText(artifact: ParseArtifact, node: DocumentOutlineNode): string {
|
||||
return artifact.elements
|
||||
.filter((element) => elementSectionStartsWith(element.sectionPath, node.sectionPath))
|
||||
.map((element) => element.text?.trim() ?? "")
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
function sectionText(artifact: ParseArtifact, node: DocumentOutlineNode, maxChars: number): string {
|
||||
// Build only the admitted prefix. Joining an entire large table/HTML section before truncating
|
||||
// multiplies the document size by the number of ancestor outline nodes.
|
||||
let text = "";
|
||||
let selectedElements = 0;
|
||||
const materializationLimit = maxChars + 1;
|
||||
|
||||
for (const element of artifact.elements) {
|
||||
if (!elementSectionStartsWith(element.sectionPath, node.sectionPath)) {
|
||||
continue;
|
||||
}
|
||||
const elementText = element.text?.trim() ?? "";
|
||||
if (!elementText) {
|
||||
continue;
|
||||
}
|
||||
if (selectedElements > 0) {
|
||||
const separator = "\n\n".slice(0, materializationLimit - text.length);
|
||||
text += separator;
|
||||
}
|
||||
const remaining = materializationLimit - text.length;
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
text += elementText.slice(0, remaining);
|
||||
selectedElements += 1;
|
||||
if (elementText.length > remaining || text.length >= materializationLimit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return truncateText(text, maxChars);
|
||||
}
|
||||
|
||||
function elementSectionStartsWith(
|
||||
|
||||
@ -0,0 +1,49 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("LLM semantic chunker memory admission", () => {
|
||||
it("preflights a production-sized spreadsheet artifact under a 128 MiB V8 heap", () => {
|
||||
const coreUrl = new URL("../../core/src/index.ts", import.meta.url).href;
|
||||
const chunkerUrl = new URL("./llm-semantic-chunker.ts", import.meta.url).href;
|
||||
const script = `
|
||||
import { ParseArtifactSchema } from ${JSON.stringify(coreUrl)};
|
||||
import { preflightLlmSemanticWindows } from ${JSON.stringify(chunkerUrl)};
|
||||
const text = "行数据字段值。".repeat(90_159);
|
||||
const html = \`<table>\${"x".repeat(797_511)}</table>\`;
|
||||
const parseArtifact = ParseArtifactSchema.parse({
|
||||
artifactHash: "a".repeat(64),
|
||||
contentType: "structured",
|
||||
createdAt: "2026-08-26T00:00:00.000Z",
|
||||
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42",
|
||||
elements: [{
|
||||
id: "sheet-1",
|
||||
metadata: { table: { html }, textAsHtml: html, text_as_html: html },
|
||||
sectionPath: ["知识库"],
|
||||
text,
|
||||
type: "table",
|
||||
}],
|
||||
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43",
|
||||
metadata: {},
|
||||
parser: "unstructured",
|
||||
version: 1,
|
||||
});
|
||||
const result = preflightLlmSemanticWindows({ parseArtifact });
|
||||
console.log(JSON.stringify({ ...result, heapUsed: process.memoryUsage().heapUsed }));
|
||||
`;
|
||||
const completed = spawnSync(
|
||||
process.execPath,
|
||||
["--max-old-space-size=128", "--import", "tsx", "--input-type=module", "-e", script],
|
||||
{ encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 15_000 },
|
||||
);
|
||||
|
||||
expect(completed.status, completed.stderr).toBe(0);
|
||||
const result = JSON.parse(completed.stdout.trim()) as {
|
||||
heapUsed: number;
|
||||
maximumWindowCount: number;
|
||||
unitCount: number;
|
||||
};
|
||||
expect(result).toMatchObject({ maximumWindowCount: 132, unitCount: 526 });
|
||||
expect(result.heapUsed).toBeLessThan(128 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
@ -371,6 +371,59 @@ describe("LLM semantic chunker", () => {
|
||||
expect(nodes[1]?.endOffset).toBe(new TextEncoder().encode(text).byteLength);
|
||||
});
|
||||
|
||||
it("does not amplify large parser metadata across fragmented table units", async () => {
|
||||
const largeTableHtml = `<table>${"<tr><td>value</td></tr>".repeat(24_000)}</table>`;
|
||||
const text = Array.from(
|
||||
{ length: 240 },
|
||||
(_, index) => `第 ${index + 1} 行 | 字段 ${index + 1} | 这是用于检索的表格内容`,
|
||||
).join("\n");
|
||||
const parseArtifact = artifact([
|
||||
{
|
||||
id: "large-spreadsheet-table",
|
||||
metadata: {
|
||||
assetRef: { objectKey: "assets/spreadsheet-table.png" },
|
||||
table: { html: largeTableHtml },
|
||||
textAsHtml: largeTableHtml,
|
||||
title: "知识库",
|
||||
},
|
||||
sectionPath: ["知识库"],
|
||||
text,
|
||||
type: "table",
|
||||
},
|
||||
]);
|
||||
const config = { maxChunkChars: 80, maxWindowChars: 320 } as const;
|
||||
const preflight = preflightLlmSemanticWindows({ config, parseArtifact });
|
||||
const provider = new ScriptedProvider([echoEachUnit]);
|
||||
const nodes = await createLlmSemanticChunker({
|
||||
...config,
|
||||
reasoningProviderFactory: () => provider,
|
||||
}).chunk({
|
||||
knowledgeSpaceId: KNOWLEDGE_SPACE_ID,
|
||||
parseArtifact,
|
||||
retrievalProfile: profile(),
|
||||
});
|
||||
|
||||
expect(preflight.unitCount).toBeGreaterThan(20);
|
||||
expect(nodes).toHaveLength(preflight.unitCount);
|
||||
expect(nodes.map((node) => node.text).join("")).toBe(text);
|
||||
expect(nodes.at(-1)?.endOffset).toBe(new TextEncoder().encode(text).byteLength);
|
||||
for (const node of nodes) {
|
||||
expect(node.metadata).toMatchObject({
|
||||
assetRef: { objectKey: "assets/spreadsheet-table.png" },
|
||||
sourceMetadataProjection: {
|
||||
completeElement: false,
|
||||
omitted: [
|
||||
{ field: "table", reason: "fragmented-source-element" },
|
||||
{ field: "textAsHtml", reason: "fragmented-source-element" },
|
||||
],
|
||||
},
|
||||
title: "知识库",
|
||||
});
|
||||
expect(node.metadata).not.toHaveProperty("table");
|
||||
expect(node.metadata).not.toHaveProperty("textAsHtml");
|
||||
}
|
||||
});
|
||||
|
||||
it("records legacy overlap as unapplied provenance and keeps semantic chunks contiguous", async () => {
|
||||
const chunker = createLlmSemanticChunker({
|
||||
reasoningProviderFactory: () => new ScriptedProvider([echoEachUnit]),
|
||||
|
||||
@ -9,6 +9,7 @@ import {
|
||||
type KnowledgeSpaceRetrievalProfile,
|
||||
type ParseArtifact,
|
||||
emptyImageElementIndexText,
|
||||
knowledgeNodeSourceMetadataWithProjection,
|
||||
stableJson,
|
||||
} from "@knowledge/core";
|
||||
import {
|
||||
@ -243,7 +244,7 @@ interface EffectiveChunkConfig {
|
||||
interface MaterializedElement {
|
||||
readonly elementId: string;
|
||||
readonly elementIndex: number;
|
||||
readonly elementMetadata: Record<string, unknown>;
|
||||
readonly elementMetadata: Readonly<Record<string, unknown>>;
|
||||
readonly elementType: ParseArtifact["elements"][number]["type"];
|
||||
readonly endCodeUnit: number;
|
||||
readonly endOffset: number;
|
||||
@ -256,7 +257,6 @@ interface MaterializedElement {
|
||||
|
||||
interface AtomicUnit {
|
||||
readonly elementId: string;
|
||||
readonly elementMetadata: Record<string, unknown>;
|
||||
readonly elementType: ParseArtifact["elements"][number]["type"];
|
||||
readonly endCodeUnit: number;
|
||||
readonly endOffset: number;
|
||||
@ -265,6 +265,8 @@ interface AtomicUnit {
|
||||
readonly isolationKey?: string | undefined;
|
||||
readonly pageNumber?: number | undefined;
|
||||
readonly sectionPath: readonly string[];
|
||||
/** Shared source element reference; never clone parser metadata per atomic unit. */
|
||||
readonly sourceElement: MaterializedElement;
|
||||
readonly startCodeUnit: number;
|
||||
readonly startOffset: number;
|
||||
readonly text: string;
|
||||
@ -1692,7 +1694,7 @@ function materializeElements(parseArtifact: ParseArtifact): {
|
||||
elements.push({
|
||||
elementId: element.id,
|
||||
elementIndex,
|
||||
elementMetadata: cloneJsonObject(element.metadata),
|
||||
elementMetadata: element.metadata,
|
||||
elementType: element.type,
|
||||
endCodeUnit: canonicalText.length,
|
||||
endOffset: span.endOffset,
|
||||
@ -1720,17 +1722,26 @@ function materializeAtomicUnits(
|
||||
for (const element of elements) {
|
||||
const sentenceRanges = semanticRanges(element);
|
||||
let atomicIndex = 0;
|
||||
let byteCursorCodeUnit = 0;
|
||||
let byteCursorOffset = element.startOffset;
|
||||
for (const range of sentenceRanges) {
|
||||
const sentence = element.text.slice(range.start, range.end);
|
||||
for (const hardRange of graphemeRanges(sentence, maxChunkChars)) {
|
||||
const localStart = range.start + hardRange.start;
|
||||
const localEnd = range.start + hardRange.end;
|
||||
const text = element.text.slice(localStart, localEnd);
|
||||
const startOffset = element.startOffset + utf8ByteLength(element.text.slice(0, localStart));
|
||||
const endOffset = element.startOffset + utf8ByteLength(element.text.slice(0, localEnd));
|
||||
if (localStart < byteCursorCodeUnit) {
|
||||
throw new Error("LLM semantic chunking atomic units must preserve source order");
|
||||
}
|
||||
if (localStart > byteCursorCodeUnit) {
|
||||
byteCursorOffset += utf8ByteLength(element.text.slice(byteCursorCodeUnit, localStart));
|
||||
}
|
||||
const startOffset = byteCursorOffset;
|
||||
const endOffset = startOffset + utf8ByteLength(text);
|
||||
byteCursorCodeUnit = localEnd;
|
||||
byteCursorOffset = endOffset;
|
||||
units.push({
|
||||
elementId: element.elementId,
|
||||
elementMetadata: cloneJsonObject(element.elementMetadata),
|
||||
elementType: element.elementType,
|
||||
endCodeUnit: element.startCodeUnit + localEnd,
|
||||
endOffset,
|
||||
@ -1742,7 +1753,8 @@ function materializeAtomicUnits(
|
||||
? { isolationKey: `${element.elementType}:${element.elementId}` }
|
||||
: {}),
|
||||
...(element.pageNumber === undefined ? {} : { pageNumber: element.pageNumber }),
|
||||
sectionPath: [...element.sectionPath],
|
||||
sectionPath: element.sectionPath,
|
||||
sourceElement: element,
|
||||
startCodeUnit: element.startCodeUnit + localStart,
|
||||
startOffset,
|
||||
text,
|
||||
@ -2701,7 +2713,16 @@ function materializeKnowledgeNode({
|
||||
textNormalization: DOCUMENT_ELEMENT_TEXT_NORMALIZATION,
|
||||
};
|
||||
if (uniqueStrings(chunk.units.map((unit) => unit.elementId)).length === 1) {
|
||||
mergeSingleElementMetadata(metadata, first.elementMetadata);
|
||||
const completeElement =
|
||||
first.sourceElement === last.sourceElement &&
|
||||
first.startCodeUnit === first.sourceElement.startCodeUnit &&
|
||||
last.endCodeUnit === first.sourceElement.endCodeUnit;
|
||||
Object.assign(
|
||||
metadata,
|
||||
knowledgeNodeSourceMetadataWithProjection(first.sourceElement.elementMetadata, {
|
||||
completeElement,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const pageNumber = commonPageNumber(chunk.units);
|
||||
|
||||
@ -2791,25 +2812,6 @@ function commonPageNumber(units: readonly AtomicUnit[]): number | undefined {
|
||||
return units.every((unit) => unit.pageNumber === pageNumber) ? pageNumber : undefined;
|
||||
}
|
||||
|
||||
function mergeSingleElementMetadata(
|
||||
target: Record<string, unknown>,
|
||||
source: Readonly<Record<string, unknown>>,
|
||||
): void {
|
||||
for (const key of [
|
||||
"assetRef",
|
||||
"boundingBox",
|
||||
"caption",
|
||||
"ocrText",
|
||||
"table",
|
||||
"textAsHtml",
|
||||
"title",
|
||||
]) {
|
||||
if (Object.hasOwn(source, key)) {
|
||||
target[key] = JSON.parse(JSON.stringify(source[key])) as unknown;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validatePositiveInteger(name: string, value: number): void {
|
||||
if (!Number.isInteger(value) || value < 1) {
|
||||
throw new Error(`LLM semantic chunking ${name} must be at least 1`);
|
||||
|
||||
57
knowledge-fs/packages/compute/src/compute-memory.test.ts
Normal file
57
knowledge-fs/packages/compute/src/compute-memory.test.ts
Normal file
@ -0,0 +1,57 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
describe("deterministic chunker memory admission", () => {
|
||||
it("chunks a production-sized spreadsheet artifact under a 128 MiB V8 heap", () => {
|
||||
const coreUrl = new URL("../../core/src/index.ts", import.meta.url).href;
|
||||
const computeUrl = new URL("./index.ts", import.meta.url).href;
|
||||
const script = `
|
||||
import { ParseArtifactSchema } from ${JSON.stringify(coreUrl)};
|
||||
import { createTypeScriptComputeRuntime } from ${JSON.stringify(computeUrl)};
|
||||
const text = "行数据字段值。".repeat(90_159);
|
||||
const html = \`<table>\${"x".repeat(797_511)}</table>\`;
|
||||
const parseArtifact = ParseArtifactSchema.parse({
|
||||
artifactHash: "a".repeat(64),
|
||||
contentType: "structured",
|
||||
createdAt: "2026-08-26T00:00:00.000Z",
|
||||
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42",
|
||||
elements: [{
|
||||
id: "sheet-1",
|
||||
metadata: { table: { html }, textAsHtml: html, text_as_html: html },
|
||||
sectionPath: ["知识库"],
|
||||
text,
|
||||
type: "table",
|
||||
}],
|
||||
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43",
|
||||
metadata: {},
|
||||
parser: "unstructured",
|
||||
version: 1,
|
||||
});
|
||||
const nodes = createTypeScriptComputeRuntime().chunkParseArtifact({
|
||||
config: { overlapChars: 0 },
|
||||
knowledgeSpaceId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
|
||||
parseArtifact,
|
||||
});
|
||||
console.log(JSON.stringify({
|
||||
heapUsed: process.memoryUsage().heapUsed,
|
||||
lastEndOffset: nodes.at(-1)?.endOffset,
|
||||
nodeCount: nodes.length,
|
||||
}));
|
||||
`;
|
||||
const completed = spawnSync(
|
||||
process.execPath,
|
||||
["--max-old-space-size=128", "--import", "tsx", "--input-type=module", "-e", script],
|
||||
{ encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 15_000 },
|
||||
);
|
||||
|
||||
expect(completed.status, completed.stderr).toBe(0);
|
||||
const result = JSON.parse(completed.stdout.trim()) as {
|
||||
heapUsed: number;
|
||||
lastEndOffset: number;
|
||||
nodeCount: number;
|
||||
};
|
||||
expect(result).toMatchObject({ lastEndOffset: 1_893_339, nodeCount: 526 });
|
||||
expect(result.heapUsed).toBeLessThan(128 * 1024 * 1024);
|
||||
});
|
||||
});
|
||||
@ -293,6 +293,47 @@ describe("createTypeScriptComputeRuntime", () => {
|
||||
expect(nodes[2]?.metadata).toMatchObject({ table: { columns: 2 }, textAsHtml: "<table />" });
|
||||
});
|
||||
|
||||
it("does not copy large table metadata onto every deterministic fragment", () => {
|
||||
const tableHtml = `<table>${"<tr><td>value</td></tr>".repeat(16_000)}</table>`;
|
||||
const text = "表格字段".repeat(40);
|
||||
const nodes = runtime.chunkParseArtifact({
|
||||
config: { maxChunkChars: 16, overlapChars: 0 },
|
||||
knowledgeSpaceId,
|
||||
parseArtifact: artifact([
|
||||
{
|
||||
id: "large-table",
|
||||
metadata: {
|
||||
assetRef: { objectKey: "space/large-table.png" },
|
||||
table: { html: tableHtml },
|
||||
textAsHtml: tableHtml,
|
||||
title: "知识库",
|
||||
},
|
||||
sectionPath: ["知识库"],
|
||||
text,
|
||||
type: "table",
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
expect(nodes).toHaveLength(10);
|
||||
expect(nodes.map((node) => node.text).join("")).toBe(text);
|
||||
for (const node of nodes) {
|
||||
expect(node.metadata).toMatchObject({
|
||||
assetRef: { objectKey: "space/large-table.png" },
|
||||
sourceMetadataProjection: {
|
||||
completeElement: false,
|
||||
omitted: [
|
||||
{ field: "table", reason: "fragmented-source-element" },
|
||||
{ field: "textAsHtml", reason: "fragmented-source-element" },
|
||||
],
|
||||
},
|
||||
title: "知识库",
|
||||
});
|
||||
expect(node.metadata).not.toHaveProperty("table");
|
||||
expect(node.metadata).not.toHaveProperty("textAsHtml");
|
||||
}
|
||||
});
|
||||
|
||||
it("returns independent output objects and enforces all configured bounds", () => {
|
||||
const input = {
|
||||
config: { maxChunkChars: 120, overlapChars: 0 },
|
||||
|
||||
@ -6,6 +6,7 @@ import {
|
||||
type ParseArtifact,
|
||||
ParseArtifactSchema,
|
||||
emptyImageElementIndexText,
|
||||
knowledgeNodeSourceMetadataWithProjection,
|
||||
} from "@knowledge/core";
|
||||
import { isAlphabetic } from "unicode-segmenter/general";
|
||||
import {
|
||||
@ -275,9 +276,11 @@ interface TextSegment {
|
||||
elementType: string;
|
||||
endOffset: number;
|
||||
graphemeLength: number;
|
||||
metadata: Record<string, unknown>;
|
||||
metadata: Readonly<Record<string, unknown>>;
|
||||
pageNumber?: number | undefined;
|
||||
sectionPath: string[];
|
||||
sourceEndOffset: number;
|
||||
sourceStartOffset: number;
|
||||
startOffset: number;
|
||||
text: string;
|
||||
}
|
||||
@ -452,7 +455,14 @@ function chunkParseArtifact(input: ChunkParseArtifactInput): KnowledgeNode[] {
|
||||
textNormalization: DOCUMENT_ELEMENT_TEXT_NORMALIZATION,
|
||||
};
|
||||
if (nodeSegments.length === 1) {
|
||||
mergeSingleSegmentMetadata(metadata, first.metadata);
|
||||
Object.assign(
|
||||
metadata,
|
||||
knowledgeNodeSourceMetadataWithProjection(first.metadata, {
|
||||
completeElement:
|
||||
first.startOffset === first.sourceStartOffset &&
|
||||
first.endOffset === first.sourceEndOffset,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const pageNumber = commonPageNumber(nodeSegments);
|
||||
|
||||
@ -549,9 +559,11 @@ function materializeSegments(parseArtifact: ParseArtifact): TextSegment[] {
|
||||
elementType: element.type,
|
||||
endOffset,
|
||||
graphemeLength: countGraphemes(text),
|
||||
metadata: jsonClone(element.metadata),
|
||||
metadata: element.metadata,
|
||||
...(element.pageNumber === undefined ? {} : { pageNumber: element.pageNumber }),
|
||||
sectionPath: [...element.sectionPath],
|
||||
sourceEndOffset: endOffset,
|
||||
sourceStartOffset: startOffset,
|
||||
startOffset,
|
||||
text,
|
||||
});
|
||||
@ -907,25 +919,6 @@ function toGraphemeSpan(boundaries: GraphemeBoundary[]): GraphemeSpan {
|
||||
};
|
||||
}
|
||||
|
||||
function mergeSingleSegmentMetadata(
|
||||
target: Record<string, unknown>,
|
||||
source: Record<string, unknown>,
|
||||
): void {
|
||||
for (const key of [
|
||||
"assetRef",
|
||||
"boundingBox",
|
||||
"caption",
|
||||
"ocrText",
|
||||
"table",
|
||||
"textAsHtml",
|
||||
"title",
|
||||
]) {
|
||||
if (Object.hasOwn(source, key)) {
|
||||
target[key] = jsonClone(source[key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function commonPageNumber(segments: TextSegment[]): number | undefined {
|
||||
const first = (segments[0] as TextSegment).pageNumber;
|
||||
return segments.every((segment) => segment.pageNumber === first) ? first : undefined;
|
||||
|
||||
@ -3,3 +3,4 @@ export * from "./models";
|
||||
export * from "./command-registry";
|
||||
export * from "./json-utils";
|
||||
export * from "./document-element-index-text";
|
||||
export * from "./source-element-metadata";
|
||||
|
||||
111
knowledge-fs/packages/core/src/source-element-metadata.test.ts
Normal file
111
knowledge-fs/packages/core/src/source-element-metadata.test.ts
Normal file
@ -0,0 +1,111 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
knowledgeNodeSourceMetadataWithProjection,
|
||||
projectParseElementMetadataForKnowledgeNode,
|
||||
} from "./source-element-metadata";
|
||||
|
||||
describe("knowledge node source metadata projection", () => {
|
||||
it("keeps compact reference fields but omits amplified payloads from source fragments", () => {
|
||||
const metadata = knowledgeNodeSourceMetadataWithProjection(
|
||||
{
|
||||
assetRef: { objectKey: "assets/table.png" },
|
||||
boundingBox: { height: 20, width: 40, x: 1, y: 2 },
|
||||
caption: "Quarterly metrics",
|
||||
ocrText: "large OCR payload",
|
||||
table: { html: "<table>large table</table>" },
|
||||
textAsHtml: "<table>large table</table>",
|
||||
title: "Metrics",
|
||||
},
|
||||
{ completeElement: false },
|
||||
);
|
||||
|
||||
expect(metadata).toMatchObject({
|
||||
assetRef: { objectKey: "assets/table.png" },
|
||||
boundingBox: { height: 20, width: 40, x: 1, y: 2 },
|
||||
caption: "Quarterly metrics",
|
||||
sourceMetadataProjection: {
|
||||
completeElement: false,
|
||||
omitted: [
|
||||
{ field: "ocrText", reason: "fragmented-source-element" },
|
||||
{ field: "table", reason: "fragmented-source-element" },
|
||||
{ field: "textAsHtml", reason: "fragmented-source-element" },
|
||||
],
|
||||
},
|
||||
title: "Metrics",
|
||||
});
|
||||
expect(metadata).not.toHaveProperty("ocrText");
|
||||
expect(metadata).not.toHaveProperty("table");
|
||||
expect(metadata).not.toHaveProperty("textAsHtml");
|
||||
});
|
||||
|
||||
it("retains complete element metadata within the byte budget and clones output values", () => {
|
||||
const source = {
|
||||
table: { rows: [{ metric: "ARR", value: 42 }] },
|
||||
textAsHtml: "<table><tr><td>ARR</td></tr></table>",
|
||||
title: "Metrics",
|
||||
};
|
||||
const projected = projectParseElementMetadataForKnowledgeNode(source, {
|
||||
completeElement: true,
|
||||
});
|
||||
|
||||
expect(projected.omissions).toEqual([]);
|
||||
expect(projected.metadata).toEqual(source);
|
||||
expect(projected.metadata.table).not.toBe(source.table);
|
||||
});
|
||||
|
||||
it("omits oversized complete-element fields deterministically", () => {
|
||||
const metadata = knowledgeNodeSourceMetadataWithProjection(
|
||||
{
|
||||
assetRef: { objectKey: "assets/chart.png" },
|
||||
table: { html: "x".repeat(1_000) },
|
||||
title: "Chart",
|
||||
},
|
||||
{ completeElement: true, maxBytes: 128 },
|
||||
);
|
||||
|
||||
expect(metadata).toMatchObject({
|
||||
assetRef: { objectKey: "assets/chart.png" },
|
||||
sourceMetadataProjection: {
|
||||
completeElement: true,
|
||||
maxBytes: 128,
|
||||
omitted: [{ field: "table", reason: "size-limit" }],
|
||||
},
|
||||
title: "Chart",
|
||||
});
|
||||
expect(metadata).not.toHaveProperty("table");
|
||||
});
|
||||
|
||||
it("uses the smaller default budget for metadata repeated across fragments", () => {
|
||||
const metadata = knowledgeNodeSourceMetadataWithProjection(
|
||||
{
|
||||
assetRef: { objectKey: "assets/chart.png" },
|
||||
caption: "x".repeat(20_000),
|
||||
title: "Chart",
|
||||
},
|
||||
{ completeElement: false },
|
||||
);
|
||||
|
||||
expect(metadata).toMatchObject({
|
||||
assetRef: { objectKey: "assets/chart.png" },
|
||||
sourceMetadataProjection: {
|
||||
completeElement: false,
|
||||
maxBytes: 16 * 1024,
|
||||
omitted: [{ field: "caption", reason: "size-limit" }],
|
||||
},
|
||||
title: "Chart",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an invalid projection budget before serializing fields", () => {
|
||||
expect(() =>
|
||||
projectParseElementMetadataForKnowledgeNode(
|
||||
{ table: { rows: 1 } },
|
||||
{
|
||||
completeElement: true,
|
||||
maxBytes: 0,
|
||||
},
|
||||
),
|
||||
).toThrow("maxBytes must be at least 1");
|
||||
});
|
||||
});
|
||||
120
knowledge-fs/packages/core/src/source-element-metadata.ts
Normal file
120
knowledge-fs/packages/core/src/source-element-metadata.ts
Normal file
@ -0,0 +1,120 @@
|
||||
export const MAX_KNOWLEDGE_NODE_SOURCE_METADATA_BYTES = 256 * 1024;
|
||||
export const MAX_KNOWLEDGE_NODE_FRAGMENT_SOURCE_METADATA_BYTES = 16 * 1024;
|
||||
|
||||
const FRAGMENT_SAFE_SOURCE_METADATA_KEYS = new Set(["assetRef", "boundingBox", "caption", "title"]);
|
||||
const KNOWLEDGE_NODE_SOURCE_METADATA_KEYS = [
|
||||
"assetRef",
|
||||
"boundingBox",
|
||||
"caption",
|
||||
"title",
|
||||
"ocrText",
|
||||
"table",
|
||||
"textAsHtml",
|
||||
] as const;
|
||||
|
||||
export type SourceMetadataOmissionReason = "fragmented-source-element" | "size-limit";
|
||||
|
||||
export interface SourceMetadataOmission {
|
||||
readonly field: string;
|
||||
readonly reason: SourceMetadataOmissionReason;
|
||||
}
|
||||
|
||||
export interface KnowledgeNodeSourceMetadataProjection {
|
||||
readonly metadata: Record<string, unknown>;
|
||||
readonly omissions: readonly SourceMetadataOmission[];
|
||||
readonly projectedBytes: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Projects parser element metadata onto a knowledge node without amplifying large source payloads.
|
||||
*
|
||||
* A parser may attach the complete OCR text or HTML representation of a table to one source
|
||||
* element. Copying that payload onto every chunk produced from the element turns a linear parse
|
||||
* artifact into an O(chunks * metadata) allocation. Fragment nodes therefore retain only the
|
||||
* compact location/reference fields. Complete elements may retain the richer fields, but only
|
||||
* within a deterministic serialized-byte budget.
|
||||
*/
|
||||
export function projectParseElementMetadataForKnowledgeNode(
|
||||
source: Readonly<Record<string, unknown>>,
|
||||
{
|
||||
completeElement,
|
||||
maxBytes = completeElement
|
||||
? MAX_KNOWLEDGE_NODE_SOURCE_METADATA_BYTES
|
||||
: MAX_KNOWLEDGE_NODE_FRAGMENT_SOURCE_METADATA_BYTES,
|
||||
}: {
|
||||
readonly completeElement: boolean;
|
||||
readonly maxBytes?: number | undefined;
|
||||
},
|
||||
): KnowledgeNodeSourceMetadataProjection {
|
||||
if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) {
|
||||
throw new Error("Knowledge node source metadata maxBytes must be at least 1");
|
||||
}
|
||||
|
||||
const metadata: Record<string, unknown> = {};
|
||||
const omissions: SourceMetadataOmission[] = [];
|
||||
let projectedBytes = 2; // Opening and closing braces of the projected JSON object.
|
||||
let projectedFields = 0;
|
||||
|
||||
for (const field of KNOWLEDGE_NODE_SOURCE_METADATA_KEYS) {
|
||||
if (!Object.hasOwn(source, field) || source[field] === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (!completeElement && !FRAGMENT_SAFE_SOURCE_METADATA_KEYS.has(field)) {
|
||||
omissions.push({ field, reason: "fragmented-source-element" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const serialized = JSON.stringify({ [field]: source[field] });
|
||||
if (serialized === undefined || serialized === "{}") {
|
||||
continue;
|
||||
}
|
||||
const entryBytes = utf8ByteLength(serialized) - 2 + (projectedFields === 0 ? 0 : 1);
|
||||
if (projectedBytes + entryBytes > maxBytes) {
|
||||
omissions.push({ field, reason: "size-limit" });
|
||||
continue;
|
||||
}
|
||||
|
||||
Object.assign(metadata, JSON.parse(serialized) as Record<string, unknown>);
|
||||
projectedBytes += entryBytes;
|
||||
projectedFields += 1;
|
||||
}
|
||||
|
||||
return { metadata, omissions, projectedBytes };
|
||||
}
|
||||
|
||||
/** Adds compact, deterministic provenance only when source metadata had to be omitted. */
|
||||
export function knowledgeNodeSourceMetadataWithProjection(
|
||||
source: Readonly<Record<string, unknown>>,
|
||||
options: {
|
||||
readonly completeElement: boolean;
|
||||
readonly maxBytes?: number | undefined;
|
||||
},
|
||||
): Record<string, unknown> {
|
||||
const projection = projectParseElementMetadataForKnowledgeNode(source, options);
|
||||
if (projection.omissions.length === 0) {
|
||||
return projection.metadata;
|
||||
}
|
||||
|
||||
return {
|
||||
...projection.metadata,
|
||||
sourceMetadataProjection: {
|
||||
completeElement: options.completeElement,
|
||||
maxBytes:
|
||||
options.maxBytes ??
|
||||
(options.completeElement
|
||||
? MAX_KNOWLEDGE_NODE_SOURCE_METADATA_BYTES
|
||||
: MAX_KNOWLEDGE_NODE_FRAGMENT_SOURCE_METADATA_BYTES),
|
||||
omitted: projection.omissions.map(({ field, reason }) => ({ field, reason })),
|
||||
projectedBytes: projection.projectedBytes,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function utf8ByteLength(value: string): number {
|
||||
let length = 0;
|
||||
for (const character of value) {
|
||||
const codePoint = character.codePointAt(0) as number;
|
||||
length += codePoint <= 0x7f ? 1 : codePoint <= 0x7ff ? 2 : codePoint <= 0xffff ? 3 : 4;
|
||||
}
|
||||
return length;
|
||||
}
|
||||
@ -318,7 +318,7 @@ export function createUnstructuredParserClient({
|
||||
requestGate.run(async () => {
|
||||
const deadline = createUnstructuredRequestDeadline(input.signal, requestTimeoutMs);
|
||||
try {
|
||||
const parserVersion = options.parserVersion ?? "unstructured@5";
|
||||
const parserVersion = options.parserVersion ?? "unstructured@6";
|
||||
const partitionStrategy = unstructuredPartitionStrategy(input);
|
||||
const providerImageBlockTypes = unstructuredProviderImageBlockTypes(input);
|
||||
assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes);
|
||||
@ -1683,9 +1683,13 @@ function unstructuredParseElementMetadata({
|
||||
}): Record<string, unknown> {
|
||||
// `image_base64` can be several megabytes. Move it into the short-lived assetRef URI consumed
|
||||
// by the multimodal extractor instead of retaining a second copy in ParseElement metadata.
|
||||
// `text_as_html` is normalized below. Keeping the provider spelling as well would retain the
|
||||
// same potentially multi-megabyte table HTML three times (`text_as_html`, `textAsHtml`, and
|
||||
// `table.html`) in every parse artifact.
|
||||
const {
|
||||
image_base64: _imageBase64,
|
||||
page_number: _pageNumber,
|
||||
text_as_html: _textAsHtml,
|
||||
...metadataWithoutInlineImage
|
||||
} = metadata;
|
||||
const parsed = cloneMetadata(metadataWithoutInlineImage);
|
||||
|
||||
@ -1049,7 +1049,7 @@ describe("parser adapters", () => {
|
||||
metadata: {
|
||||
filename: "report.pdf",
|
||||
mimeType: "application/pdf",
|
||||
parserVersion: "unstructured@5",
|
||||
parserVersion: "unstructured@6",
|
||||
},
|
||||
parser: "unstructured",
|
||||
version: 1,
|
||||
@ -1099,7 +1099,6 @@ describe("parser adapters", () => {
|
||||
metadata: {
|
||||
table: { html: "<table><tr><td>ARR</td></tr></table>" },
|
||||
textAsHtml: "<table><tr><td>ARR</td></tr></table>",
|
||||
text_as_html: "<table><tr><td>ARR</td></tr></table>",
|
||||
unstructuredType: "Table",
|
||||
},
|
||||
pageNumber: 3,
|
||||
@ -1139,7 +1138,7 @@ describe("parser adapters", () => {
|
||||
version: 1,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
metadata: { parserVersion: "unstructured@5" },
|
||||
metadata: { parserVersion: "unstructured@6" },
|
||||
parser: "unstructured",
|
||||
});
|
||||
},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user