diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 61de62202dc..3507535aaaa 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "fdd9cde96eb839ce06c8230c3e4a38e2e6d87369", + "subtreeTree": "a9810fc0e4c3dfd52c32d167d7015e82b58006fb", "openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796", "capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", diff --git a/knowledge-fs/.harness/changes/2026-08-26-structured-table-semantic-chunking.md b/knowledge-fs/.harness/changes/2026-08-26-structured-table-semantic-chunking.md new file mode 100644 index 00000000000..a6673a4c608 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-08-26-structured-table-semantic-chunking.md @@ -0,0 +1,78 @@ +# Structured table semantic chunking + +## What changed + +- Kept spreadsheet, CSV, JSONL, Markdown, HTML, and Unstructured table elements typed as + `table`, but added a versioned semantic projection containing bounded columns, header and source + row counts, record count, and one of `record-list`, `single-record`, `matrix`, or `unknown`. +- Canonicalized each logical table record as one field-labelled line. Embedded newlines stay inside + their source record, duplicate or blank headers receive stable names, multi-row HTML headers and + bounded row/column spans are flattened deterministically, and worksheet metadata remains attached + to its own table element. +- Advanced native Markdown/MDX, HTML, structured-data, and Unstructured parser versions so the new + projection cannot collide with an older cached artifact. +- Added `semantic-chunking-v5`. Reliable record-list and matrix rows become semantic atomic units; + the prompt receives a table schema once per window plus record indexes and source-row provenance. + A deterministic post-processor splits any model response that groups separate records, while a + single overlong record may still be hard-split to respect the existing 1,200-grapheme limit. +- Kept `semantic-chunking-v4` replay behavior unchanged. Legacy structured artifacts with the old + header-first text plus `rowCount` are recognized without requiring a reparse. + +## Why + +The reported workbook was parsed as one table element, but its flattened text had no stable record +boundaries for semantic chunking. The model could legally return the complete worksheet as one +chunk. Prompt wording alone could not make this reliable because parser structure had already been +discarded. + +The fix makes row/record boundaries trusted parser provenance. The LLM can still enrich sections, +summaries, entities, and relations, but it can no longer merge independent business records from a +record list or matrix. + +## Memory and token boundaries + +- Table schemas are serialized once per semantic window, not once per row, and are not copied onto + every output knowledge node. Field names remain in chunk text, so independent chunks retain their + retrieval meaning. +- Projection builds output lines in one pass instead of retaining a second normalized row matrix. +- Columns are capped at 64 names and 160 characters per name in the semantic planner. Existing + parser input, row, element, response, node, window, and heap admission limits remain in force. +- A low-heap regression preflights 2,000 structured records as 2,000 units and 65 bounded windows + under a 128 MiB V8 heap. The existing production-sized flattened-table regression still passes + under the same heap limit. + +## Measured sample result + +Read-only inspection of `dify使用问题反馈.xlsx` found one worksheet, 329,567 uploaded bytes, eight +populated rows (one header plus seven records), seven business columns, and a longest cell of 66 +characters. With the v5 projection, the seven records preflight as seven units in one semantic +window, so segmentation requires one model request and deterministically produces seven record +chunks even if the model groups them. Local preflight took 6.741 ms and reported 12,060,344 bytes of +heap after module initialization. + +Those figures describe local structural preflight only. They are not claims about end-to-end import +latency, provider latency, embedding time, or production throughput. + +## Compatibility and rollout + +- Newly parsed documents use the new parser and prompt versions automatically. +- Old structured CSV/JSONL artifacts with `columns` and `rowCount` gain record splitting during a + new semantic generation without reparsing. +- Older provider artifacts that contain only irreversibly flattened table text and no HTML or row + metadata remain `unknown`; they are not guessed into records and need reindex/reparse to gain the + richer projection. +- No database migration or public HTTP contract change is required. + +## Verification + +- Parser tests cover native CSV/JSONL, Markdown, HTML, representative Unstructured XLSX, multiple + worksheets, quoted CSV newlines, empty/headerless/key-value/matrix tables, duplicate headers, and + bounded HTML spans. +- Semantic tests cover model grouping correction, long-record hard splitting, legacy row recovery, + v4 replay compatibility, prompt-schema deduplication, offsets, and low-heap admission. +- `pnpm --dir knowledge-fs --filter @knowledge/parsers test:coverage` — 72 passed; 96.25% statement + and 90.04% branch coverage. +- `pnpm --dir knowledge-fs --filter @knowledge/api test` — 4,656 passed, 3 skipped. +- Parser and API typechecks passed. +- `pnpm --dir knowledge-fs lint:backend` passed across 1,081 files. +- The KnowledgeFS contract lock was regenerated and checked after the staged subtree review. diff --git a/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts b/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts index 5372da64526..0df00ee7b2c 100644 --- a/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts +++ b/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts @@ -298,7 +298,7 @@ describe("incremental reindexer semantic generations", () => { }); }); - it("replays legacy v3 nodes without a generation receipt under the current v4 runtime", async () => { + it("replays legacy v3 nodes without a generation receipt under the current v5 runtime", async () => { const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 }); const nodes = createInMemoryKnowledgeNodeRepository({ maxBatchSize: 4, @@ -334,7 +334,7 @@ describe("incremental reindexer semantic generations", () => { compute: computeRuntime(), maxNodes: 4, nodes: nodesWithoutHistoricalReceipt, - semanticChunker: echoSemanticChunker(() => currentCalls++, "semantic-chunking-v4"), + semanticChunker: echoSemanticChunker(() => currentCalls++, "semantic-chunking-v5"), }).reindex(input), ).resolves.toMatchObject({ nodesCreated: 1, status: "rebuilt" }); expect(legacyCalls).toBe(1); diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts index 9103b7c2652..e3f32161c69 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker-memory.test.ts @@ -46,4 +46,59 @@ describe("LLM semantic chunker memory admission", () => { expect(result).toMatchObject({ maximumWindowCount: 132, unitCount: 526 }); expect(result.heapUsed).toBeLessThan(128 * 1024 * 1024); }); + + it("preflights 2,000 structured records without copying the table schema per row", () => { + const coreUrl = new URL("../../core/src/index.ts", import.meta.url).href; + const chunkerUrl = new URL("./llm-semantic-chunker.ts", import.meta.url).href; + const script = ` + import { ParseArtifactSchema } from ${JSON.stringify(coreUrl)}; + import { preflightLlmSemanticWindows } from ${JSON.stringify(chunkerUrl)}; + const columns = ["time", "question", "detail", "severity", "resolved", "resolvedAt", "resolution"]; + const text = Array.from({ length: 2_000 }, (_, index) => + columns.map((column) => \`\${column}: value-\${index}\`).join(" | ") + ).join("\\n"); + const parseArtifact = ParseArtifactSchema.parse({ + artifactHash: "b".repeat(64), + contentType: "structured", + createdAt: "2026-08-26T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + elements: [{ + id: "sheet-records", + metadata: { + table: { + columns, + headerRowCount: 1, + mode: "record-list", + recordCount: 2_000, + semanticVersion: 1, + sourceRowCount: 2_001, + }, + }, + sectionPath: ["Issue Log"], + text, + type: "table", + }], + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c45", + metadata: {}, + parser: "unstructured", + version: 1, + }); + const result = preflightLlmSemanticWindows({ parseArtifact }); + console.log(JSON.stringify({ ...result, heapUsed: process.memoryUsage().heapUsed })); + `; + const completed = spawnSync( + process.execPath, + ["--max-old-space-size=128", "--import", "tsx", "--input-type=module", "-e", script], + { encoding: "utf8", maxBuffer: 1024 * 1024, timeout: 15_000 }, + ); + + expect(completed.status, completed.stderr).toBe(0); + const result = JSON.parse(completed.stdout.trim()) as { + heapUsed: number; + maximumWindowCount: number; + unitCount: number; + }; + expect(result).toMatchObject({ maximumWindowCount: 65, unitCount: 2_000 }); + expect(result.heapUsed).toBeLessThan(128 * 1024 * 1024); + }); }); diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts index 2ceaed62e95..d41aad73ff1 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts @@ -37,15 +37,28 @@ interface PromptUnit { readonly id: string; readonly sourceElementId?: string; readonly sourceSectionPath?: readonly string[]; + readonly tableMode?: "matrix" | "record-list" | "single-record" | "unknown"; + readonly tableRecordCount?: number; + readonly tableRecordIndex?: number; + readonly tableSourceRowEnd?: number; + readonly tableSourceRowStart?: number; readonly text: string; readonly type: string; } +interface PromptTableSchema { + readonly columns: readonly string[]; + readonly mode: "matrix" | "record-list" | "single-record" | "unknown"; + readonly recordCount: number; + readonly sourceElementId: string; +} + interface PromptPayload { readonly atomicDocument?: boolean; readonly fixedCoreBoundary?: boolean; readonly lookAheadUnits?: readonly PromptUnit[]; readonly sectionPath: readonly string[]; + readonly tableSchemas?: readonly PromptTableSchema[]; readonly units: readonly PromptUnit[]; readonly windowId: string; } @@ -290,7 +303,7 @@ describe("LLM semantic chunker", () => { completed: true, entityCount: 2, model: "reasoner-model", - promptVersion: "semantic-chunking-v4", + promptVersion: "semantic-chunking-v5", }, relationExtraction: { completed: true, relationCount: 1 }, semanticChunking: { @@ -405,7 +418,7 @@ describe("LLM semantic chunker", () => { expect(preflight.unitCount).toBeGreaterThan(20); expect(nodes).toHaveLength(preflight.unitCount); - expect(nodes.map((node) => node.text).join("")).toBe(text); + expect(nodes.map((node) => node.text).join("\n")).toBe(text); expect(nodes.at(-1)?.endOffset).toBe(new TextEncoder().encode(text).byteLength); for (const node of nodes) { expect(node.metadata).toMatchObject({ @@ -544,7 +557,7 @@ describe("LLM semantic chunker", () => { windowPlanning: { atomicDocument: false, sourceSectionPathCount: 2, - version: "v4", + version: "v5", }, }); }); @@ -620,7 +633,7 @@ describe("LLM semantic chunker", () => { parseArtifact.elements.map((element) => element.text?.trim() ?? "").join("\n"), ); expect(nodes[0]?.metadata.semanticChunking).toMatchObject({ - windowPlanning: { version: "v4" }, + windowPlanning: { version: "v5" }, }); expect(v2Nodes[0]?.metadata.semanticChunking).toMatchObject({ windowPlanning: { version: "v2" }, @@ -857,7 +870,7 @@ describe("LLM semantic chunker", () => { windowPlanning: { atomicDocument: true, sourceSectionPathCount: 1, - version: "v4", + version: "v5", }, }); expect(nodes[0]?.metadata.extractedEntities).toEqual([ @@ -865,6 +878,246 @@ describe("LLM semantic chunker", () => { ]); }); + it("materializes one table chunk per record even when the model groups the whole table", async () => { + const parseArtifact = artifact([ + { + id: "issue-records", + metadata: { + table: { + columns: ["时间", "问题描述", "是否解决"], + headerRowCount: 1, + mode: "record-list", + recordCount: 3, + semanticVersion: 1, + sourceRowCount: 4, + }, + }, + sectionPath: ["Issue Log"], + text: [ + "时间: 2026-07-01 | 问题描述: 复制按钮无法点击 | 是否解决: 是", + "时间: 2026-07-02 | 问题描述: 工作流页面被清空 | 是否解决: 否", + "时间: 2026-08-04 | 问题描述: 插件凭证被重置 | 是否解决: 否", + ].join("\n"), + type: "table", + }, + ]); + const provider = new ScriptedProvider([echoWholeWindow]); + + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + expect(preflightLlmSemanticWindows({ parseArtifact })).toEqual({ + maximumWindowCount: 1, + unitCount: 3, + }); + expect(provider.calls).toHaveLength(1); + expect(nodes).toHaveLength(3); + expect(nodes.map((node) => node.kind)).toEqual(["table", "table", "table"]); + expect(nodes.map((node) => node.text)).toEqual(parseArtifact.elements[0]?.text?.split("\n")); + expect( + nodes.map( + (node) => (node.metadata.semanticChunking as { tableRecord?: unknown }).tableRecord, + ), + ).toEqual([ + { + count: 3, + index: 0, + mode: "record-list", + sourceRowEnd: 2, + sourceRowStart: 2, + }, + { + count: 3, + index: 1, + mode: "record-list", + sourceRowEnd: 3, + sourceRowStart: 3, + }, + { + count: 3, + index: 2, + mode: "record-list", + sourceRowEnd: 4, + sourceRowStart: 4, + }, + ]); + + const prompt = JSON.parse( + provider.calls[0]?.messages.find((message) => message.role === "user")?.content ?? "{}", + ) as PromptPayload; + expect(prompt.atomicDocument).toBe(false); + expect(prompt.tableSchemas).toEqual([ + { + columns: ["时间", "问题描述", "是否解决"], + mode: "record-list", + recordCount: 3, + sourceElementId: "issue-records", + }, + ]); + expect(prompt.units.map((unit) => unit.tableRecordIndex)).toEqual([0, 1, 2]); + expect(prompt.units.map((unit) => unit.tableSourceRowStart)).toEqual([2, 3, 4]); + expect(prompt.units.every((unit) => unit.tableMode === "record-list")).toBe(true); + expect(prompt.units.every((unit) => !Object.hasOwn(unit, "tableColumns"))).toBe(true); + }); + + it("keeps a hard-split table record bounded without combining adjacent records", async () => { + const parseArtifact = artifact([ + { + id: "long-table-record", + metadata: { + table: { + columns: ["问题", "详情"], + headerRowCount: 1, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + sourceRowCount: 3, + }, + }, + sectionPath: ["Issue Log"], + text: [ + `问题: 超长记录 | 详情: ${"长文本".repeat(24)}`, + "问题: 普通记录 | 详情: 已解决", + ].join("\n"), + type: "table", + }, + ]); + const provider = new ScriptedProvider([echoWholeWindow]); + const nodes = await createLlmSemanticChunker({ + maxChunkChars: 24, + maxWindowChars: 256, + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + expect(nodes.length).toBeGreaterThan(2); + expect(nodes.every((node) => countGraphemes(node.text) <= 24)).toBe(true); + expect( + nodes + .map((node) => node.text) + .join("\n") + .replaceAll("\n\n", "\n"), + ).toContain("问题: 普通记录 | 详情: 已解决"); + const recordIndexes = nodes.map( + (node) => + (node.metadata.semanticChunking as { tableRecord?: { index?: number } }).tableRecord?.index, + ); + expect(recordIndexes.at(-1)).toBe(1); + expect(recordIndexes.slice(0, -1).every((index) => index === 0)).toBe(true); + }); + + it("splits legacy header-first table text without requiring a reparse", async () => { + const parseArtifact = artifact([ + { + id: "legacy-rows", + metadata: { + columns: ["name", "status"], + format: "csv", + rowCount: 3, + }, + sectionPath: ["Legacy export"], + text: "name | status\nAlpha | open\nBeta | closed\nGamma | open", + type: "table", + }, + ]); + const provider = new ScriptedProvider([echoWholeWindow]); + + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + expect(preflightLlmSemanticWindows({ parseArtifact })).toEqual({ + maximumWindowCount: 1, + unitCount: 3, + }); + expect(provider.calls).toHaveLength(1); + expect(nodes.map((node) => node.text)).toEqual([ + "name | status\nAlpha | open", + "Beta | closed", + "Gamma | open", + ]); + }); + + it("replays semantic-chunking-v4 with its original table unitization", () => { + const parseArtifact = artifact([ + { + id: "versioned-table", + metadata: { + table: { + columns: ["name", "status"], + headerRowCount: 1, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + sourceRowCount: 3, + }, + }, + sectionPath: ["Versioned"], + text: "name: Alpha | status: open\nname: Beta | status: closed", + type: "table", + }, + ]); + + expect(preflightLlmSemanticWindows({ parseArtifact })).toEqual({ + maximumWindowCount: 1, + unitCount: 2, + }); + expect( + preflightLlmSemanticWindows({ + parseArtifact, + promptVersion: "semantic-chunking-v4", + }), + ).toEqual({ maximumWindowCount: 1, unitCount: 1 }); + }); + + it("does not invent a business record for a schema-only table", async () => { + const parseArtifact = artifact([ + { + id: "empty-table", + metadata: { + table: { + columns: ["Name", "Status"], + headerRowCount: 1, + mode: "unknown", + recordCount: 0, + semanticVersion: 1, + sourceRowCount: 1, + }, + }, + sectionPath: ["Empty export"], + text: "Name | Status", + type: "table", + }, + ]); + const provider = new ScriptedProvider([echoWholeWindow]); + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + const prompt = JSON.parse( + provider.calls[0]?.messages.find((message) => message.role === "user")?.content ?? "{}", + ) as PromptPayload; + expect(prompt.tableSchemas).toEqual([]); + expect(nodes).toHaveLength(1); + expect(nodes[0]?.metadata.semanticChunking).not.toHaveProperty("tableRecord"); + }); + it("fails closed when model output violates atomic-record or special-element boundaries", async () => { const atomicArtifact = artifact([ { diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts index 83481c9b88d..f210ea05124 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts @@ -62,9 +62,10 @@ const DEFAULT_MAX_ENTITIES_PER_CHUNK = 100; const DEFAULT_MAX_RELATIONS_PER_CHUNK = 100; const DEFAULT_MAX_OUTPUT_TOKENS = 6_000; const DEFAULT_MAX_RESPONSE_CHARS = 1_000_000; -const DEFAULT_PROMPT_VERSION = "semantic-chunking-v4"; +const DEFAULT_PROMPT_VERSION = "semantic-chunking-v5"; const SEMANTIC_CHUNKING_V2_PROMPT_VERSION = "semantic-chunking-v2"; const SEMANTIC_CHUNKING_V3_PROMPT_VERSION = "semantic-chunking-v3"; +const SEMANTIC_CHUNKING_V4_PROMPT_VERSION = "semantic-chunking-v4"; const V3_MAX_CORE_UNITS_PER_WINDOW = 32; const V3_MAX_LOOK_AHEAD_UNITS_PER_WINDOW = 8; const DEFAULT_MAX_CONCURRENT_WINDOWS = 4; @@ -269,9 +270,28 @@ interface AtomicUnit { readonly sourceElement: MaterializedElement; readonly startCodeUnit: number; readonly startOffset: number; + readonly tableRecord?: TableRecordProvenance | undefined; readonly text: string; } +type TableSemanticMode = "matrix" | "record-list" | "single-record" | "unknown"; + +interface TableRecordProvenance { + readonly columns: readonly string[]; + readonly count: number; + readonly index: number; + readonly key: string; + readonly mode: TableSemanticMode; + readonly sourceRowEnd: number; + readonly sourceRowStart: number; +} + +interface SemanticRange { + readonly end: number; + readonly start: number; + readonly tableRecord?: TableRecordProvenance | undefined; +} + interface SemanticWindow { readonly atomicDocument: boolean; readonly id: string; @@ -283,7 +303,14 @@ interface SemanticWindow { readonly units: readonly AtomicUnit[]; } -type SemanticWindowPlanningVersion = "v1" | "v2" | "v3" | "v4"; +interface SemanticWindowTableSchema { + readonly columns: readonly string[]; + readonly mode: TableSemanticMode; + readonly recordCount: number; + readonly sourceElementId: string; +} + +type SemanticWindowPlanningVersion = "v1" | "v2" | "v3" | "v4" | "v5"; interface SemanticWindowPlanningPolicy { readonly atomicDocument: boolean; @@ -342,7 +369,7 @@ export function preflightLlmSemanticWindows({ requestedOverlapChars: 0, }); const { canonicalText, elements } = materializeElements(parseArtifact); - const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars, promptVersion); const planningPolicy = resolveSemanticWindowPlanningPolicy({ canonicalText, maxChunkChars: effectiveConfig.maxChunkChars, @@ -418,7 +445,7 @@ export function createLlmSemanticChunker({ return []; } - const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars, promptVersion); const planningPolicy = resolveSemanticWindowPlanningPolicy({ canonicalText, maxChunkChars: effectiveConfig.maxChunkChars, @@ -515,7 +542,11 @@ export function createLlmSemanticChunker({ : {}), ...(provider.kind ? { transportProvider: provider.kind } : {}), }); - const output = parseSemanticChunkingOutput(resolvedCompletion.text); + const output = normalizeTableRecordBoundaries({ + maxChunkChars: effectiveConfig.maxChunkChars, + output: parseSemanticChunkingOutput(resolvedCompletion.text), + window, + }); const windowChunks = validateAndMaterializeWindowOutput({ maxChunkChars: effectiveConfig.maxChunkChars, maxEntitiesPerChunk, @@ -611,7 +642,7 @@ export function createLlmSemanticChunker({ windowIndex += 1; }; - if (planningPolicy.version === "v4") { + if (usesFixedCoreBoundary(planningPolicy.version)) { const windows = materializeDeterministicSemanticWindows({ canonicalText, effectiveConfig, @@ -684,7 +715,7 @@ export function assertValidLlmSemanticGenerationReplay({ throw new Error("LLM semantic replay promptVersion is required"); } const { canonicalText, elements, layoutRecomposition } = materializeElements(parseArtifact); - const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars, promptVersion); const planningPolicy = resolveSemanticWindowPlanningPolicy({ canonicalText, maxChunkChars: effectiveConfig.maxChunkChars, @@ -808,7 +839,7 @@ export function assertValidLlmSemanticGenerationReplay({ commitStart === coreStart && commitEnd !== undefined && commitEnd >= coreEnd && - (planningPolicy.version !== "v4" || commitEnd === coreEnd), + (!usesFixedCoreBoundary(planningPolicy.version) || commitEnd === coreEnd), `chunk ${chunkIndex} has invalid core or committed window ranges`, ); const resolvedWindow = materializeSemanticWindow({ @@ -1104,7 +1135,7 @@ export function assertValidLlmSemanticWindowManifestReplay({ } const { canonicalText, elements } = materializeElements(parseArtifact); - const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars, promptVersion); const planningPolicy = resolveSemanticWindowPlanningPolicy({ canonicalText, maxChunkChars: effectiveConfig.maxChunkChars, @@ -1201,7 +1232,7 @@ export function assertValidLlmSemanticWindowManifestReplay({ commitEnd !== undefined && commitEnd >= coreEnd && commitEnd <= eligibleEnd && - (planningPolicy.version !== "v4" || commitEnd === coreEnd), + (!usesFixedCoreBoundary(planningPolicy.version) || commitEnd === coreEnd), `window ${windowOrdinal} has an invalid committed boundary`, ); @@ -1716,11 +1747,13 @@ function materializeElements(parseArtifact: ParseArtifact): { function materializeAtomicUnits( elements: readonly MaterializedElement[], maxChunkChars: number, + promptVersion: string, ): AtomicUnit[] { const units: AtomicUnit[] = []; + const planningVersion = semanticWindowPlanningVersion(promptVersion); for (const element of elements) { - const sentenceRanges = semanticRanges(element); + const sentenceRanges = semanticRanges(element, planningVersion); let atomicIndex = 0; let byteCursorCodeUnit = 0; let byteCursorOffset = element.startOffset; @@ -1757,6 +1790,7 @@ function materializeAtomicUnits( sourceElement: element, startCodeUnit: element.startCodeUnit + localStart, startOffset, + ...(range.tableRecord ? { tableRecord: range.tableRecord } : {}), text, }); atomicIndex += 1; @@ -1767,7 +1801,13 @@ function materializeAtomicUnits( return units; } -function semanticRanges(element: MaterializedElement): Array<{ end: number; start: number }> { +function semanticRanges( + element: MaterializedElement, + planningVersion: SemanticWindowPlanningVersion, +): SemanticRange[] { + if (planningVersion === "v5" && element.elementType === "table") { + return semanticTableRanges(element); + } if (element.elementType !== "paragraph" && element.elementType !== "list") { return [{ end: element.text.length, start: 0 }]; } @@ -1783,6 +1823,174 @@ function semanticRanges(element: MaterializedElement): Array<{ end: number; star return ranges; } +function semanticTableRanges(element: MaterializedElement): SemanticRange[] { + const lines = nonEmptyLineRanges(element.text); + if (lines.length === 0) return [{ end: element.text.length, start: 0 }]; + + const table = isPlainObject(element.elementMetadata.table) + ? element.elementMetadata.table + : undefined; + const columns = boundedTableColumns(table?.columns ?? element.elementMetadata.columns); + const declaredMode = tableSemanticMode(table?.mode); + const semanticVersion = table?.semanticVersion; + const nestedRecordCount = nonNegativeSafeInteger(table?.recordCount); + const legacyRecordCount = positiveSafeInteger(element.elementMetadata.rowCount); + const headerRowCount = nonNegativeSafeInteger(table?.headerRowCount) ?? 0; + const sourceRowCount = positiveSafeInteger(table?.sourceRowCount); + + if (semanticVersion === 1 && nestedRecordCount === 0) { + return [{ end: element.text.length, start: 0 }]; + } + + if ( + semanticVersion === 1 && + nestedRecordCount !== undefined && + nestedRecordCount === lines.length + ) { + const mode = declaredMode ?? inferTableSemanticMode({ columns, lines }); + return lines.map((line, index) => ({ + ...line, + tableRecord: tableRecordProvenance(element, columns, mode, index, lines.length, { + headerRowCount, + sourceRowCount: sourceRowCount ?? headerRowCount + lines.length, + }), + })); + } + + if ( + legacyRecordCount !== undefined && + legacyRecordCount > 0 && + lines.length === legacyRecordCount + 1 + ) { + const legacyHeaderRowCount = element.elementMetadata.format === "csv" ? 1 : 0; + const mode = declaredMode ?? inferTableSemanticMode({ columns, lines: lines.slice(1) }); + return lines.slice(1).map((line, index) => ({ + end: line.end, + start: index === 0 ? (lines[0] as SemanticRange).start : line.start, + tableRecord: tableRecordProvenance(element, columns, mode, index, legacyRecordCount, { + headerRowCount: legacyHeaderRowCount, + sourceRowCount: legacyRecordCount + legacyHeaderRowCount, + }), + })); + } + + if (lines.length > 1 && looksLikeDelimitedTableHeader(element.text.slice(0, lines[0]?.end))) { + const recordLines = lines.slice(1); + const inferredColumns = + columns.length > 0 + ? columns + : boundedTableColumns(element.text.slice(0, lines[0]?.end).split(" | ")); + const mode = + declaredMode ?? inferTableSemanticMode({ columns: inferredColumns, lines: recordLines }); + return recordLines.map((line, index) => ({ + end: line.end, + start: index === 0 ? (lines[0] as SemanticRange).start : line.start, + tableRecord: tableRecordProvenance( + element, + inferredColumns, + mode, + index, + recordLines.length, + { headerRowCount: 1, sourceRowCount: lines.length }, + ), + })); + } + + const mode = declaredMode ?? (lines.length === 1 ? "single-record" : "unknown"); + return lines.map((line, index) => ({ + ...line, + tableRecord: tableRecordProvenance(element, columns, mode, index, lines.length, { + headerRowCount: 0, + sourceRowCount: lines.length, + }), + })); +} + +function nonEmptyLineRanges(text: string): SemanticRange[] { + const ranges: SemanticRange[] = []; + let start = 0; + while (start < text.length) { + const newline = text.indexOf("\n", start); + const end = newline === -1 ? text.length : newline; + if (text.slice(start, end).trim()) ranges.push({ end, start }); + if (newline === -1) break; + start = newline + 1; + } + return ranges; +} + +function boundedTableColumns(value: unknown): string[] { + if (!Array.isArray(value)) return []; + const columns: string[] = []; + for (const entry of value.slice(0, 64)) { + if (typeof entry !== "string") return []; + const column = entry.trim().slice(0, 160); + if (!column) return []; + columns.push(column); + } + return columns; +} + +function positiveSafeInteger(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined; +} + +function nonNegativeSafeInteger(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +} + +function tableSemanticMode(value: unknown): TableSemanticMode | undefined { + return value === "matrix" || + value === "record-list" || + value === "single-record" || + value === "unknown" + ? value + : undefined; +} + +function inferTableSemanticMode({ + columns, + lines, +}: { + readonly columns: readonly string[]; + readonly lines: readonly SemanticRange[]; +}): TableSemanticMode { + if (lines.length <= 1) return "single-record"; + if (columns.length === 0) return "unknown"; + return "record-list"; +} + +function looksLikeDelimitedTableHeader(value: string): boolean { + return value.split(" | ").filter((cell) => cell.trim()).length >= 2; +} + +function tableRecordProvenance( + element: MaterializedElement, + columns: readonly string[], + mode: TableSemanticMode, + index: number, + count: number, + { + headerRowCount, + sourceRowCount, + }: { readonly headerRowCount: number; readonly sourceRowCount: number }, +): TableRecordProvenance { + const sourceRowStart = headerRowCount + index + 1; + const sourceRowEnd = + mode === "single-record" && count === 1 + ? Math.max(sourceRowStart, sourceRowCount) + : sourceRowStart; + return { + columns, + count, + index, + key: `${element.elementId}:record:${index}`, + mode, + sourceRowEnd, + sourceRowStart, + }; +} + function graphemeRanges( text: string, maxChunkChars: number, @@ -1872,14 +2080,7 @@ function resolveSemanticWindowPlanningPolicy({ readonly promptVersion: string; readonly units: readonly AtomicUnit[]; }): SemanticWindowPlanningPolicy { - const version: SemanticWindowPlanningVersion = - promptVersion === DEFAULT_PROMPT_VERSION - ? "v4" - : promptVersion === SEMANTIC_CHUNKING_V3_PROMPT_VERSION - ? "v3" - : promptVersion === SEMANTIC_CHUNKING_V2_PROMPT_VERSION - ? "v2" - : "v1"; + const version = semanticWindowPlanningVersion(promptVersion); if (version === "v1") { return { atomicDocument: false, version }; } @@ -1894,10 +2095,15 @@ function resolveSemanticWindowPlanningPolicy({ const hasNarrativeText = units.some( (unit) => unit.elementType !== "image" && unit.elementType !== "table", ); + const tableRecordKeys = new Set( + units.flatMap((unit) => (unit.tableRecord ? [unit.tableRecord.key] : [])), + ); + const hasMultiRecordTable = tableRecordKeys.size > 1; const atomicDocument = units.length > 0 && - ((version !== "v3" && version !== "v4") || units.length <= V3_MAX_CORE_UNITS_PER_WINDOW) && + (!usesBoundedCoreUnits(version) || units.length <= V3_MAX_CORE_UNITS_PER_WINDOW) && tableElementIds.size === 1 && + !hasMultiRecordTable && hasNarrativeText && hasCompletePageProvenance && pageNumbers.size === 1 && @@ -1905,6 +2111,22 @@ function resolveSemanticWindowPlanningPolicy({ return { atomicDocument, version }; } +function semanticWindowPlanningVersion(promptVersion: string): SemanticWindowPlanningVersion { + if (promptVersion === DEFAULT_PROMPT_VERSION) return "v5"; + if (promptVersion === SEMANTIC_CHUNKING_V4_PROMPT_VERSION) return "v4"; + if (promptVersion === SEMANTIC_CHUNKING_V3_PROMPT_VERSION) return "v3"; + if (promptVersion === SEMANTIC_CHUNKING_V2_PROMPT_VERSION) return "v2"; + return "v1"; +} + +function usesBoundedCoreUnits(version: SemanticWindowPlanningVersion): boolean { + return version === "v3" || version === "v4" || version === "v5"; +} + +function usesFixedCoreBoundary(version: SemanticWindowPlanningVersion): boolean { + return version === "v4" || version === "v5"; +} + function materializeSemanticWindow({ canonicalText, maxChunkChars, @@ -1931,7 +2153,7 @@ function materializeSemanticWindow({ let cursor = startUnitIndex; while (cursor < units.length) { if ( - (planningPolicy.version === "v3" || planningPolicy.version === "v4") && + usesBoundedCoreUnits(planningPolicy.version) && coreUnits.length >= V3_MAX_CORE_UNITS_PER_WINDOW ) { break; @@ -1957,7 +2179,7 @@ function materializeSemanticWindow({ const firstLookAhead = units[cursor]; while (firstLookAhead && cursor < units.length) { if ( - (planningPolicy.version === "v3" || planningPolicy.version === "v4") && + usesBoundedCoreUnits(planningPolicy.version) && lookAheadUnits.length >= V3_MAX_LOOK_AHEAD_UNITS_PER_WINDOW ) { break; @@ -1992,6 +2214,7 @@ function materializeSemanticWindow({ ), planningVersion: planningPolicy.version, sectionPath, + tableSchemas: semanticWindowTableSchemas([...coreUnits, ...lookAheadUnits]), units: coreUnits.map((unit) => semanticPromptUnit(unit, planningPolicy.version)), windowId: id, }; @@ -2017,6 +2240,11 @@ function semanticPromptUnit( readonly id: string; readonly sourceElementId?: string | undefined; readonly sourceSectionPath?: readonly string[] | undefined; + readonly tableMode?: TableSemanticMode | undefined; + readonly tableRecordCount?: number | undefined; + readonly tableRecordIndex?: number | undefined; + readonly tableSourceRowEnd?: number | undefined; + readonly tableSourceRowStart?: number | undefined; readonly text: string; readonly type: string; } { @@ -2033,11 +2261,35 @@ function semanticPromptUnit( sourceSectionPath: [...unit.sectionPath], } : {}), + ...(planningVersion === "v5" && unit.tableRecord + ? { + tableMode: unit.tableRecord.mode, + tableRecordCount: unit.tableRecord.count, + tableRecordIndex: unit.tableRecord.index, + tableSourceRowEnd: unit.tableRecord.sourceRowEnd, + tableSourceRowStart: unit.tableRecord.sourceRowStart, + } + : {}), text: unit.text, type: unit.elementType, }; } +function semanticWindowTableSchemas(units: readonly AtomicUnit[]): SemanticWindowTableSchema[] { + const schemas = new Map(); + for (const unit of units) { + const record = unit.tableRecord; + if (!record || schemas.has(unit.elementId)) continue; + schemas.set(unit.elementId, { + columns: [...record.columns], + mode: record.mode, + recordCount: record.count, + sourceElementId: unit.elementId, + }); + } + return [...schemas.values()]; +} + function isLegacyWindowCompatible(first: AtomicUnit, candidate: AtomicUnit): boolean { return ( sameStrings(first.sectionPath, candidate.sectionPath) && @@ -2088,7 +2340,7 @@ function semanticChunkingMessages({ : "PageIndex is disabled: preserve only the supplied sectionPath, omit sectionSummary, and do not invent child section levels.", "Return strict JSON only. Never return, rewrite, summarize, correct, or duplicate source text.", "The units field is the core: cover every core unit exactly once, in order, by contiguous inclusive ranges.", - ...(window.planningVersion === "v4" + ...(usesFixedCoreBoundary(window.planningVersion) ? [ "lookAheadUnits is read-only context. Never include a look-ahead unit in any returned range; another request owns it.", "The final range must end at the final unit in units so independent windows can be processed safely.", @@ -2113,6 +2365,13 @@ function semanticChunkingMessages({ : [ "A unit marked boundaryPolicy=isolated must occupy a chunk containing only units from that same table or image element.", ]), + ...(window.planningVersion === "v5" + ? [ + "Table records carry tableMode/tableRecordIndex/tableRecordCount; resolve their bounded columns from tableSchemas by sourceElementId.", + "For tableMode=record-list or matrix, return exactly one chunk per table record; hard-split units with the same tableRecordIndex may stay together, but never combine different records.", + "For tableMode=single-record, keep all units from that record together when the maxChunkChars limit permits.", + ] + : []), ...(window.atomicDocument ? [ "atomicDocument=true: the complete input is one short structured record. Return exactly one chunk covering every core unit; table boundaries are metadata, not retrieval boundaries.", @@ -2140,11 +2399,16 @@ function semanticChunkingMessages({ content: JSON.stringify({ ...(carriesParserProvenance ? { atomicDocument: window.atomicDocument } : {}), features: { enableGraph, enablePageIndex }, - ...(window.planningVersion === "v4" ? { fixedCoreBoundary: true } : {}), + ...(usesFixedCoreBoundary(window.planningVersion) ? { fixedCoreBoundary: true } : {}), lookAheadUnits: window.lookAheadUnits.map((unit) => semanticPromptUnit(unit, window.planningVersion), ), sectionPath: window.sectionPath, + ...(window.planningVersion === "v5" + ? { + tableSchemas: semanticWindowTableSchemas([...window.units, ...window.lookAheadUnits]), + } + : {}), units: window.units.map((unit) => semanticPromptUnit(unit, window.planningVersion)), windowId: window.id, }), @@ -2356,6 +2620,98 @@ function parseSemanticChunkingOutput(text: string): LlmSemanticChunkingOutput { } } +function normalizeTableRecordBoundaries({ + maxChunkChars, + output, + window, +}: { + readonly maxChunkChars: number; + readonly output: LlmSemanticChunkingOutput; + readonly window: SemanticWindow; +}): LlmSemanticChunkingOutput { + if (window.planningVersion !== "v5" || window.atomicDocument) return output; + + const eligibleUnits = [...window.units, ...window.lookAheadUnits]; + const unitIndex = new Map(eligibleUnits.map((unit, index) => [unit.id, index])); + const normalized: LlmSemanticChunkingOutput["chunks"] = []; + + for (const candidate of output.chunks) { + const start = unitIndex.get(candidate.startUnitId); + const end = unitIndex.get(candidate.endUnitId); + if (start === undefined || end === undefined || end < start) { + normalized.push(candidate); + continue; + } + const candidateUnits = eligibleUnits.slice(start, end + 1); + if ( + candidateUnits.length === 0 || + !candidateUnits.every( + (unit) => unit.tableRecord?.mode === "record-list" || unit.tableRecord?.mode === "matrix", + ) + ) { + normalized.push(candidate); + continue; + } + + const recordGroups: AtomicUnit[][] = []; + for (const unit of candidateUnits) { + const current = recordGroups.at(-1); + if (current && current[0]?.tableRecord?.key === unit.tableRecord?.key) { + current.push(unit); + } else { + recordGroups.push([unit]); + } + } + const boundedGroups = recordGroups.flatMap((group) => + splitTableRecordGroup(group, maxChunkChars), + ); + if (boundedGroups.length <= 1) { + normalized.push(candidate); + continue; + } + + for (const group of boundedGroups) { + const groupText = semanticUnitsText(group); + const entities = candidate.entities.filter((entity) => + groupText.includes(entity.text.trim()), + ); + const entityIds = new Set(entities.map((entity) => entity.id)); + normalized.push({ + ...candidate, + endUnitId: (group.at(-1) as AtomicUnit).id, + entities, + relations: candidate.relations.filter( + (relation) => + entityIds.has(relation.subjectEntityId) && entityIds.has(relation.objectEntityId), + ), + startUnitId: (group[0] as AtomicUnit).id, + }); + } + } + + return { chunks: normalized }; +} + +function splitTableRecordGroup( + units: readonly AtomicUnit[], + maxChunkChars: number, +): AtomicUnit[][] { + const groups: AtomicUnit[][] = []; + let current: AtomicUnit[] = []; + for (const unit of units) { + if ( + current.length > 0 && + countUnicodeGraphemes(semanticUnitsText([...current, unit])) > maxChunkChars + ) { + groups.push(current); + current = []; + } + current.push(unit); + } + if (current.length > 0) groups.push(current); + return groups; +} + function validateAndMaterializeWindowOutput({ maxChunkChars, maxEntitiesPerChunk, @@ -2392,25 +2748,15 @@ function validateAndMaterializeWindowOutput({ "LLM semantic chunking response must cover units contiguously without gaps or overlap", ); } - if (window.planningVersion === "v4" && end > coreEnd) { + if (usesFixedCoreBoundary(window.planningVersion) && end > coreEnd) { throw new Error( - "LLM semantic chunking v4 response must not commit read-only look-ahead units", + "LLM semantic chunking fixed-core response must not commit read-only look-ahead units", ); } const chunkUnits = eligibleUnits.slice(start, end + 1); const first = chunkUnits[0] as AtomicUnit; const last = chunkUnits.at(-1) as AtomicUnit; - const chunkText = chunkUnits - .map((unit) => unit.text) - .reduce((combined, text, index) => { - if (index === 0) return text; - const previous = chunkUnits[index - 1] as AtomicUnit; - const separator = - previous.endCodeUnit === (chunkUnits[index] as AtomicUnit).startCodeUnit - ? "" - : DOCUMENT_ELEMENT_SEPARATOR; - return `${combined}${separator}${text}`; - }, ""); + const chunkText = semanticUnitsText(chunkUnits); if (countUnicodeGraphemes(chunkText) > maxChunkChars) { throw new Error(`LLM semantic chunking response exceeded maxChunkChars=${maxChunkChars}`); } @@ -2497,8 +2843,8 @@ function validateAndMaterializeWindowOutput({ if (finalStart > coreEnd) { throw new Error("LLM semantic chunking final chunk must start in the core window"); } - if (window.planningVersion === "v4" && finalEnd !== coreEnd) { - throw new Error("LLM semantic chunking v4 response must end at the fixed core boundary"); + if (usesFixedCoreBoundary(window.planningVersion) && finalEnd !== coreEnd) { + throw new Error("LLM semantic chunking fixed-core response must end at the core boundary"); } const commitEndUnitId = eligibleUnits[finalEnd]?.id; if (!commitEndUnitId) { @@ -2507,11 +2853,30 @@ function validateAndMaterializeWindowOutput({ return chunks.map((chunk) => ({ ...chunk, windowCommitEndUnitId: commitEndUnitId })); } +function semanticUnitsText(units: readonly AtomicUnit[]): string { + return units.reduce((combined, unit, index) => { + if (index === 0) return unit.text; + const previous = units[index - 1] as AtomicUnit; + const separator = previous.endCodeUnit === unit.startCodeUnit ? "" : DOCUMENT_ELEMENT_SEPARATOR; + return `${combined}${separator}${unit.text}`; + }, ""); +} + function respectsIsolatedElementBoundaries(units: readonly AtomicUnit[]): boolean { const isolated = units.filter((unit) => unit.isolationKey !== undefined); if (isolated.length === 0) return true; const isolationKey = isolated[0]?.isolationKey; - return isolationKey !== undefined && units.every((unit) => unit.isolationKey === isolationKey); + if (isolationKey === undefined || !units.every((unit) => unit.isolationKey === isolationKey)) { + return false; + } + const recordKeys = new Set( + units.flatMap((unit) => + unit.tableRecord?.mode === "record-list" || unit.tableRecord?.mode === "matrix" + ? [unit.tableRecord.key] + : [], + ), + ); + return recordKeys.size <= 1; } function groundEntity(entity: LlmSemanticEntity, chunkText: string): LlmSemanticEntity | undefined { @@ -2624,6 +2989,7 @@ function materializeKnowledgeNode({ subject: relation.subject, type: relation.type, })); + const tableRecord = semanticTableRecordMetadata(chunk.units); const metadata: Record = { chunkIndex, elementIds: uniqueStrings(chunk.units.map((unit) => unit.elementId)), @@ -2681,6 +3047,7 @@ function materializeKnowledgeNode({ }, sourceSpans: semanticSourceSpans(chunk.units), strategy: SEMANTIC_CHUNKING_STRATEGY, + ...(tableRecord ? { tableRecord } : {}), unitRange: { endUnitId: chunk.endUnitId, startUnitId: chunk.startUnitId, @@ -2753,6 +3120,26 @@ function materializeKnowledgeNode({ }); } +function semanticTableRecordMetadata(units: readonly AtomicUnit[]): + | { + readonly count: number; + readonly index: number; + readonly mode: TableSemanticMode; + readonly sourceRowEnd: number; + readonly sourceRowStart: number; + } + | undefined { + const record = units[0]?.tableRecord; + if (!record || !units.every((unit) => unit.tableRecord?.key === record.key)) return undefined; + return { + count: record.count, + index: record.index, + mode: record.mode, + sourceRowEnd: record.sourceRowEnd, + sourceRowStart: record.sourceRowStart, + }; +} + function semanticSourceSpans(units: readonly AtomicUnit[]): Array<{ readonly elementId: string; readonly elementType: string; diff --git a/knowledge-fs/packages/parsers/src/index.ts b/knowledge-fs/packages/parsers/src/index.ts index 7dfaf17f7d5..33b1a650291 100644 --- a/knowledge-fs/packages/parsers/src/index.ts +++ b/knowledge-fs/packages/parsers/src/index.ts @@ -224,7 +224,7 @@ export function createNativeMarkdownParser(options: NativeParserOptions = {}): P kind: "native-markdown", parse: async (input) => { const isMdx = isMdxInput(input); - const parserVersion = options.parserVersion ?? (isMdx ? "native-mdx@1" : "native-markdown@1"); + const parserVersion = options.parserVersion ?? (isMdx ? "native-mdx@2" : "native-markdown@2"); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); const text = decodeUtf8(input.body); const tokens = marked.lexer(text, { gfm: true }); @@ -245,7 +245,7 @@ export function createNativeHtmlParser(options: NativeParserOptions = {}): Parse return { kind: "native-html", parse: async (input) => { - const parserVersion = options.parserVersion ?? "native-html@2"; + const parserVersion = options.parserVersion ?? "native-html@3"; assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); const text = decodeUtf8(input.body); const document = parseDocument(text, { @@ -274,7 +274,7 @@ export function createNativeStructuredDataParser( return { kind: "native-structured", parse: async (input) => { - const parserVersion = options.parserVersion ?? "native-structured@1"; + const parserVersion = options.parserVersion ?? "native-structured@2"; assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); const text = decodeUtf8(input.body); const format = structuredDataFormat(input); @@ -318,7 +318,7 @@ export function createUnstructuredParserClient({ requestGate.run(async () => { const deadline = createUnstructuredRequestDeadline(input.signal, requestTimeoutMs); try { - const parserVersion = options.parserVersion ?? "unstructured@6"; + const parserVersion = options.parserVersion ?? "unstructured@7"; const partitionStrategy = unstructuredPartitionStrategy(input); const providerImageBlockTypes = unstructuredProviderImageBlockTypes(input); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); @@ -1012,10 +1012,12 @@ function rowsToTableElements( } const columns = uniqueStrings(rows.flatMap((row) => Object.keys(row))); - const tableRows = [ - columns.join(" | "), - ...rows.map((row) => columns.map((column) => structuredCell(row[column])).join(" | ")), - ]; + const headerRowCount = format === "csv" ? 1 : 0; + const projection = projectTableRecords({ + columns, + headerRowCount, + rows: rows.map((row) => columns.map((column) => structuredCell(row[column]))), + }); return [ { @@ -1023,14 +1025,108 @@ function rowsToTableElements( columns, format, rowCount: rows.length, + table: projection.metadata, }, sectionPath: [], - text: tableRows.join("\n"), + text: projection.text, type: "table", }, ]; } +type TableSemanticMode = "matrix" | "record-list" | "single-record" | "unknown"; + +interface TableProjection { + readonly metadata: { + readonly columns: readonly string[]; + readonly headerRowCount: number; + readonly mode: TableSemanticMode; + readonly recordCount: number; + readonly semanticVersion: 1; + readonly sourceRowCount: number; + }; + readonly text: string; +} + +function projectTableRecords({ + columns: rawColumns, + headerRowCount, + mode, + rows, + sourceRowCount, +}: { + readonly columns: readonly string[]; + readonly headerRowCount: number; + readonly mode?: TableSemanticMode | undefined; + readonly rows: readonly (readonly string[])[]; + readonly sourceRowCount?: number | undefined; +}): TableProjection { + let width = Math.max(rawColumns.length, 1); + for (const row of rows) width = Math.max(width, row.length); + const columnCounts = new Map(); + const columns = Array.from({ length: width }, (_, index) => { + const value = normalizeTableCell(rawColumns[index] ?? ""); + const base = value || `column_${index + 1}`; + const count = (columnCounts.get(base) ?? 0) + 1; + columnCounts.set(base, count); + return count === 1 ? base : `${base}_${count}`; + }); + const lines: string[] = []; + let matrixCellCount = 0; + let numericCellCount = 0; + for (const row of rows) { + const cells: string[] = []; + for (let index = 0; index < columns.length; index += 1) { + const value = normalizeTableCell(row[index] ?? ""); + cells.push(value); + if (index === 0 || !value) continue; + matrixCellCount += 1; + if (tableCellValueKind(value) === "number") numericCellCount += 1; + } + lines.push(columns.map((column, index) => `${column}: ${cells[index]}`).join(" | ")); + } + const resolvedMode = + mode ?? + classifyTableSemanticMode({ + columnCount: columns.length, + matrixCellCount, + numericCellCount, + rowCount: rows.length, + }); + const text = lines.join("\n"); + + return { + metadata: { + columns, + headerRowCount, + mode: resolvedMode, + recordCount: rows.length, + semanticVersion: 1, + sourceRowCount: sourceRowCount ?? headerRowCount + rows.length, + }, + text: text || columns.join(" | "), + }; +} + +function classifyTableSemanticMode({ + columnCount, + matrixCellCount, + numericCellCount, + rowCount, +}: { + readonly columnCount: number; + readonly matrixCellCount: number; + readonly numericCellCount: number; + readonly rowCount: number; +}): TableSemanticMode { + if (rowCount === 0) return "unknown"; + if (rowCount === 1) return "single-record"; + if (columnCount >= 3 && matrixCellCount > 0 && numericCellCount / matrixCellCount >= 0.7) { + return "matrix"; + } + return "record-list"; +} + function structuredCell(value: unknown): string { if (value === null || value === undefined) { return ""; @@ -1131,7 +1227,10 @@ function markdownTokensToElements( if (token.type === "table") { const table = token as Tokens.Table; - pushTextElement(elements, "table", markdownTableText(table), sectionPath); + const projection = markdownTableProjection(table); + pushTextElement(elements, "table", projection.text, sectionPath, { + table: projection.metadata, + }); } } @@ -1244,7 +1343,10 @@ function visitHtmlNode(node: HtmlNode, elements: ParseElementInput[], sectionPat } if (name === "table") { - pushTextElement(elements, "table", htmlTableText(node), sectionPath); + const projection = htmlTableProjection(node); + pushTextElement(elements, "table", projection.text, sectionPath, { + table: projection.metadata, + }); return; } @@ -1279,8 +1381,10 @@ function unstructuredElementsToElements( const headingPathsByElementId = new Map(); for (const sourceElement of sourceElements) { - const text = normalizeText(sourceElement.text ?? ""); const type = unstructuredType(sourceElement.type); + const tableProjection = + type === "table" ? unstructuredTableProjection(sourceElement.metadata) : undefined; + const text = tableProjection?.text ?? normalizeText(sourceElement.text ?? ""); if (!text && !hasUnstructuredVisualMetadata(sourceElement.metadata, type)) { continue; @@ -1307,6 +1411,7 @@ function unstructuredElementsToElements( elements.push({ metadata: unstructuredParseElementMetadata({ metadata: sourceElement.metadata, + tableProjection, text, type, unstructuredType: sourceElement.type, @@ -1672,11 +1777,13 @@ function hasUnstructuredVisualMetadata( function unstructuredParseElementMetadata({ metadata, + tableProjection, text, type, unstructuredType, }: { readonly metadata: Readonly>; + readonly tableProjection?: TableProjection | undefined; readonly text: string; readonly type: ParseElement["type"]; readonly unstructuredType: string | undefined; @@ -1704,7 +1811,14 @@ function unstructuredParseElementMetadata({ ...(caption ? { caption } : {}), ...(type === "image" && text ? { ocrText: text } : {}), ...(textAsHtml ? { textAsHtml } : {}), - ...(type === "table" && textAsHtml ? { table: { html: textAsHtml } } : {}), + ...(type === "table" && (textAsHtml || tableProjection) + ? { + table: { + ...(tableProjection?.metadata ?? {}), + ...(textAsHtml ? { html: textAsHtml } : {}), + }, + } + : {}), ...(title ? { title } : {}), }; @@ -1717,6 +1831,21 @@ function unstructuredParseElementMetadata({ }; } +function unstructuredTableProjection( + metadata: Readonly>, +): TableProjection | undefined { + const textAsHtml = metadataString(metadata, "text_as_html"); + if (!textAsHtml) return undefined; + const document = parseDocument(textAsHtml, { + lowerCaseAttributeNames: true, + lowerCaseTags: true, + }); + const table = (document.children as HtmlNode[]).flatMap((node) => + node.name?.toLowerCase() === "table" ? [node] : findHtmlElements(node, "table"), + )[0]; + return table ? htmlTableProjection(table) : undefined; +} + function unstructuredAssetRef( metadata: Readonly>, ): Record | undefined { @@ -1885,11 +2014,12 @@ function compactSectionPath(sectionPath: readonly (string | undefined)[]): strin return sectionPath.filter((segment): segment is string => typeof segment === "string"); } -function markdownTableText(table: Tokens.Table): string { - const header = table.header.map((cell) => normalizeText(cell.text)).join(" | "); - const rows = table.rows.map((row) => row.map((cell) => normalizeText(cell.text)).join(" | ")); - - return [header, ...rows].filter(Boolean).join("\n"); +function markdownTableProjection(table: Tokens.Table): TableProjection { + return projectTableRecords({ + columns: table.header.map((cell) => normalizeText(cell.text)), + headerRowCount: 1, + rows: table.rows.map((row) => row.map((cell) => normalizeText(cell.text))), + }); } function htmlListText(node: HtmlNode): string { @@ -1900,17 +2030,198 @@ function htmlListText(node: HtmlNode): string { .join("\n"); } -function htmlTableText(node: HtmlNode): string { - const rows = findHtmlElements(node, "tr") - .map((row) => - (row.children ?? []) - .filter((cell) => ["td", "th"].includes(cell.name?.toLowerCase() ?? "")) - .map((cell) => normalizeText(htmlText(cell))) - .join(" | "), - ) - .filter(Boolean); +function htmlTableProjection(node: HtmlNode): TableProjection { + const rows = htmlTableRows(node); + if (rows.length === 0) { + return { + metadata: { + columns: [], + headerRowCount: 0, + mode: "unknown", + recordCount: 0, + semanticVersion: 1, + sourceRowCount: 0, + }, + text: "", + }; + } + const firstRow = rows[0] as { + readonly cells: readonly string[]; + readonly hasHeaderCell: boolean; + readonly inHeaderGroup: boolean; + }; + if (firstRow.hasHeaderCell || firstRow.inHeaderGroup) { + const headerRowCount = rows.findIndex((row) => !row.hasHeaderCell && !row.inHeaderGroup); + const resolvedHeaderRowCount = headerRowCount === -1 ? rows.length : headerRowCount; + return projectTableRecords({ + columns: flattenHtmlTableHeaders(rows.slice(0, resolvedHeaderRowCount)), + headerRowCount: resolvedHeaderRowCount, + rows: rows.slice(resolvedHeaderRowCount).map((row) => row.cells), + sourceRowCount: rows.length, + }); + } + return projectHeaderlessTableRows(rows.map((row) => row.cells)); +} - return rows.join("\n"); +function htmlTableRows(node: HtmlNode): Array<{ + readonly cells: readonly string[]; + readonly hasHeaderCell: boolean; + readonly inHeaderGroup: boolean; +}> { + const headerRows = new Set( + findHtmlElements(node, "thead").flatMap((header) => findHtmlElements(header, "tr")), + ); + let activeRowspans = new Map(); + return findHtmlElements(node, "tr") + .map((row) => { + const sourceCells = (row.children ?? []).filter((cell) => + ["td", "th"].includes(cell.name?.toLowerCase() ?? ""), + ); + const cells: string[] = []; + const nextRowspans = new Map< + number, + { readonly remaining: number; readonly value: string } + >(); + let column = 0; + const consumeRowspan = () => { + const carried = activeRowspans.get(column); + if (!carried) return false; + cells[column] = carried.value; + if (carried.remaining > 1) { + nextRowspans.set(column, { remaining: carried.remaining - 1, value: carried.value }); + } + activeRowspans.delete(column); + column += 1; + return true; + }; + for (const cell of sourceCells) { + while (consumeRowspan()) { + // A rowspan reserves this column before the next source cell. + } + const value = normalizeText(htmlText(cell)); + const columnSpan = htmlTableCellSpan(cell, "colspan"); + const rowSpan = htmlTableCellSpan(cell, "rowspan"); + for (let offset = 0; offset < columnSpan; offset += 1) { + while (consumeRowspan()) { + // A colspan only occupies columns not already reserved by a rowspan. + } + cells[column] = value; + if (rowSpan > 1) { + nextRowspans.set(column, { remaining: rowSpan - 1, value }); + } + column += 1; + } + } + while (activeRowspans.size > 0) { + if (!consumeRowspan()) column += 1; + } + activeRowspans = nextRowspans; + return { + cells, + hasHeaderCell: sourceCells.some((cell) => cell.name?.toLowerCase() === "th"), + inHeaderGroup: headerRows.has(row), + }; + }) + .filter((row) => row.cells.some(Boolean)); +} + +function flattenHtmlTableHeaders(rows: readonly { readonly cells: readonly string[] }[]): string[] { + const width = Math.max(...rows.map((row) => row.cells.length), 1); + return Array.from({ length: width }, (_, column) => { + const labels: string[] = []; + for (const row of rows) { + const label = row.cells[column]?.trim(); + if (label && labels.at(-1) !== label) labels.push(label); + } + return labels.join(" / "); + }); +} + +function htmlTableCellSpan(cell: HtmlNode, attribute: "colspan" | "rowspan"): number { + const parsed = Number.parseInt(cell.attribs?.[attribute] ?? "1", 10); + return Number.isSafeInteger(parsed) && parsed >= 1 && parsed <= 256 ? parsed : 1; +} + +function projectHeaderlessTableRows(rows: readonly (readonly string[])[]): TableProjection { + if (rows.length === 1) { + return projectTableRecords({ columns: [], headerRowCount: 0, rows }); + } + const firstRow = rows[0] ?? []; + if (looksLikeTableHeader(firstRow, rows.slice(1))) { + return projectTableRecords({ + columns: firstRow, + headerRowCount: 1, + rows: rows.slice(1), + }); + } + if (looksLikeKeyValueTable(rows)) { + return projectTableRecords({ + columns: rows.map((row) => row[0] ?? ""), + headerRowCount: 0, + mode: "single-record", + rows: [rows.map((row) => row[1] ?? "")], + sourceRowCount: rows.length, + }); + } + return projectTableRecords({ + columns: Array.from( + { length: Math.max(...rows.map((row) => row.length), 1) }, + (_, index) => `column_${index + 1}`, + ), + headerRowCount: 0, + mode: "record-list", + rows, + }); +} + +const TABLE_HEADER_LABEL_PATTERN = + /(?:^|[_\s-])(id|key|name|title|date|time|status|type|category|description|detail|score|count|amount|price|value|result)(?:$|[_\s-])|(?:编号|号码|代码|名称|姓名|标题|日期|时间|状态|类型|类别|问题|描述|详情|等级|是否|结果|分数|数量|金额|价格|解决|办法|地区|季度|备注)/iu; + +function looksLikeTableHeader( + firstRow: readonly string[], + remainingRows: readonly (readonly string[])[], +): boolean { + if (firstRow.length < 2) return false; + const populated = firstRow.filter((cell) => cell.trim()); + const labelCount = populated.filter((cell) => + TABLE_HEADER_LABEL_PATTERN.test(cell.trim()), + ).length; + if (labelCount >= Math.min(2, populated.length)) return true; + + let typedColumns = 0; + for (let index = 0; index < firstRow.length; index += 1) { + const header = firstRow[index]?.trim() ?? ""; + if (!header || tableCellValueKind(header) !== "text") continue; + const values = remainingRows + .map((row) => row[index]?.trim() ?? "") + .filter(Boolean) + .map(tableCellValueKind); + if ( + values.length > 0 && + values.filter((kind) => kind !== "text").length / values.length >= 0.7 + ) { + typedColumns += 1; + } + } + return typedColumns > 0; +} + +function looksLikeKeyValueTable(rows: readonly (readonly string[])[]): boolean { + if (!rows.every((row) => row.length === 2)) return false; + const labels = rows.map((row) => row[0]?.trim() ?? ""); + if (labels.some((label) => !label) || new Set(labels).size !== labels.length) return false; + const recognized = labels.filter((label) => TABLE_HEADER_LABEL_PATTERN.test(label)).length; + return recognized / labels.length >= 0.6; +} + +function tableCellValueKind(value: string): "boolean" | "date" | "number" | "text" { + const normalized = value.trim(); + if (/^(?:true|false|yes|no|是|否)$/iu.test(normalized)) return "boolean"; + if (/^\d{4}[-/.年]\d{1,2}(?:[-/.月]\d{1,2}日?)?(?:\s|$)/u.test(normalized)) return "date"; + if (Number.isFinite(Number(normalized.replaceAll(",", "").replace(/[%￥¥$]/gu, "")))) { + return "number"; + } + return "text"; } function markdownImagesFromToken(token: Token): MarkdownImageRef[] { @@ -2081,6 +2392,10 @@ function normalizeText(text: string): string { .join("\n"); } +function normalizeTableCell(text: string): string { + return normalizeText(text).replace(/\n+/gu, " "); +} + function metadataString( metadata: Readonly>, key: string, diff --git a/knowledge-fs/packages/parsers/src/parser.test.ts b/knowledge-fs/packages/parsers/src/parser.test.ts index 8666ed238c6..2de9ba19023 100644 --- a/knowledge-fs/packages/parsers/src/parser.test.ts +++ b/knowledge-fs/packages/parsers/src/parser.test.ts @@ -108,7 +108,7 @@ describe("parser adapters", () => { metadata: { filename: "architecture.md", mimeType: "text/markdown", - parserVersion: "native-markdown@1", + parserVersion: "native-markdown@2", }, parser: "native-markdown", version: 1, @@ -152,9 +152,18 @@ describe("parser adapters", () => { }, { id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c45:element-6", - metadata: {}, + metadata: { + table: { + columns: ["A", "B"], + headerRowCount: 1, + mode: "single-record", + recordCount: 1, + semanticVersion: 1, + sourceRowCount: 2, + }, + }, sectionPath: ["Overview"], - text: "A | B\n1 | 2", + text: "A: 1 | B: 2", type: "table", }, ]); @@ -188,11 +197,11 @@ describe("parser adapters", () => { "Overview", "MDX keeps this searchable.\nNested detail", ]); - expect(artifact.metadata.parserVersion).toBe("native-mdx@1"); + expect(artifact.metadata.parserVersion).toBe("native-mdx@2"); }, ); - it("keeps plain Markdown raw HTML behavior and parser version unchanged", async () => { + it("keeps plain Markdown raw HTML behavior while using the table-aware parser version", async () => { const parser = createNativeMarkdownParser({ generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c96", now: () => createdAt, @@ -207,7 +216,38 @@ describe("parser adapters", () => { ); expect(artifact.elements).toEqual([]); - expect(artifact.metadata.parserVersion).toBe("native-markdown@1"); + expect(artifact.metadata.parserVersion).toBe("native-markdown@2"); + }); + + it("preserves the schema of a Markdown table that has no data records", async () => { + const parser = createNativeMarkdownParser({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca5", + now: () => createdAt, + }); + const artifact = await parser.parse( + createParseInput({ + body: "| Name | Status |\n| --- | --- |", + filename: "empty-table.md", + mimeType: "text/markdown", + }), + ); + + expect(artifact.elements).toEqual([ + expect.objectContaining({ + metadata: { + table: { + columns: ["Name", "Status"], + headerRowCount: 1, + mode: "unknown", + recordCount: 0, + semanticVersion: 1, + sourceRowCount: 1, + }, + }, + text: "Name | Status", + type: "table", + }), + ]); }); it("normalizes Markdown image references into image parse elements", async () => { @@ -276,7 +316,7 @@ describe("parser adapters", () => { expect(artifact.parser).toBe("native-html"); expect(artifact.metadata.documentTitle).toBe("Ignored Title"); - expect(artifact.metadata.parserVersion).toBe("native-html@2"); + expect(artifact.metadata.parserVersion).toBe("native-html@3"); expect(artifact.elements.map((element) => element.type)).toEqual([ "heading", "paragraph", @@ -289,8 +329,18 @@ describe("parser adapters", () => { "Read the docs.", "Install\nRun", "pnpm check", - "A | B\n1 | 2", + "A: 1 | B: 2", ]); + expect(artifact.elements.at(-1)?.metadata).toEqual({ + table: { + columns: ["A", "B"], + headerRowCount: 1, + mode: "single-record", + recordCount: 1, + semanticVersion: 1, + sourceRowCount: 2, + }, + }); expect(artifact.elements.map((element) => element.sectionPath)).toEqual([ ["Guide"], ["Guide"], @@ -300,6 +350,190 @@ describe("parser adapters", () => { ]); }); + it("distinguishes record lists, matrices, and key-value tables without changing table kind", async () => { + const parser = createNativeHtmlParser({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca1", + now: () => createdAt, + }); + const artifact = await parser.parse( + createParseInput({ + body: [ + "
姓名得分
Ada10
Lin9
", + "
地区Q1Q2
华东1020
华南1218
", + "
发票号码26322001
开票日期2026-08-26
价税合计566.00
", + ].join(""), + filename: "tables.html", + mimeType: "text/html", + }), + ); + + expect(artifact.elements).toEqual([ + expect.objectContaining({ + metadata: { + table: expect.objectContaining({ + columns: ["姓名", "得分"], + mode: "record-list", + recordCount: 2, + sourceRowCount: 3, + }), + }, + text: "姓名: Ada | 得分: 10\n姓名: Lin | 得分: 9", + type: "table", + }), + expect.objectContaining({ + metadata: { + table: expect.objectContaining({ + columns: ["地区", "Q1", "Q2"], + mode: "matrix", + recordCount: 2, + sourceRowCount: 3, + }), + }, + text: "地区: 华东 | Q1: 10 | Q2: 20\n地区: 华南 | Q1: 12 | Q2: 18", + type: "table", + }), + expect.objectContaining({ + metadata: { + table: expect.objectContaining({ + columns: ["发票号码", "开票日期", "价税合计"], + mode: "single-record", + recordCount: 1, + sourceRowCount: 3, + }), + }, + text: "发票号码: 26322001 | 开票日期: 2026-08-26 | 价税合计: 566.00", + type: "table", + }), + ]); + }); + + it("handles empty, header-group, inferred, duplicate, and genuinely headerless tables", async () => { + const parser = createNativeHtmlParser({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca3", + now: () => createdAt, + }); + const artifact = await parser.parse( + createParseInput({ + body: [ + "
", + "
orphan
", + "
AlphaBeta
Adaready
", + "
AlphaBeta
1true
2false
", + "
Adared
Linblue
", + "
NameName
AdaAlias
", + ].join(""), + filename: "edge-tables.html", + mimeType: "text/html", + }), + ); + + expect(artifact.elements.map((element) => element.text)).toEqual([ + "column_1: orphan", + "Alpha: Ada | Beta: ready", + "Alpha: 1 | Beta: true\nAlpha: 2 | Beta: false", + "column_1: Ada | column_2: red\ncolumn_1: Lin | column_2: blue", + "Name: Ada | Name_2: Alias | column_3:", + ]); + expect( + artifact.elements.map( + (element) => + (element.metadata.table as { mode?: string; sourceRowCount?: number } | undefined)?.mode, + ), + ).toEqual(["single-record", "single-record", "record-list", "record-list", "single-record"]); + expect( + (artifact.elements.at(-1)?.metadata.table as { columns?: string[] } | undefined)?.columns, + ).toEqual(["Name", "Name_2", "column_3"]); + }); + + it("keeps ambiguous headerless rows instead of dropping the first record", async () => { + const parser = createNativeHtmlParser({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca4", + now: () => createdAt, + }); + const artifact = await parser.parse( + createParseInput({ + body: [ + "
Ada
Lin
", + "
Statusopen
Statusclosed
", + "
open
otherclosed
", + "
AlphaBeta
2026-01-012026-02-01
2026-03-012026-04-01
", + ].join(""), + filename: "ambiguous-tables.html", + mimeType: "text/html", + }), + ); + + expect(artifact.elements.map((element) => element.text)).toEqual([ + "column_1: Ada\ncolumn_1: Lin", + "column_1: Status | column_2: open\ncolumn_1: Status | column_2: closed", + "column_1: | column_2: open\ncolumn_1: other | column_2: closed", + "Alpha: 2026-01-01 | Beta: 2026-02-01\nAlpha: 2026-03-01 | Beta: 2026-04-01", + ]); + expect( + artifact.elements.map( + (element) => (element.metadata.table as { mode?: string } | undefined)?.mode, + ), + ).toEqual(["record-list", "record-list", "record-list", "record-list"]); + }); + + it("flattens multi-row headers and HTML cell spans into a stable matrix schema", async () => { + const parser = createNativeHtmlParser({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca6", + now: () => createdAt, + }); + const artifact = await parser.parse( + createParseInput({ + body: [ + "", + '', + "", + "", + "", + "
地区收入
Q1Q2
华东1020
华南1218
", + ].join(""), + filename: "matrix.html", + mimeType: "text/html", + }), + ); + + expect(artifact.elements).toEqual([ + expect.objectContaining({ + metadata: { + table: { + columns: ["地区", "收入 / Q1", "收入 / Q2"], + headerRowCount: 2, + mode: "matrix", + recordCount: 2, + semanticVersion: 1, + sourceRowCount: 4, + }, + }, + text: "地区: 华东 | 收入 / Q1: 10 | 收入 / Q2: 20\n地区: 华南 | 收入 / Q1: 12 | 收入 / Q2: 18", + type: "table", + }), + ]); + }); + + it("bounds invalid HTML row and column spans instead of expanding them", async () => { + const parser = createNativeHtmlParser({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca8", + now: () => createdAt, + }); + const artifact = await parser.parse( + createParseInput({ + body: '
Value
A
', + filename: "bounded-spans.html", + mimeType: "text/html", + }), + ); + + expect(artifact.elements[0]).toMatchObject({ + metadata: { table: { columns: ["Value"], recordCount: 1 } }, + text: "Value: A", + type: "table", + }); + }); + it("bounds an HTML metadata title without adding it to body elements", async () => { const parser = createNativeHtmlParser({ generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c47", @@ -742,15 +976,26 @@ describe("parser adapters", () => { contentType: "structured", elements: [ { - metadata: { columns: ["name", "score"], format: "csv", rowCount: 2 }, - text: "name | score\nAda | 10\nLin | 9", + metadata: { + columns: ["name", "score"], + format: "csv", + rowCount: 2, + table: { + columns: ["name", "score"], + headerRowCount: 1, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + }, + }, + text: "name: Ada | score: 10\nname: Lin | score: 9", type: "table", }, ], metadata: { filename: "scores.csv", mimeType: "text/csv", - parserVersion: "native-structured@1", + parserVersion: "native-structured@2", }, parser: "native-structured", }); @@ -782,8 +1027,20 @@ describe("parser adapters", () => { ).resolves.toMatchObject({ elements: [ { - metadata: { columns: ["name"], format: "jsonl", rowCount: 2 }, - text: "name\nAda\nLin", + metadata: { + columns: ["name"], + format: "jsonl", + rowCount: 2, + table: { + columns: ["name"], + headerRowCount: 0, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + sourceRowCount: 2, + }, + }, + text: "name: Ada\nname: Lin", type: "table", }, ], @@ -841,8 +1098,20 @@ describe("parser adapters", () => { ).resolves.toMatchObject({ elements: [ { - metadata: { columns: ["name"], format: "jsonl", rowCount: 2 }, - text: "name\nAda\nLin", + metadata: { + columns: ["name"], + format: "jsonl", + rowCount: 2, + table: { + columns: ["name"], + headerRowCount: 0, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + sourceRowCount: 2, + }, + }, + text: "name: Ada\nname: Lin", type: "table", }, ], @@ -1049,7 +1318,7 @@ describe("parser adapters", () => { metadata: { filename: "report.pdf", mimeType: "application/pdf", - parserVersion: "unstructured@6", + parserVersion: "unstructured@7", }, parser: "unstructured", version: 1, @@ -1097,18 +1366,126 @@ describe("parser adapters", () => { { id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c50:element-4", metadata: { - table: { html: "
ARR
" }, + table: { + columns: ["column_1"], + headerRowCount: 0, + html: "
ARR
", + mode: "single-record", + recordCount: 1, + semanticVersion: 1, + sourceRowCount: 1, + }, textAsHtml: "
ARR
", unstructuredType: "Table", }, pageNumber: 3, sectionPath: ["Executive Summary"], - text: "ARR", + text: "column_1: ARR", type: "table", }, ]); }); + it("projects an Unstructured spreadsheet table into independently retrievable records", async () => { + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch: async () => + new Response( + JSON.stringify([ + { + metadata: { + page_name: "问题收集", + text_as_html: + "
时间问题描述报错详情（截图等）问题严重等级是否解决解决时间解决办法
2026-07-01复制按钮无法点击Web 地址P3是2026-07-02升级版本
2026-07-08服务器日期不对P3是2026-07-09调整时区
", + }, + text: "flattened provider text is not used", + type: "Table", + }, + ]), + { headers: { "content-type": "application/json" }, status: 200 }, + ), + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca2", + now: () => createdAt, + }); + + const artifact = await parser.parse({ + body: new Uint8Array([1, 2, 3]), + documentAssetId, + filename: "issues.xlsx", + mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + version: 1, + }); + + expect(artifact.elements).toHaveLength(1); + expect(artifact.elements[0]).toMatchObject({ + metadata: { + page_name: "问题收集", + table: { + columns: [ + "时间", + "问题描述", + "报错详情（截图等）", + "问题严重等级", + "是否解决", + "解决时间", + "解决办法", + ], + headerRowCount: 1, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + sourceRowCount: 3, + }, + }, + text: expect.stringContaining("时间: 2026-07-01 | 问题描述: 复制按钮无法点击"), + type: "table", + }); + expect(artifact.elements[0]?.text?.split("\n")).toHaveLength(2); + }); + + it("keeps worksheets as separate table elements with their own record schemas", async () => { + const table = (sheet: string, value: string) => ({ + metadata: { + page_name: sheet, + text_as_html: `
sheetvalue
${sheet}${value}
`, + }, + text: `${sheet} ${value}`, + type: "Table", + }); + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch: async () => + new Response(JSON.stringify([table("问题收集", "7"), table("归档", "12")]), { + headers: { "content-type": "application/json" }, + status: 200, + }), + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca7", + now: () => createdAt, + }); + const artifact = await parser.parse({ + body: new Uint8Array([1, 2, 3]), + documentAssetId, + filename: "multi-sheet.xlsx", + mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + version: 1, + }); + + expect(artifact.elements.map((element) => element.metadata.page_name)).toEqual([ + "问题收集", + "归档", + ]); + expect(artifact.elements.map((element) => element.text)).toEqual([ + "sheet: 问题收集 | value: 7", + "sheet: 归档 | value: 12", + ]); + expect( + artifact.elements.every( + (element) => + (element.metadata.table as { recordCount?: number } | undefined)?.recordCount === 1, + ), + ).toBe(true); + }); + it.each(["application/pdf", " Application/PDF; charset=binary "])( "requests PDF image and table payloads for MIME %s without an external handler", async (mimeType) => { @@ -1138,7 +1515,7 @@ describe("parser adapters", () => { version: 1, }), ).resolves.toMatchObject({ - metadata: { parserVersion: "unstructured@6" }, + metadata: { parserVersion: "unstructured@7" }, parser: "unstructured", }); }, @@ -2426,8 +2803,42 @@ describe("structured data parser coverage", () => { const table = artifact.elements[0]; expect(table?.type).toBe("table"); - expect(table?.text).toContain("name | count | flag | nested | empty | extra"); - expect(table?.text).toContain('a | 1 | true | {"x":1} | | '); + expect(table?.metadata).toMatchObject({ + table: { + columns: ["name", "count", "flag", "nested", "empty", "extra"], + headerRowCount: 0, + mode: "record-list", + recordCount: 2, + semanticVersion: 1, + }, + }); + expect(table?.text).toContain( + 'name: a | count: 1 | flag: true | nested: {"x":1} | empty: | extra: ', + ); + expect(table?.text).toContain("name: b | count: | flag: | nested: | empty: | extra: y"); + }); + + it("keeps quoted CSV newlines and delimiters inside their logical record", async () => { + const artifact = await structured().parse( + createParseInput({ + body: 'id,description\n1,"first line\nsecond line"\n2,"contains, comma"', + filename: "quoted.csv", + mimeType: "text/csv", + }), + ); + + expect(artifact.elements[0]).toMatchObject({ + metadata: { + table: { + columns: ["id", "description"], + mode: "record-list", + recordCount: 2, + sourceRowCount: 3, + }, + }, + text: "id: 1 | description: first line second line\nid: 2 | description: contains, comma", + type: "table", + }); }); it("renders non-tabular JSON as a code element with root type metadata", async () => {