diff --git a/knowledge-fs/apps/api/src/index.ts b/knowledge-fs/apps/api/src/index.ts index 112c54c228c..aa3c126e084 100644 --- a/knowledge-fs/apps/api/src/index.ts +++ b/knowledge-fs/apps/api/src/index.ts @@ -459,6 +459,8 @@ const documentSemanticChunker = createLlmSemanticChunker({ : {}), maxConcurrentWindows: ingestionModelRuntimeOptions.semanticExtractionMaxConcurrency, maxNodes: 20_000, + maxProviderOutputRetries: 1, + maxWindowChars: ingestionModelRuntimeOptions.semanticChunkingMaxWindowChars, metrics: operationalMetrics.ingestionModelCalls, modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate, reasoningProviderFactory: profileReasoningCapability.providerFactory, diff --git a/knowledge-fs/apps/api/src/ingestion-model-runtime-options.test.ts b/knowledge-fs/apps/api/src/ingestion-model-runtime-options.test.ts index 168f90d5ea7..7ca00577d1b 100644 --- a/knowledge-fs/apps/api/src/ingestion-model-runtime-options.test.ts +++ b/knowledge-fs/apps/api/src/ingestion-model-runtime-options.test.ts @@ -14,6 +14,7 @@ describe("createApiIngestionModelRuntimeOptions", () => { expect(options.outlineSummaryMaxConcurrency).toBe(8); expect(options.semanticExtractionBatchSize).toBe(8); expect(options.semanticExtractionMaxConcurrency).toBe(4); + expect(options.semanticChunkingMaxWindowChars).toBe(4_800); }); it("accepts bounded concurrency overrides", () => { @@ -26,6 +27,7 @@ describe("createApiIngestionModelRuntimeOptions", () => { KNOWLEDGE_OUTLINE_SUMMARY_MAX_CONCURRENCY: "12", KNOWLEDGE_SEMANTIC_EXTRACTION_BATCH_SIZE: "10", KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY: "6", + KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS: "2400", }); expect(options.globalConcurrency).toBe(24); @@ -36,6 +38,7 @@ describe("createApiIngestionModelRuntimeOptions", () => { expect(options.outlineSummaryMaxConcurrency).toBe(12); expect(options.semanticExtractionBatchSize).toBe(10); expect(options.semanticExtractionMaxConcurrency).toBe(6); + expect(options.semanticChunkingMaxWindowChars).toBe(2_400); }); it("creates an isolated hard budget for each document", () => { @@ -84,6 +87,7 @@ describe("createApiIngestionModelRuntimeOptions", () => { ["KNOWLEDGE_OUTLINE_SUMMARY_BATCH_MAX_INPUT_CHARS", "200001"], ["KNOWLEDGE_SEMANTIC_EXTRACTION_BATCH_SIZE", "33"], ["KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY", "0"], + ["KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS", "200001"], ] as const)("rejects invalid %s=%s", (name, value) => { expect(() => createApiIngestionModelRuntimeOptions({ [name]: value })).toThrow(name); }); diff --git a/knowledge-fs/apps/api/src/ingestion-model-runtime-options.ts b/knowledge-fs/apps/api/src/ingestion-model-runtime-options.ts index 9a9642317ce..eef15493897 100644 --- a/knowledge-fs/apps/api/src/ingestion-model-runtime-options.ts +++ b/knowledge-fs/apps/api/src/ingestion-model-runtime-options.ts @@ -15,6 +15,7 @@ export interface ApiIngestionModelRuntimeEnv { readonly KNOWLEDGE_OUTLINE_SUMMARY_MAX_CONCURRENCY?: string | undefined; readonly KNOWLEDGE_SEMANTIC_EXTRACTION_BATCH_SIZE?: string | undefined; readonly KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY?: string | undefined; + readonly KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS?: string | undefined; } export interface ApiIngestionModelRuntimeOptions { @@ -28,6 +29,7 @@ export interface ApiIngestionModelRuntimeOptions { readonly outlineSummaryMaxConcurrency: number; readonly semanticExtractionBatchSize: number; readonly semanticExtractionMaxConcurrency: number; + readonly semanticChunkingMaxWindowChars: number; } export interface ApiIngestionModelRuntimeMetrics { @@ -48,8 +50,10 @@ const MAX_OUTLINE_SUMMARY_BATCH_SIZE = 32; const MAX_OUTLINE_SUMMARY_CONCURRENCY = 32; const DEFAULT_SEMANTIC_EXTRACTION_BATCH_SIZE = 8; const DEFAULT_SEMANTIC_EXTRACTION_MAX_CONCURRENCY = 4; +const DEFAULT_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS = 4_800; const MAX_SEMANTIC_EXTRACTION_BATCH_SIZE = 32; const MAX_SEMANTIC_EXTRACTION_CONCURRENCY = 32; +const MAX_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS = 200_000; /** * Builds the shared ingestion-time model budget. The per-document outline bound protects fairness @@ -108,6 +112,12 @@ export function createApiIngestionModelRuntimeOptions( name: "KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY", value: env.KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY, }); + const semanticChunkingMaxWindowChars = boundedPositiveIntegerEnv({ + fallback: DEFAULT_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS, + max: MAX_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS, + name: "KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS", + value: env.KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS, + }); return { createDocumentModelBudget: () => @@ -126,6 +136,7 @@ export function createApiIngestionModelRuntimeOptions( outlineSummaryMaxConcurrency, semanticExtractionBatchSize, semanticExtractionMaxConcurrency, + semanticChunkingMaxWindowChars, }; } diff --git a/knowledge-fs/apps/api/src/parser-options.test.ts b/knowledge-fs/apps/api/src/parser-options.test.ts index 5c0e9db1eee..667d517679b 100644 --- a/knowledge-fs/apps/api/src/parser-options.test.ts +++ b/knowledge-fs/apps/api/src/parser-options.test.ts @@ -79,6 +79,31 @@ describe("createApiDocumentParser", () => { }); }); + it("forwards the configured default language to Unstructured", async () => { + let requestedLanguage: FormDataEntryValue | null = null; + const parser = createApiDocumentParser({ + env: { + UNSTRUCTURED_API_URL: "https://unstructured.example.test", + UNSTRUCTURED_DEFAULT_LANGUAGE: "zh-CN", + }, + fetch: async (input) => { + const request = input instanceof Request ? input : new Request(input); + requestedLanguage = (await request.formData()).get("languages"); + return new Response("[]", { headers: { "content-type": "application/json" } }); + }, + }); + + await parser.parse({ + body: encoder.encode("%PDF-1.7"), + documentAssetId: "00000000-0000-4000-8000-000000000007", + filename: "report.pdf", + mimeType: "application/pdf", + version: 1, + }); + + expect(requestedLanguage).toBe("zho"); + }); + it("can derive the local Unstructured URL from UNSTRUCTURED_PORT outside production", async () => { let requestedUrl = ""; const parser = createApiDocumentParser({ diff --git a/knowledge-fs/apps/api/src/parser-options.ts b/knowledge-fs/apps/api/src/parser-options.ts index f5340ff99f1..87a773424a8 100644 --- a/knowledge-fs/apps/api/src/parser-options.ts +++ b/knowledge-fs/apps/api/src/parser-options.ts @@ -11,6 +11,7 @@ export interface ApiParserEnv { readonly NODE_ENV?: string | undefined; readonly UNSTRUCTURED_API_KEY?: string | undefined; readonly UNSTRUCTURED_API_URL?: string | undefined; + readonly UNSTRUCTURED_DEFAULT_LANGUAGE?: string | undefined; readonly UNSTRUCTURED_MAX_CONCURRENCY?: string | undefined; readonly UNSTRUCTURED_MAX_RESPONSE_BYTES?: string | undefined; readonly UNSTRUCTURED_MAX_RETRIES?: string | undefined; @@ -60,6 +61,9 @@ function createApiUnstructuredParser({ } return createUnstructuredParserClient({ + ...(env.UNSTRUCTURED_DEFAULT_LANGUAGE?.trim() + ? { defaultLanguage: env.UNSTRUCTURED_DEFAULT_LANGUAGE.trim() } + : {}), endpoint, ...(env.UNSTRUCTURED_API_KEY?.trim() ? { apiKey: env.UNSTRUCTURED_API_KEY.trim() } : {}), ...(fetchImpl ? { fetch: fetchImpl } : {}), diff --git a/knowledge-fs/infra/local/.env.example b/knowledge-fs/infra/local/.env.example index f7ef70b5045..7126e55652b 100644 --- a/knowledge-fs/infra/local/.env.example +++ b/knowledge-fs/infra/local/.env.example @@ -14,6 +14,8 @@ UNSTRUCTURED_PORT=8000 UNSTRUCTURED_API_URL=http://127.0.0.1:8000 UNSTRUCTURED_API_KEY= UNSTRUCTURED_MAX_CONCURRENCY=2 +UNSTRUCTURED_DEFAULT_LANGUAGE= +KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS=4800 UNSTRUCTURED_REQUEST_TIMEOUT_MS=120000 UNSTRUCTURED_MAX_RESPONSE_BYTES=33554432 UNSTRUCTURED_MAX_RETRIES= diff --git a/knowledge-fs/infra/local/compose.yaml b/knowledge-fs/infra/local/compose.yaml index 25c47f7705e..02766559ace 100644 --- a/knowledge-fs/infra/local/compose.yaml +++ b/knowledge-fs/infra/local/compose.yaml @@ -61,6 +61,8 @@ services: KNOWLEDGE_DEV_TENANT_ID: ${KNOWLEDGE_DEV_TENANT_ID:-tenant-dev} UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL:-http://unstructured:8000} UNSTRUCTURED_MAX_CONCURRENCY: ${UNSTRUCTURED_MAX_CONCURRENCY:-2} + UNSTRUCTURED_DEFAULT_LANGUAGE: ${UNSTRUCTURED_DEFAULT_LANGUAGE:-} + KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS: ${KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS:-4800} UNSTRUCTURED_REQUEST_TIMEOUT_MS: ${UNSTRUCTURED_REQUEST_TIMEOUT_MS:-120000} UNSTRUCTURED_MAX_RESPONSE_BYTES: ${UNSTRUCTURED_MAX_RESPONSE_BYTES:-33554432} UNSTRUCTURED_MAX_RETRIES: ${UNSTRUCTURED_MAX_RETRIES:-0} diff --git a/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.test.ts b/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.test.ts index 9ec6524cd6b..31f6e9be4e8 100644 --- a/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.test.ts +++ b/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.test.ts @@ -118,8 +118,18 @@ describe("document compilation candidate runtime factories", () => { ); expect(resolved.material.projections).toEqual( expect.arrayContaining([ - expect.objectContaining({ model: vectorSpaceId, type: "dense-vector" }), - expect.objectContaining({ model: "database-fts@1", type: "fts" }), + expect.objectContaining({ + indexVersion: "dify-model-runtime-embedding-v1", + model: vectorSpaceId, + strategy: "text-dense-v1:section-context-v1", + type: "dense-vector", + }), + expect.objectContaining({ + indexVersion: "database-fts-v1", + model: "database-fts@1", + strategy: "mixed-cjk-latin-fts-v1:section-context-v1", + type: "fts", + }), ]), ); }); diff --git a/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.ts b/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.ts index b3a8a6a88ff..e9536bafbac 100644 --- a/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.ts +++ b/knowledge-fs/packages/api/src/document-compilation-candidate-runtime.ts @@ -42,6 +42,7 @@ import type { ProjectionSetPublicationMemberRepository, } from "./projection-publication-member-repository"; import type { ProjectionSetPublicationRepository } from "./projection-publication-repository"; +import { TEXT_INDEXING_STRATEGY } from "./text-indexing-strategy"; export interface DocumentCompilationFingerprintVersions { readonly chunkerVersion: string; @@ -655,11 +656,11 @@ function projectionFingerprintConfig( const visual = isVisualProjection(projection); const strategy = projection.type === "fts" - ? "mixed-cjk-latin-fts-v1" + ? `mixed-cjk-latin-fts-v1:${TEXT_INDEXING_STRATEGY}` : visual ? "visual-dense-v1" : projection.type === "dense-vector" - ? "text-dense-v1" + ? `text-dense-v1:${TEXT_INDEXING_STRATEGY}` : `${projection.type}-v1`; return { indexVersion: diff --git a/knowledge-fs/packages/api/src/gateway.test.ts b/knowledge-fs/packages/api/src/gateway.test.ts index 672b52d09e2..0d6e90e3cc3 100644 --- a/knowledge-fs/packages/api/src/gateway.test.ts +++ b/knowledge-fs/packages/api/src/gateway.test.ts @@ -8842,7 +8842,7 @@ describe("createKnowledgeGateway", () => { expect(embedding.calls[0]).toEqual({ inputType: "search_document", model: "static-upgrade@2026-05-01", - texts: ["First chunk", "Second chunk"], + texts: ["Intro\n\nFirst chunk", "Intro\n\nSecond chunk"], }); expect(evaluationCalls[0]).toEqual({ denseProjectionModel: "static-upgrade@2026-05-01", @@ -9132,7 +9132,7 @@ describe("createKnowledgeGateway", () => { { inputType: "search_document", model: "static-dense", - texts: ["First chunk", "Second chunk"], + texts: ["Intro\n\nFirst chunk", "Intro\n\nSecond chunk"], }, ]); expect(projections).toEqual([ @@ -9141,7 +9141,7 @@ describe("createKnowledgeGateway", () => { knowledgeSpaceId: firstNode.knowledgeSpaceId, metadata: expect.objectContaining({ artifactHash: firstNode.artifactHash, - denseVector: [0.1, 11], + denseVector: [0.1, 18], dimension: 2, embeddingProvider: "static", modelVersion: "static-dense", @@ -9153,7 +9153,7 @@ describe("createKnowledgeGateway", () => { type: "dense-vector", }), expect.objectContaining({ - metadata: expect.objectContaining({ denseVector: [1.1, 12] }), + metadata: expect.objectContaining({ denseVector: [1.1, 19] }), nodeId: secondNode.id, }), ]); @@ -9187,7 +9187,7 @@ describe("createKnowledgeGateway", () => { type: "dense-vector", }) ).items[0]?.metadata, - ).toEqual(expect.objectContaining({ denseVector: [0.1, 11] })); + ).toEqual(expect.objectContaining({ denseVector: [0.1, 18] })); await expect( memoryRepository.listReadyBySpace({ knowledgeSpaceId: firstNode.knowledgeSpaceId, @@ -9394,7 +9394,7 @@ describe("createKnowledgeGateway", () => { ); expect(fake.calls[0]?.sql).toContain("dense_vector"); expect(fake.calls[0]?.sql).not.toContain("First chunk"); - expect(fake.calls[0]?.params).toContain("[0.1,11]"); + expect(fake.calls[0]?.params).toContain("[0.1,18]"); expect(fake.calls[0]?.params).toContain(JSON.stringify(firstProjection.metadata)); await expect( databaseRepository.listReadyBySpace({ @@ -9635,7 +9635,8 @@ describe("createKnowledgeGateway", () => { metadata: expect.objectContaining({ artifactHash: firstNode.artifactHash, ftsLanguageStrategy: "mixed-cjk-latin-v1", - ftsText: "contract abc 123 renewal terms", + ftsText: "intro contract abc 123 renewal terms", + indexingStrategy: "section-context-v1", parser: "database-fts", }), model: "database-fts@1", @@ -9646,7 +9647,7 @@ describe("createKnowledgeGateway", () => { }), expect.objectContaining({ metadata: expect.objectContaining({ - ftsText: "error code e 42 remediation", + ftsText: "intro error code e 42 remediation", }), nodeId: secondNode.id, }), @@ -9680,7 +9681,7 @@ describe("createKnowledgeGateway", () => { ); expect(fake.calls[0]?.sql).toContain("to_tsvector('simple'"); expect(fake.calls[0]?.sql).not.toContain("Contract ABC-123"); - expect(fake.calls[0]?.params).toContain("contract abc 123 renewal terms"); + expect(fake.calls[0]?.params).toContain("intro contract abc 123 renewal terms"); expect(fake.calls[0]?.params).toContain(JSON.stringify(firstProjection.metadata)); const tidbFake = createFakeIndexProjectionExecutor(); const tidbRepository = createDatabaseIndexProjectionRepository({ @@ -9695,7 +9696,7 @@ describe("createKnowledgeGateway", () => { await expect(tidbRepository.createMany([firstProjection])).resolves.toEqual([firstProjection]); expect(tidbFake.calls[0]?.sql).toContain("INSERT INTO `index_projections`"); expect(tidbFake.calls[0]?.sql).not.toContain("to_tsvector"); - expect(tidbFake.calls[0]?.params).toContain("contract abc 123 renewal terms"); + expect(tidbFake.calls[0]?.params).toContain("intro contract abc 123 renewal terms"); }); it("idempotently reindexes parse artifacts so interrupted projection builds can be repaired", async () => { @@ -9980,7 +9981,8 @@ describe("createKnowledgeGateway", () => { expect(projections[0]?.metadata).toEqual( expect.objectContaining({ ftsLanguageStrategy: "mixed-cjk-latin-v1", - ftsText: "合 同 abc 123 续 约 terms", + ftsText: "intro 合 同 abc 123 续 约 terms", + indexingStrategy: "section-context-v1", }), ); }); diff --git a/knowledge-fs/packages/api/src/index-projection-builders.test.ts b/knowledge-fs/packages/api/src/index-projection-builders.test.ts index 8942b9a9b7a..db9bcb885b3 100644 --- a/knowledge-fs/packages/api/src/index-projection-builders.test.ts +++ b/knowledge-fs/packages/api/src/index-projection-builders.test.ts @@ -235,7 +235,11 @@ describe("index projection builders", () => { }); expect(embedCalls).toEqual([ - { inputType: "search_document", model: "model-a", texts: ["合同ABC-123续约 terms"] }, + { + inputType: "search_document", + model: "model-a", + texts: ["Intro\n\n合同ABC-123续约 terms"], + }, ]); expect(result[0]).toMatchObject({ id: "018f0d60-7a49-7cc2-9c1b-5b36f18f9000", @@ -298,7 +302,7 @@ describe("index projection builders", () => { inputType: "search_document", model: "tenant-model", tenantId: "tenant-1", - texts: ["合同ABC-123续约 terms"], + texts: ["Intro\n\n合同ABC-123续约 terms"], }, ]); expect(projection).toMatchObject({ @@ -527,7 +531,8 @@ describe("index projection builders", () => { expect(projection).toMatchObject({ metadata: { ftsLanguageStrategy: "mixed-cjk-latin-v1", - ftsText: "合 同 abc 123 续 约 terms", + ftsText: "intro 合 同 abc 123 续 约 terms", + indexingStrategy: "section-context-v1", parser: "database-fts", }, model: "database-fts@1", @@ -536,6 +541,59 @@ describe("index projection builders", () => { }); }); + it("indexes semantic section paths and summaries while preserving the node source text", async () => { + const embedCalls: EmbedTextsInput[] = []; + const embeddings: EmbeddingProvider = { + embed: async (input) => { + embedCalls.push(input); + return { + dense: [[0.1, 0.2]], + metadata: { model: "model-a@1", provider: "static" }, + model: "model-a@1", + }; + }, + kind: "static", + models: async () => [], + }; + const denseRepository = createRecordingProjectionRepository(); + const ftsRepository = createRecordingProjectionRepository(); + const node = knowledgeNode({ + metadata: { + chunkIndex: 0, + semanticChunking: { + section: { + path: ["Safety", "Tunnel"], + summary: "Checks excavation support and edge protection.", + }, + }, + }, + sourceLocation: { + endOffset: 12, + sectionPath: ["Safety", "Tunnel", "Initial support"], + startOffset: 0, + }, + text: "Verify the current work face.", + }); + + await createDenseVectorProjectionBuilder({ + embeddings, + maxBatchSize: 1, + projections: denseRepository.repository, + }).build({ model: "model-a", nodes: [node], projectionVersion: 1 }); + const [fts] = await createFtsProjectionBuilder({ + maxBatchSize: 1, + projections: ftsRepository.repository, + }).build({ nodes: [node], projectionVersion: 1 }); + + expect(embedCalls[0]?.texts).toEqual([ + "Safety > Tunnel > Initial support\n\nChecks excavation support and edge protection.\n\nVerify the current work face.", + ]); + expect(fts?.metadata.ftsText).toBe( + "safety tunnel initial support checks excavation support and edge protection verify the current work face", + ); + expect(node.text).toBe("Verify the current work face."); + }); + it("skips FTS projections for nodes without searchable text", async () => { const { created, repository } = createRecordingProjectionRepository(); const builder = createFtsProjectionBuilder({ diff --git a/knowledge-fs/packages/api/src/index-projection-builders.ts b/knowledge-fs/packages/api/src/index-projection-builders.ts index b24fb599c0a..13a3bd522ea 100644 --- a/knowledge-fs/packages/api/src/index-projection-builders.ts +++ b/knowledge-fs/packages/api/src/index-projection-builders.ts @@ -29,6 +29,7 @@ import { assertObservedEmbeddingDimension, } from "./knowledge-space-embedding-resolver"; import { normalizeMixedLanguageFtsText } from "./retrieval-text-utils"; +import { TEXT_INDEXING_STRATEGY } from "./text-indexing-strategy"; export interface BuildDenseVectorProjectionInput { /** Immutable profile captured by a compilation/profile-migration attempt. */ @@ -258,6 +259,7 @@ export function createDenseVectorProjectionBuilder({ }) : new Map(); const nodesToEmbed = parsedNodes.filter((node) => !reusableByNodeId.has(node.id)); + const embeddingTexts = nodesToEmbed.map(textIndexContentForNode); if (nodesToEmbed.length === 0) { recordIngestionModelCallMetric(metrics, { cacheHits: parsedNodes.length, @@ -273,8 +275,8 @@ export function createDenseVectorProjectionBuilder({ ); } modelBudget?.reserve({ - estimatedTokens: nodesToEmbed.reduce( - (total, node) => total + estimateDocumentModelTokens(node.text), + estimatedTokens: embeddingTexts.reduce( + (total, text) => total + estimateDocumentModelTokens(text), 0, ), itemCount: nodesToEmbed.length, @@ -287,7 +289,7 @@ export function createDenseVectorProjectionBuilder({ inputType: "search_document", model: resolvedEmbedding?.model ?? model, ...(signal ? { signal } : {}), - texts: nodesToEmbed.map((node) => node.text), + texts: embeddingTexts, ...(tenantId ? { tenantId } : {}), }); } catch (error) { @@ -373,6 +375,7 @@ export function createDenseVectorProjectionBuilder({ dimension: responseDimension, documentAssetId: node.documentAssetId, embeddingProvider: result.metadata.provider, + indexingStrategy: TEXT_INDEXING_STRATEGY, embeddingModel: result.model, ...(resolvedEmbedding ? { @@ -457,7 +460,8 @@ async function loadReusableDenseProjections({ projection.type !== "dense-vector" || projection.metadata.artifactHash !== node.artifactHash || projection.metadata.documentAssetId !== node.documentAssetId || - projection.metadata.parseArtifactId !== node.parseArtifactId + projection.metadata.parseArtifactId !== node.parseArtifactId || + projection.metadata.indexingStrategy !== TEXT_INDEXING_STRATEGY ) { throw new Error( `Persisted generation-scoped dense projection id=${projection.id} cannot be reused`, @@ -498,10 +502,11 @@ export function createFtsProjectionBuilder({ const parsedNodes = nodes.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); const ftsProjections = parsedNodes.flatMap((node) => { - const ftsText = normalizeMixedLanguageFtsText(node.text); - if (!ftsText) { + const sourceFtsText = normalizeMixedLanguageFtsText(node.text); + if (!sourceFtsText) { return []; } + const ftsText = normalizeMixedLanguageFtsText(textIndexContentForNode(node)); return [ IndexProjectionSchema.parse({ @@ -520,6 +525,7 @@ export function createFtsProjectionBuilder({ documentAssetId: node.documentAssetId, ftsLanguageStrategy: "mixed-cjk-latin-v1", ftsText, + indexingStrategy: TEXT_INDEXING_STRATEGY, ...multimodalProjectionMetadata(node), parseArtifactId: node.parseArtifactId, parser: "database-fts", @@ -564,7 +570,9 @@ export function createFtsProjectionBuilder({ projection.type !== "fts" || projection.metadata.artifactHash !== incoming.metadata.artifactHash || projection.metadata.documentAssetId !== incoming.metadata.documentAssetId || - projection.metadata.parseArtifactId !== incoming.metadata.parseArtifactId + projection.metadata.parseArtifactId !== incoming.metadata.parseArtifactId || + projection.metadata.indexingStrategy !== TEXT_INDEXING_STRATEGY || + projection.metadata.ftsText !== incoming.metadata.ftsText ) { throw new Error( `Persisted generation-scoped FTS projection id=${projection.id} cannot be reused`, @@ -594,6 +602,28 @@ export function createFtsProjectionBuilder({ }; } +/** + * Add trusted semantic navigation context to search indexes without changing the cited source + * text stored on the knowledge node. Table rows and list continuations commonly omit their parent + * heading, so indexing only `node.text` makes them impossible to recall from a heading-led query. + */ +function textIndexContentForNode(node: KnowledgeNode): string { + const sectionPath = node.sourceLocation.sectionPath + .map((segment) => segment.trim()) + .filter(Boolean) + .join(" > "); + const semanticChunking = isPlainObject(node.metadata.semanticChunking) + ? node.metadata.semanticChunking + : undefined; + const section = + semanticChunking && isPlainObject(semanticChunking.section) + ? semanticChunking.section + : undefined; + const summary = section && typeof section.summary === "string" ? section.summary.trim() : ""; + + return [sectionPath, summary, node.text.trim()].filter(Boolean).join("\n\n"); +} + export function createVisualEmbeddingProjectionBuilder({ generateId, maxBatchSize, diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts index d41aad73ff1..46e4a3aae3b 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts @@ -303,7 +303,7 @@ describe("LLM semantic chunker", () => { completed: true, entityCount: 2, model: "reasoner-model", - promptVersion: "semantic-chunking-v5", + promptVersion: "semantic-chunking-v6", }, relationExtraction: { completed: true, relationCount: 1 }, semanticChunking: { @@ -353,6 +353,9 @@ describe("LLM semantic chunker", () => { expect(provider.calls[0]?.messages[0]?.content).toContain( "prefer natural topic boundaries over filling chunks", ); + expect(provider.calls[0]?.messages[0]?.content).toContain( + "keep the complete list together under its heading", + ); }); it("hard-splits an overlong sentence by Unicode grapheme without overlap", async () => { @@ -557,7 +560,7 @@ describe("LLM semantic chunker", () => { windowPlanning: { atomicDocument: false, sourceSectionPathCount: 2, - version: "v5", + version: "v6", }, }); }); @@ -633,13 +636,61 @@ describe("LLM semantic chunker", () => { parseArtifact.elements.map((element) => element.text?.trim() ?? "").join("\n"), ); expect(nodes[0]?.metadata.semanticChunking).toMatchObject({ - windowPlanning: { version: "v5" }, + windowPlanning: { version: "v6" }, }); expect(v2Nodes[0]?.metadata.semanticChunking).toMatchObject({ windowPlanning: { version: "v2" }, }); }); + it("extends a fixed core boundary to keep a numbered list in one window", async () => { + const parseArtifact = artifact([ + ...Array.from({ length: 24 }, (_, index) => ({ + id: `preamble-${index}`, + metadata: {}, + sectionPath: ["Preamble"], + text: `Preamble ${index}.`, + type: "paragraph" as const, + })), + { + id: "list-heading", + metadata: {}, + sectionPath: ["Negative list"], + text: "施工企业安全穿透式管理负面清单", + type: "paragraph" as const, + }, + ...Array.from({ length: 9 }, (_, index) => ({ + id: `list-item-${index + 1}`, + metadata: {}, + sectionPath: ["Negative list"], + text: `${index + 1}. 第 ${index + 1} 项负面情形。`, + type: "paragraph" as const, + })), + ]); + const provider = new ScriptedProvider([echoEachUnit]); + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + expect(preflightLlmSemanticWindows({ parseArtifact })).toEqual({ + maximumWindowCount: 1, + unitCount: 43, + }); + expect(provider.calls).toHaveLength(1); + const prompt = JSON.parse( + provider.calls[0]?.messages.find((message) => message.role === "user")?.content ?? "{}", + ) as PromptPayload; + expect(prompt.units).toHaveLength(43); + expect(prompt.units.at(-1)?.text).toBe("第 9 项负面情形。"); + expect(nodes.map((node) => node.text).join("")).toBe( + parseArtifact.elements.map((element) => element.text?.trim() ?? "").join(""), + ); + }); + it("runs fixed-core semantic windows concurrently while preserving document order", async () => { const parseArtifact = artifact( Array.from({ length: 96 }, (_, index) => ({ @@ -870,7 +921,7 @@ describe("LLM semantic chunker", () => { windowPlanning: { atomicDocument: true, sourceSectionPathCount: 1, - version: "v5", + version: "v6", }, }); expect(nodes[0]?.metadata.extractedEntities).toEqual([ @@ -2351,6 +2402,18 @@ describe("LLM semantic chunker", () => { expect(() => createLlmSemanticChunker({ reasoningProviderFactory: factory, temperature: -1 }), ).toThrow("temperature must be non-negative"); + expect(() => + createLlmSemanticChunker({ + maxProviderOutputRetries: 4, + reasoningProviderFactory: factory, + }), + ).toThrow("maxProviderOutputRetries must be at most 3"); + expect(() => + createLlmSemanticChunker({ + maxProviderOutputRetries: -1, + reasoningProviderFactory: factory, + }), + ).toThrow("maxProviderOutputRetries must be a non-negative integer"); for (const [name, options] of [ ["maxEntitiesPerChunk", { maxEntitiesPerChunk: 0 }], ["maxNodes", { maxNodes: 0 }], @@ -2836,6 +2899,29 @@ describe("LLM semantic chunker", () => { ).rejects.toThrow("contiguously without gaps or overlap"); }); + it("retries one invalid provider response before committing the window", async () => { + const provider = new ScriptedProvider([() => ({ chunks: "invalid" }), echoWholeWindow]); + const nodes = await createLlmSemanticChunker({ + maxProviderOutputRetries: 1, + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "retry-provider-output", + metadata: {}, + sectionPath: [], + text: "Alpha. Beta.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }); + + expect(nodes).toHaveLength(1); + expect(provider.calls).toHaveLength(2); + }); + it("accepts JSON wrapped in provider prose but strictly caps joint extraction arrays", async () => { const input = { knowledgeSpaceId: KNOWLEDGE_SPACE_ID, diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts index f210ea05124..4197bfd6d30 100644 --- a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts @@ -62,14 +62,16 @@ const DEFAULT_MAX_ENTITIES_PER_CHUNK = 100; const DEFAULT_MAX_RELATIONS_PER_CHUNK = 100; const DEFAULT_MAX_OUTPUT_TOKENS = 6_000; const DEFAULT_MAX_RESPONSE_CHARS = 1_000_000; -const DEFAULT_PROMPT_VERSION = "semantic-chunking-v5"; +const DEFAULT_PROMPT_VERSION = "semantic-chunking-v6"; const SEMANTIC_CHUNKING_V2_PROMPT_VERSION = "semantic-chunking-v2"; const SEMANTIC_CHUNKING_V3_PROMPT_VERSION = "semantic-chunking-v3"; const SEMANTIC_CHUNKING_V4_PROMPT_VERSION = "semantic-chunking-v4"; +const SEMANTIC_CHUNKING_V5_PROMPT_VERSION = "semantic-chunking-v5"; const V3_MAX_CORE_UNITS_PER_WINDOW = 32; const V3_MAX_LOOK_AHEAD_UNITS_PER_WINDOW = 8; const DEFAULT_MAX_CONCURRENT_WINDOWS = 4; const MAX_CONCURRENT_WINDOWS = 32; +const MAX_PROVIDER_OUTPUT_RETRIES = 3; const SEMANTIC_CHUNKING_STRATEGY = "llm-semantic-v1"; const SEMANTIC_CHUNKING_SCHEMA_VERSION = 1; /** @@ -152,6 +154,7 @@ export interface LlmSemanticChunkerOptions { readonly maxEntitiesPerChunk?: number | undefined; readonly maxNodes?: number | undefined; readonly maxOutputTokens?: number | undefined; + readonly maxProviderOutputRetries?: number | undefined; readonly maxRelationsPerChunk?: number | undefined; readonly maxResponseChars?: number | undefined; readonly maxWindowChars?: number | undefined; @@ -310,7 +313,7 @@ interface SemanticWindowTableSchema { readonly sourceElementId: string; } -type SemanticWindowPlanningVersion = "v1" | "v2" | "v3" | "v4" | "v5"; +type SemanticWindowPlanningVersion = "v1" | "v2" | "v3" | "v4" | "v5" | "v6"; interface SemanticWindowPlanningPolicy { readonly atomicDocument: boolean; @@ -395,6 +398,7 @@ export function createLlmSemanticChunker({ maxEntitiesPerChunk = DEFAULT_MAX_ENTITIES_PER_CHUNK, maxNodes = DEFAULT_MAX_NODES, maxOutputTokens = DEFAULT_MAX_OUTPUT_TOKENS, + maxProviderOutputRetries = 0, maxRelationsPerChunk = DEFAULT_MAX_RELATIONS_PER_CHUNK, maxResponseChars = DEFAULT_MAX_RESPONSE_CHARS, maxWindowChars = DEFAULT_MAX_WINDOW_CHARS, @@ -410,6 +414,7 @@ export function createLlmSemanticChunker({ validatePositiveInteger("maxEntitiesPerChunk", maxEntitiesPerChunk); validatePositiveInteger("maxNodes", maxNodes); validatePositiveInteger("maxOutputTokens", maxOutputTokens); + validateNonnegativeInteger("maxProviderOutputRetries", maxProviderOutputRetries); validatePositiveInteger("maxRelationsPerChunk", maxRelationsPerChunk); validatePositiveInteger("maxResponseChars", maxResponseChars); validatePositiveInteger("maxWindowChars", maxWindowChars); @@ -421,6 +426,11 @@ export function createLlmSemanticChunker({ `LLM semantic chunking maxConcurrentWindows must be at most ${MAX_CONCURRENT_WINDOWS}`, ); } + if (maxProviderOutputRetries > MAX_PROVIDER_OUTPUT_RETRIES) { + throw new Error( + `LLM semantic chunking maxProviderOutputRetries must be at most ${MAX_PROVIDER_OUTPUT_RETRIES}`, + ); + } if (!promptVersion.trim()) { throw new Error("LLM semantic chunking promptVersion is required"); } @@ -479,7 +489,7 @@ export function createLlmSemanticChunker({ let nextUnitIndex = 0; let windowIndex = 0; - const processWindow = async (window: SemanticWindow) => { + const processWindowAttempt = async (window: SemanticWindow, retryCount: number) => { input.signal?.throwIfAborted(); const messages = semanticChunkingMessages({ enableGraph: input.enableGraph !== false, @@ -597,7 +607,7 @@ export function createLlmSemanticChunker({ itemCount: window.units.length, outcome: "succeeded", providerCalls: checkpointHit ? 0 : 1, - retries: 0, + retries: retryCount, stage: "semantic-chunking", ...(checkpointHit ? {} : ingestionModelUsageFromMetadata(resolvedCompletion.metadata)), }); @@ -609,7 +619,7 @@ export function createLlmSemanticChunker({ itemCount: window.units.length, outcome: "failed", providerCalls: checkpointHit ? 0 : 1, - retries: 0, + retries: retryCount, stage: "semantic-chunking", ...(checkpointHit || !completion ? {} @@ -619,6 +629,23 @@ export function createLlmSemanticChunker({ } }; + const processWindow = async (window: SemanticWindow) => { + let retryCount = 0; + while (true) { + try { + return await processWindowAttempt(window, retryCount); + } catch (error) { + if ( + retryCount >= maxProviderOutputRetries || + !isRetryableSemanticProviderOutputError(error) + ) { + throw error; + } + retryCount += 1; + } + } + }; + const appendProcessedWindow = ( processed: Awaited>, ): void => { @@ -1805,7 +1832,7 @@ function semanticRanges( element: MaterializedElement, planningVersion: SemanticWindowPlanningVersion, ): SemanticRange[] { - if (planningVersion === "v5" && element.elementType === "table") { + if ((planningVersion === "v5" || planningVersion === "v6") && element.elementType === "table") { return semanticTableRanges(element); } if (element.elementType !== "paragraph" && element.elementType !== "list") { @@ -2112,7 +2139,8 @@ function resolveSemanticWindowPlanningPolicy({ } function semanticWindowPlanningVersion(promptVersion: string): SemanticWindowPlanningVersion { - if (promptVersion === DEFAULT_PROMPT_VERSION) return "v5"; + if (promptVersion === DEFAULT_PROMPT_VERSION) return "v6"; + if (promptVersion === SEMANTIC_CHUNKING_V5_PROMPT_VERSION) return "v5"; if (promptVersion === SEMANTIC_CHUNKING_V4_PROMPT_VERSION) return "v4"; if (promptVersion === SEMANTIC_CHUNKING_V3_PROMPT_VERSION) return "v3"; if (promptVersion === SEMANTIC_CHUNKING_V2_PROMPT_VERSION) return "v2"; @@ -2120,11 +2148,11 @@ function semanticWindowPlanningVersion(promptVersion: string): SemanticWindowPla } function usesBoundedCoreUnits(version: SemanticWindowPlanningVersion): boolean { - return version === "v3" || version === "v4" || version === "v5"; + return version === "v3" || version === "v4" || version === "v5" || version === "v6"; } function usesFixedCoreBoundary(version: SemanticWindowPlanningVersion): boolean { - return version === "v4" || version === "v5"; + return version === "v4" || version === "v5" || version === "v6"; } function materializeSemanticWindow({ @@ -2152,13 +2180,14 @@ function materializeSemanticWindow({ const coreUnits: AtomicUnit[] = []; let cursor = startUnitIndex; while (cursor < units.length) { + const candidate = units[cursor] as AtomicUnit; if ( usesBoundedCoreUnits(planningPolicy.version) && - coreUnits.length >= V3_MAX_CORE_UNITS_PER_WINDOW + coreUnits.length >= V3_MAX_CORE_UNITS_PER_WINDOW && + !(planningPolicy.version === "v6" && continuesNumberedList(coreUnits.at(-1), candidate)) ) { break; } - const candidate = units[cursor] as AtomicUnit; if (planningPolicy.version === "v1" && !isLegacyWindowCompatible(first, candidate)) { break; } @@ -2231,6 +2260,25 @@ function materializeSemanticWindow({ }; } +function continuesNumberedList(previous: AtomicUnit | undefined, candidate: AtomicUnit): boolean { + if (!previous) return false; + if (previous.sourceElement.elementId === candidate.sourceElement.elementId) return true; + const previousOrdinal = numberedListOrdinal(previous.sourceElement.text); + const candidateOrdinal = numberedListOrdinal(candidate.sourceElement.text); + return ( + previousOrdinal !== undefined && + candidateOrdinal !== undefined && + candidateOrdinal === previousOrdinal + 1 + ); +} + +function numberedListOrdinal(text: string): number | undefined { + const match = /^\s*(\d{1,4})\s*[.)、．]\s*/u.exec(text); + if (!match?.[1]) return undefined; + const ordinal = Number.parseInt(match[1], 10); + return Number.isSafeInteger(ordinal) ? ordinal : undefined; +} + function semanticPromptUnit( unit: AtomicUnit, planningVersion: SemanticWindowPlanningVersion, @@ -2261,7 +2309,7 @@ function semanticPromptUnit( sourceSectionPath: [...unit.sectionPath], } : {}), - ...(planningVersion === "v5" && unit.tableRecord + ...((planningVersion === "v5" || planningVersion === "v6") && unit.tableRecord ? { tableMode: unit.tableRecord.mode, tableRecordCount: unit.tableRecord.count, @@ -2350,6 +2398,7 @@ function semanticChunkingMessages({ "Never emit a chunk that starts wholly in lookAheadUnits. Units not consumed from look-ahead will be reconsidered in the next request.", ]), "Ranges may be smaller than the maximum; prefer natural topic boundaries over filling chunks.", + "When a contiguous numbered or bulleted list fits within the chunk limit, keep the complete list together under its heading.", `Every range must contain at most ${maxChunkChars} Unicode graphemes including separators.`, `Return at most ${maxEntitiesPerChunk} entities and ${maxRelationsPerChunk} relations per chunk.`, "Allowed entity types: date, metric, organization, person, policy, product, term.", @@ -2365,7 +2414,7 @@ function semanticChunkingMessages({ : [ "A unit marked boundaryPolicy=isolated must occupy a chunk containing only units from that same table or image element.", ]), - ...(window.planningVersion === "v5" + ...(window.planningVersion === "v5" || window.planningVersion === "v6" ? [ "Table records carry tableMode/tableRecordIndex/tableRecordCount; resolve their bounded columns from tableSchemas by sourceElementId.", "For tableMode=record-list or matrix, return exactly one chunk per table record; hard-split units with the same tableRecordIndex may stay together, but never combine different records.", @@ -2404,7 +2453,7 @@ function semanticChunkingMessages({ semanticPromptUnit(unit, window.planningVersion), ), sectionPath: window.sectionPath, - ...(window.planningVersion === "v5" + ...(window.planningVersion === "v5" || window.planningVersion === "v6" ? { tableSchemas: semanticWindowTableSchemas([...window.units, ...window.lookAheadUnits]), } @@ -2620,6 +2669,14 @@ function parseSemanticChunkingOutput(text: string): LlmSemanticChunkingOutput { } } +function isRetryableSemanticProviderOutputError(error: unknown): boolean { + return ( + error instanceof Error && + (error.message === "LLM semantic chunking provider returned invalid JSON" || + error.message === "LLM semantic chunking provider returned an invalid response schema") + ); +} + function normalizeTableRecordBoundaries({ maxChunkChars, output, @@ -2629,7 +2686,12 @@ function normalizeTableRecordBoundaries({ readonly output: LlmSemanticChunkingOutput; readonly window: SemanticWindow; }): LlmSemanticChunkingOutput { - if (window.planningVersion !== "v5" || window.atomicDocument) return output; + if ( + (window.planningVersion !== "v5" && window.planningVersion !== "v6") || + window.atomicDocument + ) { + return output; + } const eligibleUnits = [...window.units, ...window.lookAheadUnits]; const unitIndex = new Map(eligibleUnits.map((unit, index) => [unit.id, index])); diff --git a/knowledge-fs/packages/api/src/text-indexing-strategy.ts b/knowledge-fs/packages/api/src/text-indexing-strategy.ts new file mode 100644 index 00000000000..e71f089b23b --- /dev/null +++ b/knowledge-fs/packages/api/src/text-indexing-strategy.ts @@ -0,0 +1,5 @@ +/** + * Bump when indexed text composition or normalization changes. The strategy participates in + * publication fingerprints and prevents reuse of projections built with an older composition. + */ +export const TEXT_INDEXING_STRATEGY = "section-context-v1"; diff --git a/knowledge-fs/packages/parsers/src/index.ts b/knowledge-fs/packages/parsers/src/index.ts index 9f899ac1666..5ef1f8e1763 100644 --- a/knowledge-fs/packages/parsers/src/index.ts +++ b/knowledge-fs/packages/parsers/src/index.ts @@ -135,6 +135,7 @@ export interface NativeParserOptions { export interface UnstructuredParserClientOptions extends NativeParserOptions { readonly apiKey?: string; + readonly defaultLanguage?: string; readonly endpoint: string; readonly fetch?: typeof fetch; readonly maxResponseBytes?: number; @@ -303,6 +304,7 @@ export function createNativeStructuredDataParser( export function createUnstructuredParserClient({ apiKey, + defaultLanguage, endpoint, fetch: fetchImpl = fetch, maxConcurrency = defaultMaxConcurrency, @@ -323,9 +325,12 @@ export function createUnstructuredParserClient({ requestGate.run(async () => { const deadline = createUnstructuredRequestDeadline(input.signal, requestTimeoutMs); try { - const parserVersion = options.parserVersion ?? "unstructured@9"; + const parserVersion = options.parserVersion ?? "unstructured@10"; const partitionStrategy = unstructuredPartitionStrategy(input); const providerImageBlockTypes = unstructuredProviderImageBlockTypes(input); + const providerLanguage = unstructuredLanguage( + input.parserHints?.language ?? defaultLanguage, + ); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); const response = await fetchWithRetries({ buildRequest: () => { @@ -337,6 +342,7 @@ export function createUnstructuredParserClient({ form.set("files", new File([fileBody], input.filename, { type: input.mimeType })); form.set("coordinates", "true"); form.set("strategy", partitionStrategy); + if (providerLanguage) form.set("languages", providerLanguage); if (providerImageBlockTypes.length > 0) { for (const blockType of providerImageBlockTypes) { form.append("extract_image_block_types", blockType); @@ -388,6 +394,7 @@ export function createUnstructuredParserClient({ artifactHashContext: unstructuredArtifactHashContext(input, { partitionStrategy, providerImageBlockTypes, + providerLanguage, }), elements, input, @@ -436,6 +443,28 @@ function shouldRequestProviderImages(input: ParseDocumentInput): boolean { return unstructuredProviderImageBlockTypes(input).length > 0; } +function unstructuredLanguage(language: string | undefined): string | undefined { + const normalized = language?.trim().toLowerCase(); + if (!normalized) return undefined; + + const baseLanguage = normalized.split("-", 1)[0] ?? normalized; + return ( + { + ar: "ara", + de: "deu", + en: "eng", + es: "spa", + fr: "fra", + hi: "hin", + ja: "jpn", + ko: "kor", + pt: "por", + ru: "rus", + zh: "zho", + }[baseLanguage] ?? normalized + ); +} + function unstructuredProviderImageBlockTypes( input: ParseDocumentInput, ): readonly ("Image" | "Table")[] { @@ -472,6 +501,7 @@ function unstructuredArtifactHashContext( request: { readonly partitionStrategy: "auto" | "fast" | "hi_res"; readonly providerImageBlockTypes: readonly ("Image" | "Table")[]; + readonly providerLanguage?: string | undefined; }, ): string { const hints = input.parserHints; @@ -491,6 +521,7 @@ function unstructuredArtifactHashContext( coordinates: true, imageBlockTypes: request.providerImageBlockTypes, imagePayload: request.providerImageBlockTypes.length > 0, + language: request.providerLanguage ?? null, strategy: request.partitionStrategy, }, }); @@ -2307,7 +2338,10 @@ function unstructuredElementsToElements( const type = unstructuredType(sourceElement.type); const tableProjection = type === "table" ? unstructuredTableProjection(sourceElement.metadata) : undefined; - const text = tableProjection?.text ?? normalizeText(sourceElement.text ?? ""); + const providerText = tableProjection?.text ?? normalizeText(sourceElement.text ?? ""); + const text = hasChineseOcrLanguage(sourceElement.metadata) + ? normalizeChineseOcrText(providerText) + : providerText; if (!text && !hasUnstructuredVisualMetadata(sourceElement.metadata, type)) { continue; @@ -2349,6 +2383,26 @@ function unstructuredElementsToElements( return elements; } +function hasChineseOcrLanguage(metadata: Readonly>): boolean { + const languages = metadata.languages; + return ( + Array.isArray(languages) && + languages.some( + (language) => + typeof language === "string" && + (language.trim().toLowerCase() === "zho" || + language.trim().toLowerCase().startsWith("zh-")), + ) + ); +} + +function normalizeChineseOcrText(text: string): string { + return text.replace( + /(?<=[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff])[\t \u3000]+(?=[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff])/gu, + "", + ); +} + function unstructuredCategoryDepth( metadata: Readonly>, ): number | undefined { diff --git a/knowledge-fs/packages/parsers/src/parser.test.ts b/knowledge-fs/packages/parsers/src/parser.test.ts index 8d9affc0577..3ce5be73899 100644 --- a/knowledge-fs/packages/parsers/src/parser.test.ts +++ b/knowledge-fs/packages/parsers/src/parser.test.ts @@ -1342,7 +1342,7 @@ describe("parser adapters", () => { metadata: { filename: "report.pdf", mimeType: "application/pdf", - parserVersion: "unstructured@9", + parserVersion: "unstructured@10", }, parser: "unstructured", version: 1, @@ -1410,6 +1410,54 @@ describe("parser adapters", () => { ]); }); + it("normalizes spacing between Chinese OCR characters without rewriting words", async () => { + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch: async () => + new Response( + JSON.stringify([ + { + metadata: { languages: ["zho"], page_number: 1 }, + text: "随 道 工 程 。源 道 初 期 支 护 与 开 挖 作 业", + type: "NarrativeText", + }, + { + metadata: { + languages: ["zho"], + page_number: 1, + text_as_html: "
施 工 条 件临 时 用 电
", + }, + text: "施 工 条 件 临 时 用 电", + type: "Table", + }, + { + metadata: { languages: ["zho"], page_number: 1 }, + text: "随着道路工程推进", + type: "NarrativeText", + }, + ]), + { headers: { "content-type": "application/json" }, status: 200 }, + ), + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c51", + now: () => createdAt, + }); + + const artifact = await parser.parse({ + body: new Uint8Array([1, 2, 3]), + documentAssetId, + filename: "tunnel.pdf", + mimeType: "application/pdf", + version: 1, + }); + + expect(artifact.elements.map((element) => element.text)).toEqual([ + "随道工程 。源道初期支护与开挖作业", + "column_1: 施工条件 | column_2: 临时用电", + "随着道路工程推进", + ]); + expect(artifact.metadata.parserVersion).toBe("unstructured@10"); + }); + it("projects an Unstructured spreadsheet table into independently retrievable records", async () => { const parser = createUnstructuredParserClient({ endpoint: "https://unstructured.example.test", @@ -1539,7 +1587,7 @@ describe("parser adapters", () => { version: 1, }), ).resolves.toMatchObject({ - metadata: { parserVersion: "unstructured@9" }, + metadata: { parserVersion: "unstructured@10" }, parser: "unstructured", }); }, @@ -1632,6 +1680,7 @@ describe("parser adapters", () => { }); const parseWithHints = (parserHints: { readonly imagesHandledExternally?: boolean; + readonly language?: string; readonly layoutComplexity?: "complex" | "simple"; readonly requiresImages?: boolean; readonly requiresOcr?: boolean; @@ -1646,12 +1695,13 @@ describe("parser adapters", () => { version: 1, }); - const [fast, ocr, tables, providerImages, externalImages] = await Promise.all([ + const [fast, ocr, tables, providerImages, externalImages, chinese] = await Promise.all([ parseWithHints({ layoutComplexity: "simple" }), parseWithHints({ requiresOcr: true }), parseWithHints({ requiresTables: true }), parseWithHints({ requiresImages: true }), parseWithHints({ imagesHandledExternally: true, requiresImages: true }), + parseWithHints({ language: "zh-CN" }), ]); expect( @@ -1661,8 +1711,9 @@ describe("parser adapters", () => { tables.artifactHash, providerImages.artifactHash, externalImages.artifactHash, + chinese.artifactHash, ]).size, - ).toBe(5); + ).toBe(6); }); it.each([