mirror of
https://github.com/langgenius/dify.git
synced 2026-09-05 00:31:19 +08:00
fix(knowledge-fs): improve Chinese PDF ingestion and retrieval
This commit is contained in:
Stephen Zhou
parent
84c53bdfde
commit
138e810f9c
@ -459,6 +459,8 @@ const documentSemanticChunker = createLlmSemanticChunker({
|
||||
: {}),
|
||||
maxConcurrentWindows: ingestionModelRuntimeOptions.semanticExtractionMaxConcurrency,
|
||||
maxNodes: 20_000,
|
||||
maxProviderOutputRetries: 1,
|
||||
maxWindowChars: ingestionModelRuntimeOptions.semanticChunkingMaxWindowChars,
|
||||
metrics: operationalMetrics.ingestionModelCalls,
|
||||
modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate,
|
||||
reasoningProviderFactory: profileReasoningCapability.providerFactory,
|
||||
|
||||
@ -14,6 +14,7 @@ describe("createApiIngestionModelRuntimeOptions", () => {
|
||||
expect(options.outlineSummaryMaxConcurrency).toBe(8);
|
||||
expect(options.semanticExtractionBatchSize).toBe(8);
|
||||
expect(options.semanticExtractionMaxConcurrency).toBe(4);
|
||||
expect(options.semanticChunkingMaxWindowChars).toBe(4_800);
|
||||
});
|
||||
|
||||
it("accepts bounded concurrency overrides", () => {
|
||||
@ -26,6 +27,7 @@ describe("createApiIngestionModelRuntimeOptions", () => {
|
||||
KNOWLEDGE_OUTLINE_SUMMARY_MAX_CONCURRENCY: "12",
|
||||
KNOWLEDGE_SEMANTIC_EXTRACTION_BATCH_SIZE: "10",
|
||||
KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY: "6",
|
||||
KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS: "2400",
|
||||
});
|
||||
|
||||
expect(options.globalConcurrency).toBe(24);
|
||||
@ -36,6 +38,7 @@ describe("createApiIngestionModelRuntimeOptions", () => {
|
||||
expect(options.outlineSummaryMaxConcurrency).toBe(12);
|
||||
expect(options.semanticExtractionBatchSize).toBe(10);
|
||||
expect(options.semanticExtractionMaxConcurrency).toBe(6);
|
||||
expect(options.semanticChunkingMaxWindowChars).toBe(2_400);
|
||||
});
|
||||
|
||||
it("creates an isolated hard budget for each document", () => {
|
||||
@ -84,6 +87,7 @@ describe("createApiIngestionModelRuntimeOptions", () => {
|
||||
["KNOWLEDGE_OUTLINE_SUMMARY_BATCH_MAX_INPUT_CHARS", "200001"],
|
||||
["KNOWLEDGE_SEMANTIC_EXTRACTION_BATCH_SIZE", "33"],
|
||||
["KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY", "0"],
|
||||
["KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS", "200001"],
|
||||
] as const)("rejects invalid %s=%s", (name, value) => {
|
||||
expect(() => createApiIngestionModelRuntimeOptions({ [name]: value })).toThrow(name);
|
||||
});
|
||||
|
||||
@ -15,6 +15,7 @@ export interface ApiIngestionModelRuntimeEnv {
|
||||
readonly KNOWLEDGE_OUTLINE_SUMMARY_MAX_CONCURRENCY?: string | undefined;
|
||||
readonly KNOWLEDGE_SEMANTIC_EXTRACTION_BATCH_SIZE?: string | undefined;
|
||||
readonly KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY?: string | undefined;
|
||||
readonly KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS?: string | undefined;
|
||||
}
|
||||
|
||||
export interface ApiIngestionModelRuntimeOptions {
|
||||
@ -28,6 +29,7 @@ export interface ApiIngestionModelRuntimeOptions {
|
||||
readonly outlineSummaryMaxConcurrency: number;
|
||||
readonly semanticExtractionBatchSize: number;
|
||||
readonly semanticExtractionMaxConcurrency: number;
|
||||
readonly semanticChunkingMaxWindowChars: number;
|
||||
}
|
||||
|
||||
export interface ApiIngestionModelRuntimeMetrics {
|
||||
@ -48,8 +50,10 @@ const MAX_OUTLINE_SUMMARY_BATCH_SIZE = 32;
|
||||
const MAX_OUTLINE_SUMMARY_CONCURRENCY = 32;
|
||||
const DEFAULT_SEMANTIC_EXTRACTION_BATCH_SIZE = 8;
|
||||
const DEFAULT_SEMANTIC_EXTRACTION_MAX_CONCURRENCY = 4;
|
||||
const DEFAULT_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS = 4_800;
|
||||
const MAX_SEMANTIC_EXTRACTION_BATCH_SIZE = 32;
|
||||
const MAX_SEMANTIC_EXTRACTION_CONCURRENCY = 32;
|
||||
const MAX_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS = 200_000;
|
||||
|
||||
/**
|
||||
* Builds the shared ingestion-time model budget. The per-document outline bound protects fairness
|
||||
@ -108,6 +112,12 @@ export function createApiIngestionModelRuntimeOptions(
|
||||
name: "KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY",
|
||||
value: env.KNOWLEDGE_SEMANTIC_EXTRACTION_MAX_CONCURRENCY,
|
||||
});
|
||||
const semanticChunkingMaxWindowChars = boundedPositiveIntegerEnv({
|
||||
fallback: DEFAULT_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS,
|
||||
max: MAX_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS,
|
||||
name: "KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS",
|
||||
value: env.KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS,
|
||||
});
|
||||
|
||||
return {
|
||||
createDocumentModelBudget: () =>
|
||||
@ -126,6 +136,7 @@ export function createApiIngestionModelRuntimeOptions(
|
||||
outlineSummaryMaxConcurrency,
|
||||
semanticExtractionBatchSize,
|
||||
semanticExtractionMaxConcurrency,
|
||||
semanticChunkingMaxWindowChars,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -79,6 +79,31 @@ describe("createApiDocumentParser", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards the configured default language to Unstructured", async () => {
|
||||
let requestedLanguage: FormDataEntryValue | null = null;
|
||||
const parser = createApiDocumentParser({
|
||||
env: {
|
||||
UNSTRUCTURED_API_URL: "https://unstructured.example.test",
|
||||
UNSTRUCTURED_DEFAULT_LANGUAGE: "zh-CN",
|
||||
},
|
||||
fetch: async (input) => {
|
||||
const request = input instanceof Request ? input : new Request(input);
|
||||
requestedLanguage = (await request.formData()).get("languages");
|
||||
return new Response("[]", { headers: { "content-type": "application/json" } });
|
||||
},
|
||||
});
|
||||
|
||||
await parser.parse({
|
||||
body: encoder.encode("%PDF-1.7"),
|
||||
documentAssetId: "00000000-0000-4000-8000-000000000007",
|
||||
filename: "report.pdf",
|
||||
mimeType: "application/pdf",
|
||||
version: 1,
|
||||
});
|
||||
|
||||
expect(requestedLanguage).toBe("zho");
|
||||
});
|
||||
|
||||
it("can derive the local Unstructured URL from UNSTRUCTURED_PORT outside production", async () => {
|
||||
let requestedUrl = "";
|
||||
const parser = createApiDocumentParser({
|
||||
|
||||
@ -11,6 +11,7 @@ export interface ApiParserEnv {
|
||||
readonly NODE_ENV?: string | undefined;
|
||||
readonly UNSTRUCTURED_API_KEY?: string | undefined;
|
||||
readonly UNSTRUCTURED_API_URL?: string | undefined;
|
||||
readonly UNSTRUCTURED_DEFAULT_LANGUAGE?: string | undefined;
|
||||
readonly UNSTRUCTURED_MAX_CONCURRENCY?: string | undefined;
|
||||
readonly UNSTRUCTURED_MAX_RESPONSE_BYTES?: string | undefined;
|
||||
readonly UNSTRUCTURED_MAX_RETRIES?: string | undefined;
|
||||
@ -60,6 +61,9 @@ function createApiUnstructuredParser({
|
||||
}
|
||||
|
||||
return createUnstructuredParserClient({
|
||||
...(env.UNSTRUCTURED_DEFAULT_LANGUAGE?.trim()
|
||||
? { defaultLanguage: env.UNSTRUCTURED_DEFAULT_LANGUAGE.trim() }
|
||||
: {}),
|
||||
endpoint,
|
||||
...(env.UNSTRUCTURED_API_KEY?.trim() ? { apiKey: env.UNSTRUCTURED_API_KEY.trim() } : {}),
|
||||
...(fetchImpl ? { fetch: fetchImpl } : {}),
|
||||
|
||||
@ -14,6 +14,8 @@ UNSTRUCTURED_PORT=8000
|
||||
UNSTRUCTURED_API_URL=http://127.0.0.1:8000
|
||||
UNSTRUCTURED_API_KEY=
|
||||
UNSTRUCTURED_MAX_CONCURRENCY=2
|
||||
UNSTRUCTURED_DEFAULT_LANGUAGE=
|
||||
KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS=4800
|
||||
UNSTRUCTURED_REQUEST_TIMEOUT_MS=120000
|
||||
UNSTRUCTURED_MAX_RESPONSE_BYTES=33554432
|
||||
UNSTRUCTURED_MAX_RETRIES=
|
||||
|
||||
@ -61,6 +61,8 @@ services:
|
||||
KNOWLEDGE_DEV_TENANT_ID: ${KNOWLEDGE_DEV_TENANT_ID:-tenant-dev}
|
||||
UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL:-http://unstructured:8000}
|
||||
UNSTRUCTURED_MAX_CONCURRENCY: ${UNSTRUCTURED_MAX_CONCURRENCY:-2}
|
||||
UNSTRUCTURED_DEFAULT_LANGUAGE: ${UNSTRUCTURED_DEFAULT_LANGUAGE:-}
|
||||
KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS: ${KNOWLEDGE_SEMANTIC_CHUNKING_MAX_WINDOW_CHARS:-4800}
|
||||
UNSTRUCTURED_REQUEST_TIMEOUT_MS: ${UNSTRUCTURED_REQUEST_TIMEOUT_MS:-120000}
|
||||
UNSTRUCTURED_MAX_RESPONSE_BYTES: ${UNSTRUCTURED_MAX_RESPONSE_BYTES:-33554432}
|
||||
UNSTRUCTURED_MAX_RETRIES: ${UNSTRUCTURED_MAX_RETRIES:-0}
|
||||
|
||||
@ -118,8 +118,18 @@ describe("document compilation candidate runtime factories", () => {
|
||||
);
|
||||
expect(resolved.material.projections).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ model: vectorSpaceId, type: "dense-vector" }),
|
||||
expect.objectContaining({ model: "database-fts@1", type: "fts" }),
|
||||
expect.objectContaining({
|
||||
indexVersion: "dify-model-runtime-embedding-v1",
|
||||
model: vectorSpaceId,
|
||||
strategy: "text-dense-v1:section-context-v1",
|
||||
type: "dense-vector",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
indexVersion: "database-fts-v1",
|
||||
model: "database-fts@1",
|
||||
strategy: "mixed-cjk-latin-fts-v1:section-context-v1",
|
||||
type: "fts",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
@ -42,6 +42,7 @@ import type {
|
||||
ProjectionSetPublicationMemberRepository,
|
||||
} from "./projection-publication-member-repository";
|
||||
import type { ProjectionSetPublicationRepository } from "./projection-publication-repository";
|
||||
import { TEXT_INDEXING_STRATEGY } from "./text-indexing-strategy";
|
||||
|
||||
export interface DocumentCompilationFingerprintVersions {
|
||||
readonly chunkerVersion: string;
|
||||
@ -655,11 +656,11 @@ function projectionFingerprintConfig(
|
||||
const visual = isVisualProjection(projection);
|
||||
const strategy =
|
||||
projection.type === "fts"
|
||||
? "mixed-cjk-latin-fts-v1"
|
||||
? `mixed-cjk-latin-fts-v1:${TEXT_INDEXING_STRATEGY}`
|
||||
: visual
|
||||
? "visual-dense-v1"
|
||||
: projection.type === "dense-vector"
|
||||
? "text-dense-v1"
|
||||
? `text-dense-v1:${TEXT_INDEXING_STRATEGY}`
|
||||
: `${projection.type}-v1`;
|
||||
return {
|
||||
indexVersion:
|
||||
|
||||
@ -8842,7 +8842,7 @@ describe("createKnowledgeGateway", () => {
|
||||
expect(embedding.calls[0]).toEqual({
|
||||
inputType: "search_document",
|
||||
model: "static-upgrade@2026-05-01",
|
||||
texts: ["First chunk", "Second chunk"],
|
||||
texts: ["Intro\n\nFirst chunk", "Intro\n\nSecond chunk"],
|
||||
});
|
||||
expect(evaluationCalls[0]).toEqual({
|
||||
denseProjectionModel: "static-upgrade@2026-05-01",
|
||||
@ -9132,7 +9132,7 @@ describe("createKnowledgeGateway", () => {
|
||||
{
|
||||
inputType: "search_document",
|
||||
model: "static-dense",
|
||||
texts: ["First chunk", "Second chunk"],
|
||||
texts: ["Intro\n\nFirst chunk", "Intro\n\nSecond chunk"],
|
||||
},
|
||||
]);
|
||||
expect(projections).toEqual([
|
||||
@ -9141,7 +9141,7 @@ describe("createKnowledgeGateway", () => {
|
||||
knowledgeSpaceId: firstNode.knowledgeSpaceId,
|
||||
metadata: expect.objectContaining({
|
||||
artifactHash: firstNode.artifactHash,
|
||||
denseVector: [0.1, 11],
|
||||
denseVector: [0.1, 18],
|
||||
dimension: 2,
|
||||
embeddingProvider: "static",
|
||||
modelVersion: "static-dense",
|
||||
@ -9153,7 +9153,7 @@ describe("createKnowledgeGateway", () => {
|
||||
type: "dense-vector",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({ denseVector: [1.1, 12] }),
|
||||
metadata: expect.objectContaining({ denseVector: [1.1, 19] }),
|
||||
nodeId: secondNode.id,
|
||||
}),
|
||||
]);
|
||||
@ -9187,7 +9187,7 @@ describe("createKnowledgeGateway", () => {
|
||||
type: "dense-vector",
|
||||
})
|
||||
).items[0]?.metadata,
|
||||
).toEqual(expect.objectContaining({ denseVector: [0.1, 11] }));
|
||||
).toEqual(expect.objectContaining({ denseVector: [0.1, 18] }));
|
||||
await expect(
|
||||
memoryRepository.listReadyBySpace({
|
||||
knowledgeSpaceId: firstNode.knowledgeSpaceId,
|
||||
@ -9394,7 +9394,7 @@ describe("createKnowledgeGateway", () => {
|
||||
);
|
||||
expect(fake.calls[0]?.sql).toContain("dense_vector");
|
||||
expect(fake.calls[0]?.sql).not.toContain("First chunk");
|
||||
expect(fake.calls[0]?.params).toContain("[0.1,11]");
|
||||
expect(fake.calls[0]?.params).toContain("[0.1,18]");
|
||||
expect(fake.calls[0]?.params).toContain(JSON.stringify(firstProjection.metadata));
|
||||
await expect(
|
||||
databaseRepository.listReadyBySpace({
|
||||
@ -9635,7 +9635,8 @@ describe("createKnowledgeGateway", () => {
|
||||
metadata: expect.objectContaining({
|
||||
artifactHash: firstNode.artifactHash,
|
||||
ftsLanguageStrategy: "mixed-cjk-latin-v1",
|
||||
ftsText: "contract abc 123 renewal terms",
|
||||
ftsText: "intro contract abc 123 renewal terms",
|
||||
indexingStrategy: "section-context-v1",
|
||||
parser: "database-fts",
|
||||
}),
|
||||
model: "database-fts@1",
|
||||
@ -9646,7 +9647,7 @@ describe("createKnowledgeGateway", () => {
|
||||
}),
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
ftsText: "error code e 42 remediation",
|
||||
ftsText: "intro error code e 42 remediation",
|
||||
}),
|
||||
nodeId: secondNode.id,
|
||||
}),
|
||||
@ -9680,7 +9681,7 @@ describe("createKnowledgeGateway", () => {
|
||||
);
|
||||
expect(fake.calls[0]?.sql).toContain("to_tsvector('simple'");
|
||||
expect(fake.calls[0]?.sql).not.toContain("Contract ABC-123");
|
||||
expect(fake.calls[0]?.params).toContain("contract abc 123 renewal terms");
|
||||
expect(fake.calls[0]?.params).toContain("intro contract abc 123 renewal terms");
|
||||
expect(fake.calls[0]?.params).toContain(JSON.stringify(firstProjection.metadata));
|
||||
const tidbFake = createFakeIndexProjectionExecutor();
|
||||
const tidbRepository = createDatabaseIndexProjectionRepository({
|
||||
@ -9695,7 +9696,7 @@ describe("createKnowledgeGateway", () => {
|
||||
await expect(tidbRepository.createMany([firstProjection])).resolves.toEqual([firstProjection]);
|
||||
expect(tidbFake.calls[0]?.sql).toContain("INSERT INTO `index_projections`");
|
||||
expect(tidbFake.calls[0]?.sql).not.toContain("to_tsvector");
|
||||
expect(tidbFake.calls[0]?.params).toContain("contract abc 123 renewal terms");
|
||||
expect(tidbFake.calls[0]?.params).toContain("intro contract abc 123 renewal terms");
|
||||
});
|
||||
|
||||
it("idempotently reindexes parse artifacts so interrupted projection builds can be repaired", async () => {
|
||||
@ -9980,7 +9981,8 @@ describe("createKnowledgeGateway", () => {
|
||||
expect(projections[0]?.metadata).toEqual(
|
||||
expect.objectContaining({
|
||||
ftsLanguageStrategy: "mixed-cjk-latin-v1",
|
||||
ftsText: "合 同 abc 123 续 约 terms",
|
||||
ftsText: "intro 合 同 abc 123 续 约 terms",
|
||||
indexingStrategy: "section-context-v1",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@ -235,7 +235,11 @@ describe("index projection builders", () => {
|
||||
});
|
||||
|
||||
expect(embedCalls).toEqual([
|
||||
{ inputType: "search_document", model: "model-a", texts: ["合同ABC-123续约 terms"] },
|
||||
{
|
||||
inputType: "search_document",
|
||||
model: "model-a",
|
||||
texts: ["Intro\n\n合同ABC-123续约 terms"],
|
||||
},
|
||||
]);
|
||||
expect(result[0]).toMatchObject({
|
||||
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f9000",
|
||||
@ -298,7 +302,7 @@ describe("index projection builders", () => {
|
||||
inputType: "search_document",
|
||||
model: "tenant-model",
|
||||
tenantId: "tenant-1",
|
||||
texts: ["合同ABC-123续约 terms"],
|
||||
texts: ["Intro\n\n合同ABC-123续约 terms"],
|
||||
},
|
||||
]);
|
||||
expect(projection).toMatchObject({
|
||||
@ -527,7 +531,8 @@ describe("index projection builders", () => {
|
||||
expect(projection).toMatchObject({
|
||||
metadata: {
|
||||
ftsLanguageStrategy: "mixed-cjk-latin-v1",
|
||||
ftsText: "合 同 abc 123 续 约 terms",
|
||||
ftsText: "intro 合 同 abc 123 续 约 terms",
|
||||
indexingStrategy: "section-context-v1",
|
||||
parser: "database-fts",
|
||||
},
|
||||
model: "database-fts@1",
|
||||
@ -536,6 +541,59 @@ describe("index projection builders", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("indexes semantic section paths and summaries while preserving the node source text", async () => {
|
||||
const embedCalls: EmbedTextsInput[] = [];
|
||||
const embeddings: EmbeddingProvider = {
|
||||
embed: async (input) => {
|
||||
embedCalls.push(input);
|
||||
return {
|
||||
dense: [[0.1, 0.2]],
|
||||
metadata: { model: "model-a@1", provider: "static" },
|
||||
model: "model-a@1",
|
||||
};
|
||||
},
|
||||
kind: "static",
|
||||
models: async () => [],
|
||||
};
|
||||
const denseRepository = createRecordingProjectionRepository();
|
||||
const ftsRepository = createRecordingProjectionRepository();
|
||||
const node = knowledgeNode({
|
||||
metadata: {
|
||||
chunkIndex: 0,
|
||||
semanticChunking: {
|
||||
section: {
|
||||
path: ["Safety", "Tunnel"],
|
||||
summary: "Checks excavation support and edge protection.",
|
||||
},
|
||||
},
|
||||
},
|
||||
sourceLocation: {
|
||||
endOffset: 12,
|
||||
sectionPath: ["Safety", "Tunnel", "Initial support"],
|
||||
startOffset: 0,
|
||||
},
|
||||
text: "Verify the current work face.",
|
||||
});
|
||||
|
||||
await createDenseVectorProjectionBuilder({
|
||||
embeddings,
|
||||
maxBatchSize: 1,
|
||||
projections: denseRepository.repository,
|
||||
}).build({ model: "model-a", nodes: [node], projectionVersion: 1 });
|
||||
const [fts] = await createFtsProjectionBuilder({
|
||||
maxBatchSize: 1,
|
||||
projections: ftsRepository.repository,
|
||||
}).build({ nodes: [node], projectionVersion: 1 });
|
||||
|
||||
expect(embedCalls[0]?.texts).toEqual([
|
||||
"Safety > Tunnel > Initial support\n\nChecks excavation support and edge protection.\n\nVerify the current work face.",
|
||||
]);
|
||||
expect(fts?.metadata.ftsText).toBe(
|
||||
"safety tunnel initial support checks excavation support and edge protection verify the current work face",
|
||||
);
|
||||
expect(node.text).toBe("Verify the current work face.");
|
||||
});
|
||||
|
||||
it("skips FTS projections for nodes without searchable text", async () => {
|
||||
const { created, repository } = createRecordingProjectionRepository();
|
||||
const builder = createFtsProjectionBuilder({
|
||||
|
||||
@ -29,6 +29,7 @@ import {
|
||||
assertObservedEmbeddingDimension,
|
||||
} from "./knowledge-space-embedding-resolver";
|
||||
import { normalizeMixedLanguageFtsText } from "./retrieval-text-utils";
|
||||
import { TEXT_INDEXING_STRATEGY } from "./text-indexing-strategy";
|
||||
|
||||
export interface BuildDenseVectorProjectionInput {
|
||||
/** Immutable profile captured by a compilation/profile-migration attempt. */
|
||||
@ -258,6 +259,7 @@ export function createDenseVectorProjectionBuilder({
|
||||
})
|
||||
: new Map<string, IndexProjection>();
|
||||
const nodesToEmbed = parsedNodes.filter((node) => !reusableByNodeId.has(node.id));
|
||||
const embeddingTexts = nodesToEmbed.map(textIndexContentForNode);
|
||||
if (nodesToEmbed.length === 0) {
|
||||
recordIngestionModelCallMetric(metrics, {
|
||||
cacheHits: parsedNodes.length,
|
||||
@ -273,8 +275,8 @@ export function createDenseVectorProjectionBuilder({
|
||||
);
|
||||
}
|
||||
modelBudget?.reserve({
|
||||
estimatedTokens: nodesToEmbed.reduce(
|
||||
(total, node) => total + estimateDocumentModelTokens(node.text),
|
||||
estimatedTokens: embeddingTexts.reduce(
|
||||
(total, text) => total + estimateDocumentModelTokens(text),
|
||||
0,
|
||||
),
|
||||
itemCount: nodesToEmbed.length,
|
||||
@ -287,7 +289,7 @@ export function createDenseVectorProjectionBuilder({
|
||||
inputType: "search_document",
|
||||
model: resolvedEmbedding?.model ?? model,
|
||||
...(signal ? { signal } : {}),
|
||||
texts: nodesToEmbed.map((node) => node.text),
|
||||
texts: embeddingTexts,
|
||||
...(tenantId ? { tenantId } : {}),
|
||||
});
|
||||
} catch (error) {
|
||||
@ -373,6 +375,7 @@ export function createDenseVectorProjectionBuilder({
|
||||
dimension: responseDimension,
|
||||
documentAssetId: node.documentAssetId,
|
||||
embeddingProvider: result.metadata.provider,
|
||||
indexingStrategy: TEXT_INDEXING_STRATEGY,
|
||||
embeddingModel: result.model,
|
||||
...(resolvedEmbedding
|
||||
? {
|
||||
@ -457,7 +460,8 @@ async function loadReusableDenseProjections({
|
||||
projection.type !== "dense-vector" ||
|
||||
projection.metadata.artifactHash !== node.artifactHash ||
|
||||
projection.metadata.documentAssetId !== node.documentAssetId ||
|
||||
projection.metadata.parseArtifactId !== node.parseArtifactId
|
||||
projection.metadata.parseArtifactId !== node.parseArtifactId ||
|
||||
projection.metadata.indexingStrategy !== TEXT_INDEXING_STRATEGY
|
||||
) {
|
||||
throw new Error(
|
||||
`Persisted generation-scoped dense projection id=${projection.id} cannot be reused`,
|
||||
@ -498,10 +502,11 @@ export function createFtsProjectionBuilder({
|
||||
|
||||
const parsedNodes = nodes.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node)));
|
||||
const ftsProjections = parsedNodes.flatMap((node) => {
|
||||
const ftsText = normalizeMixedLanguageFtsText(node.text);
|
||||
if (!ftsText) {
|
||||
const sourceFtsText = normalizeMixedLanguageFtsText(node.text);
|
||||
if (!sourceFtsText) {
|
||||
return [];
|
||||
}
|
||||
const ftsText = normalizeMixedLanguageFtsText(textIndexContentForNode(node));
|
||||
|
||||
return [
|
||||
IndexProjectionSchema.parse({
|
||||
@ -520,6 +525,7 @@ export function createFtsProjectionBuilder({
|
||||
documentAssetId: node.documentAssetId,
|
||||
ftsLanguageStrategy: "mixed-cjk-latin-v1",
|
||||
ftsText,
|
||||
indexingStrategy: TEXT_INDEXING_STRATEGY,
|
||||
...multimodalProjectionMetadata(node),
|
||||
parseArtifactId: node.parseArtifactId,
|
||||
parser: "database-fts",
|
||||
@ -564,7 +570,9 @@ export function createFtsProjectionBuilder({
|
||||
projection.type !== "fts" ||
|
||||
projection.metadata.artifactHash !== incoming.metadata.artifactHash ||
|
||||
projection.metadata.documentAssetId !== incoming.metadata.documentAssetId ||
|
||||
projection.metadata.parseArtifactId !== incoming.metadata.parseArtifactId
|
||||
projection.metadata.parseArtifactId !== incoming.metadata.parseArtifactId ||
|
||||
projection.metadata.indexingStrategy !== TEXT_INDEXING_STRATEGY ||
|
||||
projection.metadata.ftsText !== incoming.metadata.ftsText
|
||||
) {
|
||||
throw new Error(
|
||||
`Persisted generation-scoped FTS projection id=${projection.id} cannot be reused`,
|
||||
@ -594,6 +602,28 @@ export function createFtsProjectionBuilder({
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Add trusted semantic navigation context to search indexes without changing the cited source
|
||||
* text stored on the knowledge node. Table rows and list continuations commonly omit their parent
|
||||
* heading, so indexing only `node.text` makes them impossible to recall from a heading-led query.
|
||||
*/
|
||||
function textIndexContentForNode(node: KnowledgeNode): string {
|
||||
const sectionPath = node.sourceLocation.sectionPath
|
||||
.map((segment) => segment.trim())
|
||||
.filter(Boolean)
|
||||
.join(" > ");
|
||||
const semanticChunking = isPlainObject(node.metadata.semanticChunking)
|
||||
? node.metadata.semanticChunking
|
||||
: undefined;
|
||||
const section =
|
||||
semanticChunking && isPlainObject(semanticChunking.section)
|
||||
? semanticChunking.section
|
||||
: undefined;
|
||||
const summary = section && typeof section.summary === "string" ? section.summary.trim() : "";
|
||||
|
||||
return [sectionPath, summary, node.text.trim()].filter(Boolean).join("\n\n");
|
||||
}
|
||||
|
||||
export function createVisualEmbeddingProjectionBuilder({
|
||||
generateId,
|
||||
maxBatchSize,
|
||||
|
||||
@ -303,7 +303,7 @@ describe("LLM semantic chunker", () => {
|
||||
completed: true,
|
||||
entityCount: 2,
|
||||
model: "reasoner-model",
|
||||
promptVersion: "semantic-chunking-v5",
|
||||
promptVersion: "semantic-chunking-v6",
|
||||
},
|
||||
relationExtraction: { completed: true, relationCount: 1 },
|
||||
semanticChunking: {
|
||||
@ -353,6 +353,9 @@ describe("LLM semantic chunker", () => {
|
||||
expect(provider.calls[0]?.messages[0]?.content).toContain(
|
||||
"prefer natural topic boundaries over filling chunks",
|
||||
);
|
||||
expect(provider.calls[0]?.messages[0]?.content).toContain(
|
||||
"keep the complete list together under its heading",
|
||||
);
|
||||
});
|
||||
|
||||
it("hard-splits an overlong sentence by Unicode grapheme without overlap", async () => {
|
||||
@ -557,7 +560,7 @@ describe("LLM semantic chunker", () => {
|
||||
windowPlanning: {
|
||||
atomicDocument: false,
|
||||
sourceSectionPathCount: 2,
|
||||
version: "v5",
|
||||
version: "v6",
|
||||
},
|
||||
});
|
||||
});
|
||||
@ -633,13 +636,61 @@ describe("LLM semantic chunker", () => {
|
||||
parseArtifact.elements.map((element) => element.text?.trim() ?? "").join("\n"),
|
||||
);
|
||||
expect(nodes[0]?.metadata.semanticChunking).toMatchObject({
|
||||
windowPlanning: { version: "v5" },
|
||||
windowPlanning: { version: "v6" },
|
||||
});
|
||||
expect(v2Nodes[0]?.metadata.semanticChunking).toMatchObject({
|
||||
windowPlanning: { version: "v2" },
|
||||
});
|
||||
});
|
||||
|
||||
it("extends a fixed core boundary to keep a numbered list in one window", async () => {
|
||||
const parseArtifact = artifact([
|
||||
...Array.from({ length: 24 }, (_, index) => ({
|
||||
id: `preamble-${index}`,
|
||||
metadata: {},
|
||||
sectionPath: ["Preamble"],
|
||||
text: `Preamble ${index}.`,
|
||||
type: "paragraph" as const,
|
||||
})),
|
||||
{
|
||||
id: "list-heading",
|
||||
metadata: {},
|
||||
sectionPath: ["Negative list"],
|
||||
text: "施工企业安全穿透式管理负面清单",
|
||||
type: "paragraph" as const,
|
||||
},
|
||||
...Array.from({ length: 9 }, (_, index) => ({
|
||||
id: `list-item-${index + 1}`,
|
||||
metadata: {},
|
||||
sectionPath: ["Negative list"],
|
||||
text: `${index + 1}. 第 ${index + 1} 项负面情形。`,
|
||||
type: "paragraph" as const,
|
||||
})),
|
||||
]);
|
||||
const provider = new ScriptedProvider([echoEachUnit]);
|
||||
const nodes = await createLlmSemanticChunker({
|
||||
reasoningProviderFactory: () => provider,
|
||||
}).chunk({
|
||||
knowledgeSpaceId: KNOWLEDGE_SPACE_ID,
|
||||
parseArtifact,
|
||||
retrievalProfile: profile(),
|
||||
});
|
||||
|
||||
expect(preflightLlmSemanticWindows({ parseArtifact })).toEqual({
|
||||
maximumWindowCount: 1,
|
||||
unitCount: 43,
|
||||
});
|
||||
expect(provider.calls).toHaveLength(1);
|
||||
const prompt = JSON.parse(
|
||||
provider.calls[0]?.messages.find((message) => message.role === "user")?.content ?? "{}",
|
||||
) as PromptPayload;
|
||||
expect(prompt.units).toHaveLength(43);
|
||||
expect(prompt.units.at(-1)?.text).toBe("第 9 项负面情形。");
|
||||
expect(nodes.map((node) => node.text).join("")).toBe(
|
||||
parseArtifact.elements.map((element) => element.text?.trim() ?? "").join(""),
|
||||
);
|
||||
});
|
||||
|
||||
it("runs fixed-core semantic windows concurrently while preserving document order", async () => {
|
||||
const parseArtifact = artifact(
|
||||
Array.from({ length: 96 }, (_, index) => ({
|
||||
@ -870,7 +921,7 @@ describe("LLM semantic chunker", () => {
|
||||
windowPlanning: {
|
||||
atomicDocument: true,
|
||||
sourceSectionPathCount: 1,
|
||||
version: "v5",
|
||||
version: "v6",
|
||||
},
|
||||
});
|
||||
expect(nodes[0]?.metadata.extractedEntities).toEqual([
|
||||
@ -2351,6 +2402,18 @@ describe("LLM semantic chunker", () => {
|
||||
expect(() =>
|
||||
createLlmSemanticChunker({ reasoningProviderFactory: factory, temperature: -1 }),
|
||||
).toThrow("temperature must be non-negative");
|
||||
expect(() =>
|
||||
createLlmSemanticChunker({
|
||||
maxProviderOutputRetries: 4,
|
||||
reasoningProviderFactory: factory,
|
||||
}),
|
||||
).toThrow("maxProviderOutputRetries must be at most 3");
|
||||
expect(() =>
|
||||
createLlmSemanticChunker({
|
||||
maxProviderOutputRetries: -1,
|
||||
reasoningProviderFactory: factory,
|
||||
}),
|
||||
).toThrow("maxProviderOutputRetries must be a non-negative integer");
|
||||
for (const [name, options] of [
|
||||
["maxEntitiesPerChunk", { maxEntitiesPerChunk: 0 }],
|
||||
["maxNodes", { maxNodes: 0 }],
|
||||
@ -2836,6 +2899,29 @@ describe("LLM semantic chunker", () => {
|
||||
).rejects.toThrow("contiguously without gaps or overlap");
|
||||
});
|
||||
|
||||
it("retries one invalid provider response before committing the window", async () => {
|
||||
const provider = new ScriptedProvider([() => ({ chunks: "invalid" }), echoWholeWindow]);
|
||||
const nodes = await createLlmSemanticChunker({
|
||||
maxProviderOutputRetries: 1,
|
||||
reasoningProviderFactory: () => provider,
|
||||
}).chunk({
|
||||
knowledgeSpaceId: KNOWLEDGE_SPACE_ID,
|
||||
parseArtifact: artifact([
|
||||
{
|
||||
id: "retry-provider-output",
|
||||
metadata: {},
|
||||
sectionPath: [],
|
||||
text: "Alpha. Beta.",
|
||||
type: "paragraph",
|
||||
},
|
||||
]),
|
||||
retrievalProfile: profile(),
|
||||
});
|
||||
|
||||
expect(nodes).toHaveLength(1);
|
||||
expect(provider.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("accepts JSON wrapped in provider prose but strictly caps joint extraction arrays", async () => {
|
||||
const input = {
|
||||
knowledgeSpaceId: KNOWLEDGE_SPACE_ID,
|
||||
|
||||
@ -62,14 +62,16 @@ const DEFAULT_MAX_ENTITIES_PER_CHUNK = 100;
|
||||
const DEFAULT_MAX_RELATIONS_PER_CHUNK = 100;
|
||||
const DEFAULT_MAX_OUTPUT_TOKENS = 6_000;
|
||||
const DEFAULT_MAX_RESPONSE_CHARS = 1_000_000;
|
||||
const DEFAULT_PROMPT_VERSION = "semantic-chunking-v5";
|
||||
const DEFAULT_PROMPT_VERSION = "semantic-chunking-v6";
|
||||
const SEMANTIC_CHUNKING_V2_PROMPT_VERSION = "semantic-chunking-v2";
|
||||
const SEMANTIC_CHUNKING_V3_PROMPT_VERSION = "semantic-chunking-v3";
|
||||
const SEMANTIC_CHUNKING_V4_PROMPT_VERSION = "semantic-chunking-v4";
|
||||
const SEMANTIC_CHUNKING_V5_PROMPT_VERSION = "semantic-chunking-v5";
|
||||
const V3_MAX_CORE_UNITS_PER_WINDOW = 32;
|
||||
const V3_MAX_LOOK_AHEAD_UNITS_PER_WINDOW = 8;
|
||||
const DEFAULT_MAX_CONCURRENT_WINDOWS = 4;
|
||||
const MAX_CONCURRENT_WINDOWS = 32;
|
||||
const MAX_PROVIDER_OUTPUT_RETRIES = 3;
|
||||
const SEMANTIC_CHUNKING_STRATEGY = "llm-semantic-v1";
|
||||
const SEMANTIC_CHUNKING_SCHEMA_VERSION = 1;
|
||||
/**
|
||||
@ -152,6 +154,7 @@ export interface LlmSemanticChunkerOptions {
|
||||
readonly maxEntitiesPerChunk?: number | undefined;
|
||||
readonly maxNodes?: number | undefined;
|
||||
readonly maxOutputTokens?: number | undefined;
|
||||
readonly maxProviderOutputRetries?: number | undefined;
|
||||
readonly maxRelationsPerChunk?: number | undefined;
|
||||
readonly maxResponseChars?: number | undefined;
|
||||
readonly maxWindowChars?: number | undefined;
|
||||
@ -310,7 +313,7 @@ interface SemanticWindowTableSchema {
|
||||
readonly sourceElementId: string;
|
||||
}
|
||||
|
||||
type SemanticWindowPlanningVersion = "v1" | "v2" | "v3" | "v4" | "v5";
|
||||
type SemanticWindowPlanningVersion = "v1" | "v2" | "v3" | "v4" | "v5" | "v6";
|
||||
|
||||
interface SemanticWindowPlanningPolicy {
|
||||
readonly atomicDocument: boolean;
|
||||
@ -395,6 +398,7 @@ export function createLlmSemanticChunker({
|
||||
maxEntitiesPerChunk = DEFAULT_MAX_ENTITIES_PER_CHUNK,
|
||||
maxNodes = DEFAULT_MAX_NODES,
|
||||
maxOutputTokens = DEFAULT_MAX_OUTPUT_TOKENS,
|
||||
maxProviderOutputRetries = 0,
|
||||
maxRelationsPerChunk = DEFAULT_MAX_RELATIONS_PER_CHUNK,
|
||||
maxResponseChars = DEFAULT_MAX_RESPONSE_CHARS,
|
||||
maxWindowChars = DEFAULT_MAX_WINDOW_CHARS,
|
||||
@ -410,6 +414,7 @@ export function createLlmSemanticChunker({
|
||||
validatePositiveInteger("maxEntitiesPerChunk", maxEntitiesPerChunk);
|
||||
validatePositiveInteger("maxNodes", maxNodes);
|
||||
validatePositiveInteger("maxOutputTokens", maxOutputTokens);
|
||||
validateNonnegativeInteger("maxProviderOutputRetries", maxProviderOutputRetries);
|
||||
validatePositiveInteger("maxRelationsPerChunk", maxRelationsPerChunk);
|
||||
validatePositiveInteger("maxResponseChars", maxResponseChars);
|
||||
validatePositiveInteger("maxWindowChars", maxWindowChars);
|
||||
@ -421,6 +426,11 @@ export function createLlmSemanticChunker({
|
||||
`LLM semantic chunking maxConcurrentWindows must be at most ${MAX_CONCURRENT_WINDOWS}`,
|
||||
);
|
||||
}
|
||||
if (maxProviderOutputRetries > MAX_PROVIDER_OUTPUT_RETRIES) {
|
||||
throw new Error(
|
||||
`LLM semantic chunking maxProviderOutputRetries must be at most ${MAX_PROVIDER_OUTPUT_RETRIES}`,
|
||||
);
|
||||
}
|
||||
if (!promptVersion.trim()) {
|
||||
throw new Error("LLM semantic chunking promptVersion is required");
|
||||
}
|
||||
@ -479,7 +489,7 @@ export function createLlmSemanticChunker({
|
||||
let nextUnitIndex = 0;
|
||||
let windowIndex = 0;
|
||||
|
||||
const processWindow = async (window: SemanticWindow) => {
|
||||
const processWindowAttempt = async (window: SemanticWindow, retryCount: number) => {
|
||||
input.signal?.throwIfAborted();
|
||||
const messages = semanticChunkingMessages({
|
||||
enableGraph: input.enableGraph !== false,
|
||||
@ -597,7 +607,7 @@ export function createLlmSemanticChunker({
|
||||
itemCount: window.units.length,
|
||||
outcome: "succeeded",
|
||||
providerCalls: checkpointHit ? 0 : 1,
|
||||
retries: 0,
|
||||
retries: retryCount,
|
||||
stage: "semantic-chunking",
|
||||
...(checkpointHit ? {} : ingestionModelUsageFromMetadata(resolvedCompletion.metadata)),
|
||||
});
|
||||
@ -609,7 +619,7 @@ export function createLlmSemanticChunker({
|
||||
itemCount: window.units.length,
|
||||
outcome: "failed",
|
||||
providerCalls: checkpointHit ? 0 : 1,
|
||||
retries: 0,
|
||||
retries: retryCount,
|
||||
stage: "semantic-chunking",
|
||||
...(checkpointHit || !completion
|
||||
? {}
|
||||
@ -619,6 +629,23 @@ export function createLlmSemanticChunker({
|
||||
}
|
||||
};
|
||||
|
||||
const processWindow = async (window: SemanticWindow) => {
|
||||
let retryCount = 0;
|
||||
while (true) {
|
||||
try {
|
||||
return await processWindowAttempt(window, retryCount);
|
||||
} catch (error) {
|
||||
if (
|
||||
retryCount >= maxProviderOutputRetries ||
|
||||
!isRetryableSemanticProviderOutputError(error)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
retryCount += 1;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const appendProcessedWindow = (
|
||||
processed: Awaited<ReturnType<typeof processWindow>>,
|
||||
): void => {
|
||||
@ -1805,7 +1832,7 @@ function semanticRanges(
|
||||
element: MaterializedElement,
|
||||
planningVersion: SemanticWindowPlanningVersion,
|
||||
): SemanticRange[] {
|
||||
if (planningVersion === "v5" && element.elementType === "table") {
|
||||
if ((planningVersion === "v5" || planningVersion === "v6") && element.elementType === "table") {
|
||||
return semanticTableRanges(element);
|
||||
}
|
||||
if (element.elementType !== "paragraph" && element.elementType !== "list") {
|
||||
@ -2112,7 +2139,8 @@ function resolveSemanticWindowPlanningPolicy({
|
||||
}
|
||||
|
||||
function semanticWindowPlanningVersion(promptVersion: string): SemanticWindowPlanningVersion {
|
||||
if (promptVersion === DEFAULT_PROMPT_VERSION) return "v5";
|
||||
if (promptVersion === DEFAULT_PROMPT_VERSION) return "v6";
|
||||
if (promptVersion === SEMANTIC_CHUNKING_V5_PROMPT_VERSION) return "v5";
|
||||
if (promptVersion === SEMANTIC_CHUNKING_V4_PROMPT_VERSION) return "v4";
|
||||
if (promptVersion === SEMANTIC_CHUNKING_V3_PROMPT_VERSION) return "v3";
|
||||
if (promptVersion === SEMANTIC_CHUNKING_V2_PROMPT_VERSION) return "v2";
|
||||
@ -2120,11 +2148,11 @@ function semanticWindowPlanningVersion(promptVersion: string): SemanticWindowPla
|
||||
}
|
||||
|
||||
function usesBoundedCoreUnits(version: SemanticWindowPlanningVersion): boolean {
|
||||
return version === "v3" || version === "v4" || version === "v5";
|
||||
return version === "v3" || version === "v4" || version === "v5" || version === "v6";
|
||||
}
|
||||
|
||||
function usesFixedCoreBoundary(version: SemanticWindowPlanningVersion): boolean {
|
||||
return version === "v4" || version === "v5";
|
||||
return version === "v4" || version === "v5" || version === "v6";
|
||||
}
|
||||
|
||||
function materializeSemanticWindow({
|
||||
@ -2152,13 +2180,14 @@ function materializeSemanticWindow({
|
||||
const coreUnits: AtomicUnit[] = [];
|
||||
let cursor = startUnitIndex;
|
||||
while (cursor < units.length) {
|
||||
const candidate = units[cursor] as AtomicUnit;
|
||||
if (
|
||||
usesBoundedCoreUnits(planningPolicy.version) &&
|
||||
coreUnits.length >= V3_MAX_CORE_UNITS_PER_WINDOW
|
||||
coreUnits.length >= V3_MAX_CORE_UNITS_PER_WINDOW &&
|
||||
!(planningPolicy.version === "v6" && continuesNumberedList(coreUnits.at(-1), candidate))
|
||||
) {
|
||||
break;
|
||||
}
|
||||
const candidate = units[cursor] as AtomicUnit;
|
||||
if (planningPolicy.version === "v1" && !isLegacyWindowCompatible(first, candidate)) {
|
||||
break;
|
||||
}
|
||||
@ -2231,6 +2260,25 @@ function materializeSemanticWindow({
|
||||
};
|
||||
}
|
||||
|
||||
function continuesNumberedList(previous: AtomicUnit | undefined, candidate: AtomicUnit): boolean {
|
||||
if (!previous) return false;
|
||||
if (previous.sourceElement.elementId === candidate.sourceElement.elementId) return true;
|
||||
const previousOrdinal = numberedListOrdinal(previous.sourceElement.text);
|
||||
const candidateOrdinal = numberedListOrdinal(candidate.sourceElement.text);
|
||||
return (
|
||||
previousOrdinal !== undefined &&
|
||||
candidateOrdinal !== undefined &&
|
||||
candidateOrdinal === previousOrdinal + 1
|
||||
);
|
||||
}
|
||||
|
||||
function numberedListOrdinal(text: string): number | undefined {
|
||||
const match = /^\s*(\d{1,4})\s*[.)、．]\s*/u.exec(text);
|
||||
if (!match?.[1]) return undefined;
|
||||
const ordinal = Number.parseInt(match[1], 10);
|
||||
return Number.isSafeInteger(ordinal) ? ordinal : undefined;
|
||||
}
|
||||
|
||||
function semanticPromptUnit(
|
||||
unit: AtomicUnit,
|
||||
planningVersion: SemanticWindowPlanningVersion,
|
||||
@ -2261,7 +2309,7 @@ function semanticPromptUnit(
|
||||
sourceSectionPath: [...unit.sectionPath],
|
||||
}
|
||||
: {}),
|
||||
...(planningVersion === "v5" && unit.tableRecord
|
||||
...((planningVersion === "v5" || planningVersion === "v6") && unit.tableRecord
|
||||
? {
|
||||
tableMode: unit.tableRecord.mode,
|
||||
tableRecordCount: unit.tableRecord.count,
|
||||
@ -2350,6 +2398,7 @@ function semanticChunkingMessages({
|
||||
"Never emit a chunk that starts wholly in lookAheadUnits. Units not consumed from look-ahead will be reconsidered in the next request.",
|
||||
]),
|
||||
"Ranges may be smaller than the maximum; prefer natural topic boundaries over filling chunks.",
|
||||
"When a contiguous numbered or bulleted list fits within the chunk limit, keep the complete list together under its heading.",
|
||||
`Every range must contain at most ${maxChunkChars} Unicode graphemes including separators.`,
|
||||
`Return at most ${maxEntitiesPerChunk} entities and ${maxRelationsPerChunk} relations per chunk.`,
|
||||
"Allowed entity types: date, metric, organization, person, policy, product, term.",
|
||||
@ -2365,7 +2414,7 @@ function semanticChunkingMessages({
|
||||
: [
|
||||
"A unit marked boundaryPolicy=isolated must occupy a chunk containing only units from that same table or image element.",
|
||||
]),
|
||||
...(window.planningVersion === "v5"
|
||||
...(window.planningVersion === "v5" || window.planningVersion === "v6"
|
||||
? [
|
||||
"Table records carry tableMode/tableRecordIndex/tableRecordCount; resolve their bounded columns from tableSchemas by sourceElementId.",
|
||||
"For tableMode=record-list or matrix, return exactly one chunk per table record; hard-split units with the same tableRecordIndex may stay together, but never combine different records.",
|
||||
@ -2404,7 +2453,7 @@ function semanticChunkingMessages({
|
||||
semanticPromptUnit(unit, window.planningVersion),
|
||||
),
|
||||
sectionPath: window.sectionPath,
|
||||
...(window.planningVersion === "v5"
|
||||
...(window.planningVersion === "v5" || window.planningVersion === "v6"
|
||||
? {
|
||||
tableSchemas: semanticWindowTableSchemas([...window.units, ...window.lookAheadUnits]),
|
||||
}
|
||||
@ -2620,6 +2669,14 @@ function parseSemanticChunkingOutput(text: string): LlmSemanticChunkingOutput {
|
||||
}
|
||||
}
|
||||
|
||||
function isRetryableSemanticProviderOutputError(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
(error.message === "LLM semantic chunking provider returned invalid JSON" ||
|
||||
error.message === "LLM semantic chunking provider returned an invalid response schema")
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeTableRecordBoundaries({
|
||||
maxChunkChars,
|
||||
output,
|
||||
@ -2629,7 +2686,12 @@ function normalizeTableRecordBoundaries({
|
||||
readonly output: LlmSemanticChunkingOutput;
|
||||
readonly window: SemanticWindow;
|
||||
}): LlmSemanticChunkingOutput {
|
||||
if (window.planningVersion !== "v5" || window.atomicDocument) return output;
|
||||
if (
|
||||
(window.planningVersion !== "v5" && window.planningVersion !== "v6") ||
|
||||
window.atomicDocument
|
||||
) {
|
||||
return output;
|
||||
}
|
||||
|
||||
const eligibleUnits = [...window.units, ...window.lookAheadUnits];
|
||||
const unitIndex = new Map(eligibleUnits.map((unit, index) => [unit.id, index]));
|
||||
|
||||
5
knowledge-fs/packages/api/src/text-indexing-strategy.ts
Normal file
5
knowledge-fs/packages/api/src/text-indexing-strategy.ts
Normal file
@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Bump when indexed text composition or normalization changes. The strategy participates in
|
||||
* publication fingerprints and prevents reuse of projections built with an older composition.
|
||||
*/
|
||||
export const TEXT_INDEXING_STRATEGY = "section-context-v1";
|
||||
@ -135,6 +135,7 @@ export interface NativeParserOptions {
|
||||
|
||||
export interface UnstructuredParserClientOptions extends NativeParserOptions {
|
||||
readonly apiKey?: string;
|
||||
readonly defaultLanguage?: string;
|
||||
readonly endpoint: string;
|
||||
readonly fetch?: typeof fetch;
|
||||
readonly maxResponseBytes?: number;
|
||||
@ -303,6 +304,7 @@ export function createNativeStructuredDataParser(
|
||||
|
||||
export function createUnstructuredParserClient({
|
||||
apiKey,
|
||||
defaultLanguage,
|
||||
endpoint,
|
||||
fetch: fetchImpl = fetch,
|
||||
maxConcurrency = defaultMaxConcurrency,
|
||||
@ -323,9 +325,12 @@ export function createUnstructuredParserClient({
|
||||
requestGate.run(async () => {
|
||||
const deadline = createUnstructuredRequestDeadline(input.signal, requestTimeoutMs);
|
||||
try {
|
||||
const parserVersion = options.parserVersion ?? "unstructured@9";
|
||||
const parserVersion = options.parserVersion ?? "unstructured@10";
|
||||
const partitionStrategy = unstructuredPartitionStrategy(input);
|
||||
const providerImageBlockTypes = unstructuredProviderImageBlockTypes(input);
|
||||
const providerLanguage = unstructuredLanguage(
|
||||
input.parserHints?.language ?? defaultLanguage,
|
||||
);
|
||||
assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes);
|
||||
const response = await fetchWithRetries({
|
||||
buildRequest: () => {
|
||||
@ -337,6 +342,7 @@ export function createUnstructuredParserClient({
|
||||
form.set("files", new File([fileBody], input.filename, { type: input.mimeType }));
|
||||
form.set("coordinates", "true");
|
||||
form.set("strategy", partitionStrategy);
|
||||
if (providerLanguage) form.set("languages", providerLanguage);
|
||||
if (providerImageBlockTypes.length > 0) {
|
||||
for (const blockType of providerImageBlockTypes) {
|
||||
form.append("extract_image_block_types", blockType);
|
||||
@ -388,6 +394,7 @@ export function createUnstructuredParserClient({
|
||||
artifactHashContext: unstructuredArtifactHashContext(input, {
|
||||
partitionStrategy,
|
||||
providerImageBlockTypes,
|
||||
providerLanguage,
|
||||
}),
|
||||
elements,
|
||||
input,
|
||||
@ -436,6 +443,28 @@ function shouldRequestProviderImages(input: ParseDocumentInput): boolean {
|
||||
return unstructuredProviderImageBlockTypes(input).length > 0;
|
||||
}
|
||||
|
||||
function unstructuredLanguage(language: string | undefined): string | undefined {
|
||||
const normalized = language?.trim().toLowerCase();
|
||||
if (!normalized) return undefined;
|
||||
|
||||
const baseLanguage = normalized.split("-", 1)[0] ?? normalized;
|
||||
return (
|
||||
{
|
||||
ar: "ara",
|
||||
de: "deu",
|
||||
en: "eng",
|
||||
es: "spa",
|
||||
fr: "fra",
|
||||
hi: "hin",
|
||||
ja: "jpn",
|
||||
ko: "kor",
|
||||
pt: "por",
|
||||
ru: "rus",
|
||||
zh: "zho",
|
||||
}[baseLanguage] ?? normalized
|
||||
);
|
||||
}
|
||||
|
||||
function unstructuredProviderImageBlockTypes(
|
||||
input: ParseDocumentInput,
|
||||
): readonly ("Image" | "Table")[] {
|
||||
@ -472,6 +501,7 @@ function unstructuredArtifactHashContext(
|
||||
request: {
|
||||
readonly partitionStrategy: "auto" | "fast" | "hi_res";
|
||||
readonly providerImageBlockTypes: readonly ("Image" | "Table")[];
|
||||
readonly providerLanguage?: string | undefined;
|
||||
},
|
||||
): string {
|
||||
const hints = input.parserHints;
|
||||
@ -491,6 +521,7 @@ function unstructuredArtifactHashContext(
|
||||
coordinates: true,
|
||||
imageBlockTypes: request.providerImageBlockTypes,
|
||||
imagePayload: request.providerImageBlockTypes.length > 0,
|
||||
language: request.providerLanguage ?? null,
|
||||
strategy: request.partitionStrategy,
|
||||
},
|
||||
});
|
||||
@ -2307,7 +2338,10 @@ function unstructuredElementsToElements(
|
||||
const type = unstructuredType(sourceElement.type);
|
||||
const tableProjection =
|
||||
type === "table" ? unstructuredTableProjection(sourceElement.metadata) : undefined;
|
||||
const text = tableProjection?.text ?? normalizeText(sourceElement.text ?? "");
|
||||
const providerText = tableProjection?.text ?? normalizeText(sourceElement.text ?? "");
|
||||
const text = hasChineseOcrLanguage(sourceElement.metadata)
|
||||
? normalizeChineseOcrText(providerText)
|
||||
: providerText;
|
||||
|
||||
if (!text && !hasUnstructuredVisualMetadata(sourceElement.metadata, type)) {
|
||||
continue;
|
||||
@ -2349,6 +2383,26 @@ function unstructuredElementsToElements(
|
||||
return elements;
|
||||
}
|
||||
|
||||
function hasChineseOcrLanguage(metadata: Readonly<Record<string, unknown>>): boolean {
|
||||
const languages = metadata.languages;
|
||||
return (
|
||||
Array.isArray(languages) &&
|
||||
languages.some(
|
||||
(language) =>
|
||||
typeof language === "string" &&
|
||||
(language.trim().toLowerCase() === "zho" ||
|
||||
language.trim().toLowerCase().startsWith("zh-")),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeChineseOcrText(text: string): string {
|
||||
return text.replace(
|
||||
/(?<=[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff])[\t \u3000]+(?=[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff])/gu,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
function unstructuredCategoryDepth(
|
||||
metadata: Readonly<Record<string, unknown>>,
|
||||
): number | undefined {
|
||||
|
||||
@ -1342,7 +1342,7 @@ describe("parser adapters", () => {
|
||||
metadata: {
|
||||
filename: "report.pdf",
|
||||
mimeType: "application/pdf",
|
||||
parserVersion: "unstructured@9",
|
||||
parserVersion: "unstructured@10",
|
||||
},
|
||||
parser: "unstructured",
|
||||
version: 1,
|
||||
@ -1410,6 +1410,54 @@ describe("parser adapters", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("normalizes spacing between Chinese OCR characters without rewriting words", async () => {
|
||||
const parser = createUnstructuredParserClient({
|
||||
endpoint: "https://unstructured.example.test",
|
||||
fetch: async () =>
|
||||
new Response(
|
||||
JSON.stringify([
|
||||
{
|
||||
metadata: { languages: ["zho"], page_number: 1 },
|
||||
text: "随 道 工 程 。源 道 初 期 支 护 与 开 挖 作 业",
|
||||
type: "NarrativeText",
|
||||
},
|
||||
{
|
||||
metadata: {
|
||||
languages: ["zho"],
|
||||
page_number: 1,
|
||||
text_as_html: "<table><tr><td>施 工 条 件</td><td>临 时 用 电</td></tr></table>",
|
||||
},
|
||||
text: "施 工 条 件 临 时 用 电",
|
||||
type: "Table",
|
||||
},
|
||||
{
|
||||
metadata: { languages: ["zho"], page_number: 1 },
|
||||
text: "随着道路工程推进",
|
||||
type: "NarrativeText",
|
||||
},
|
||||
]),
|
||||
{ headers: { "content-type": "application/json" }, status: 200 },
|
||||
),
|
||||
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c51",
|
||||
now: () => createdAt,
|
||||
});
|
||||
|
||||
const artifact = await parser.parse({
|
||||
body: new Uint8Array([1, 2, 3]),
|
||||
documentAssetId,
|
||||
filename: "tunnel.pdf",
|
||||
mimeType: "application/pdf",
|
||||
version: 1,
|
||||
});
|
||||
|
||||
expect(artifact.elements.map((element) => element.text)).toEqual([
|
||||
"随道工程 。源道初期支护与开挖作业",
|
||||
"column_1: 施工条件 | column_2: 临时用电",
|
||||
"随着道路工程推进",
|
||||
]);
|
||||
expect(artifact.metadata.parserVersion).toBe("unstructured@10");
|
||||
});
|
||||
|
||||
it("projects an Unstructured spreadsheet table into independently retrievable records", async () => {
|
||||
const parser = createUnstructuredParserClient({
|
||||
endpoint: "https://unstructured.example.test",
|
||||
@ -1539,7 +1587,7 @@ describe("parser adapters", () => {
|
||||
version: 1,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
metadata: { parserVersion: "unstructured@9" },
|
||||
metadata: { parserVersion: "unstructured@10" },
|
||||
parser: "unstructured",
|
||||
});
|
||||
},
|
||||
@ -1632,6 +1680,7 @@ describe("parser adapters", () => {
|
||||
});
|
||||
const parseWithHints = (parserHints: {
|
||||
readonly imagesHandledExternally?: boolean;
|
||||
readonly language?: string;
|
||||
readonly layoutComplexity?: "complex" | "simple";
|
||||
readonly requiresImages?: boolean;
|
||||
readonly requiresOcr?: boolean;
|
||||
@ -1646,12 +1695,13 @@ describe("parser adapters", () => {
|
||||
version: 1,
|
||||
});
|
||||
|
||||
const [fast, ocr, tables, providerImages, externalImages] = await Promise.all([
|
||||
const [fast, ocr, tables, providerImages, externalImages, chinese] = await Promise.all([
|
||||
parseWithHints({ layoutComplexity: "simple" }),
|
||||
parseWithHints({ requiresOcr: true }),
|
||||
parseWithHints({ requiresTables: true }),
|
||||
parseWithHints({ requiresImages: true }),
|
||||
parseWithHints({ imagesHandledExternally: true, requiresImages: true }),
|
||||
parseWithHints({ language: "zh-CN" }),
|
||||
]);
|
||||
|
||||
expect(
|
||||
@ -1661,8 +1711,9 @@ describe("parser adapters", () => {
|
||||
tables.artifactHash,
|
||||
providerImages.artifactHash,
|
||||
externalImages.artifactHash,
|
||||
chinese.artifactHash,
|
||||
]).size,
|
||||
).toBe(5);
|
||||
).toBe(6);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
Loading…
Reference in New Issue
Block a user