From ffd2a7094d427171a5070cb024192f71a647be2f Mon Sep 17 00:00:00 2001 From: Jyong Date: Thu, 13 Aug 2026 13:10:39 -0400 Subject: [PATCH] restore semantic document compilation pipeline --- api/knowledge-fs-contract.lock.json | 2 +- ...mantic-document-compilation-restoration.md | 103 + .../docs/consolidated-iteration-plan.md | 21 +- ...c-document-compilation-restoration-plan.md | 180 ++ ...cument-compilation-runtime-options.test.ts | 17 + .../document-compilation-runtime-options.ts | 26 +- knowledge-fs/apps/api/src/index.ts | 6 + knowledge-fs/apps/api/src/migrate.test.ts | 6 + knowledge-fs/package.json | 8 +- .../adapters/src/migration-runner.test.ts | 2 + knowledge-fs/packages/api/package.json | 1 + .../src/document-compilation-worker.test.ts | 90 +- .../api/src/document-compilation-worker.ts | 94 +- .../src/document-layout-recomposer.test.ts | 172 ++ .../api/src/document-layout-recomposer.ts | 157 ++ .../api/src/document-outline-builder.test.ts | 42 + .../api/src/document-outline-builder.ts | 20 +- ...ment-semantic-enrichment-processor.test.ts | 495 +++- .../document-semantic-enrichment-processor.ts | 255 +- ...cument-semantic-enrichment-runtime.test.ts | 2 + .../api/src/index-reindexer-semantic.test.ts | 752 ++++++ .../packages/api/src/index-reindexer.test.ts | 54 + .../packages/api/src/index-reindexer.ts | 900 ++++++- knowledge-fs/packages/api/src/index.ts | 2 + ...node-generation-receipt-repository.test.ts | 504 ++++ .../api/src/knowledge-node-repository.ts | 776 +++++- ...rofile-migration-candidate-builder.test.ts | 338 ++- ...ace-profile-migration-candidate-builder.ts | 735 +++++- .../api/src/llm-semantic-chunker.test.ts | 2020 ++++++++++++++ .../packages/api/src/llm-semantic-chunker.ts | 2326 +++++++++++++++++ .../api/src/semantic-generation-receipt.ts | 169 ++ ..._semantic_generation_receipts.postgres.sql | 44 + ...0043_semantic_generation_receipts.tidb.sql | 46 + .../src/migration-artifacts.generated.ts | 2 + .../database/src/migration-file.test.ts | 4 + .../packages/database/src/schema.test.ts | 34 + knowledge-fs/packages/database/src/schema.ts | 86 + knowledge-fs/pnpm-lock.yaml | 3 + .../scripts/semantic-compilation-rollout.mjs | 374 +++ .../semantic-compilation-rollout.test.mjs | 186 ++ 40 files changed, 10753 insertions(+), 301 deletions(-) create mode 100644 knowledge-fs/.harness/changes/2026-08-13-semantic-document-compilation-restoration.md create mode 100644 knowledge-fs/.harness/docs/semantic-document-compilation-restoration-plan.md create mode 100644 knowledge-fs/packages/api/src/document-layout-recomposer.test.ts create mode 100644 knowledge-fs/packages/api/src/document-layout-recomposer.ts create mode 100644 knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts create mode 100644 knowledge-fs/packages/api/src/knowledge-node-generation-receipt-repository.test.ts create mode 100644 knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts create mode 100644 knowledge-fs/packages/api/src/llm-semantic-chunker.ts create mode 100644 knowledge-fs/packages/api/src/semantic-generation-receipt.ts create mode 100644 knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.postgres.sql create mode 100644 knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.tidb.sql create mode 100644 knowledge-fs/scripts/semantic-compilation-rollout.mjs create mode 100644 knowledge-fs/scripts/semantic-compilation-rollout.test.mjs diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 3a48ada84a5..5d7cb542d4d 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "d89aa89a543c1c1a5b3d042881597d9af2a1a47b", + "subtreeTree": "73665ebbd8b6e07538c7d07f6983f17922dce439", "openapiSha256": "47936a7d9ffdc27e2b2b8982a90e1936dc3bf59a64c316f452a6912ec1d2fcd6", "capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", diff --git a/knowledge-fs/.harness/changes/2026-08-13-semantic-document-compilation-restoration.md b/knowledge-fs/.harness/changes/2026-08-13-semantic-document-compilation-restoration.md new file mode 100644 index 00000000000..a802652c22b --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-08-13-semantic-document-compilation-restoration.md @@ -0,0 +1,103 @@ +# Semantic Document Compilation Restoration + +## Summary + +Restores the missing reasoning-model semantic compilation stage between parsing and indexing. +Parser output is normalized without mutating the stored parse artifact, the configured reasoning +model selects contiguous source ranges, and the service materializes authoritative chunk text from +those source elements. Outline, summaries, PageIndex, dense/full-text projections, and Graph facts +now share one immutable publication generation. + +## Behavior and invariants + +- Untrusted Unstructured `Title`/`Heading` classifications no longer become compulsory chunk + boundaries; explicit parser hierarchy remains trusted. +- Semantic requests are bounded by elements, windows, response bytes, chunks, entities, and + relations. Tables/images remain atomic at model-window boundaries. +- Model output supplies source ranges and metadata, never authoritative document text. Coverage, + order, terminal model identity, and Unicode grapheme limits fail closed. +- A compact immutable generation receipt makes completed retries provider-free and detects + conflicting or corrupt replay. +- Node rows and their receipt are persisted in one database transaction; all-excluded generations + are represented explicitly. +- New semantic compilations materialize quality-controlled joint Graph facts synchronously before + candidate publication. Graph facts remain generation- and source-node-scoped. +- Reasoning-profile migrations rebuild semantic nodes, outline, paths, PageIndex, search + projections, and Graph together. Embedding-only migrations clone the exact semantic node + generation and rerun only the affected projections/Graph materialization without another LLM + segmentation call. +- Existing published generations remain readable until candidate evaluation and publication CAS + succeed. No semantic profile silently falls back to legacy fixed-size chunking. + +## Data model + +- Adds paired PostgreSQL/TiDB migration `0043_semantic_generation_receipts`. +- Adds the schema/catalog entry, migration registry artifacts, migration runner expectations, and + receipt repository transaction support. + +## Product diagnostics + +- Document compilation continues to expose parsed, outline-built, nodes-generated, + projection-built, evaluated, and published checkpoints. +- Existing document-list failure hover behavior shows the actionable failure reason directly and + keeps the trace id as secondary support information. +- Document outlines are derived from semantic-node section paths and semantic summaries rather + than parser newline rendering. + +## Rollout + +1. Apply migration 0043 before starting the new API/workers. +2. Deploy API and worker runtime together; missing semantic/Graph runtime dependencies fail startup. +3. Canary new imports and compare chunk coherence, outline localization, retrieval recall, provider + calls, latency, and Graph provenance. +4. Rebuild existing documents through normal reindex/profile-migration candidate publication. +5. Drill publication-head rollback before removing the legacy read path. + +The rollout is now executable through guarded, bounded commands: + +- `semantic:rollout:static` verifies migration 0043 and its generated registry without network IO. +- `semantic:rollout:preflight` reads health, settings, documents, and failed reindex baselines. +- `semantic:rollout:canary` accepts only explicit document asset ids, polls every accepted job, and + verifies non-empty semantic outline provenance. +- `semantic:rollout:backfill` uses the existing bounded whole-space bulk reindex path. +- `semantic:rollout:rollback` submits a CAS-bound immutable document revision rollback and verifies + that the requested revision becomes active. +- Canary, backfill, and rollback require `SEMANTIC_ROLLOUT_APPLY=1` plus an exact + `semantic::` confirmation. The script never applies database DDL. + +## Verification + +Passed locally on 2026-08-13: + +- `pnpm --dir knowledge-fs check`, including workspace typechecks/tests, OpenAPI and capability + exports, migration checks, retrieval/phase-4 evaluations, Swagger checks, workflow gates, and + static Docker/Compose smoke checks. +- `pnpm --dir knowledge-fs build` (12/12 tasks). +- `pnpm --dir knowledge-fs test` (22/22 tasks); the API suite passed 4,475 tests with 3 skipped, + API-app passed 252 tests, database passed 114 tests, and adapters passed 105 tests. +- `pnpm --dir knowledge-fs semantic:rollout:test` passed all 5 guarded rollout tests, including + static migration evidence, mutation confirmation, bounded preflight, explicit-document canary, + and package-script registration. +- Retrieval evaluation: recall 0.890, citation 0.880, no-answer rate 0.060, answer accuracy 0.890, + and faithfulness 0.910. Phase-4 regression deltas remained within their configured bounds. +- Repository CI coverage task passed. The semantic compilation change set now reports 90.01% + branch coverage. An additional informational full API coverage run reports 89.48% branches. The + API package is intentionally excluded from the repository CI coverage task; its remaining + historical-package gap stays a tracked cleanup gate before deleting the legacy compiler. +- Biome passed for every changed TypeScript/TSX/JSON source file, and `git diff --check` passed. +- KnowledgeFS contract generation/check passed after regenerating the contract lock from the exact + KnowledgeFS change set. + +Known repository-wide baseline: + +- `pnpm --dir knowledge-fs lint` is not globally green because of pre-existing Admin formatting + findings, generated capability/OpenAPI formatting findings, and the generated OpenAPI size limit. + None is in the semantic compiler change set; all changed files pass Biome. These unrelated files + were deliberately not rewritten in this iteration. + +Not performed locally: + +- Production migration, deployment smoke, canary imports, historical-document rebuild/backfill, + sampled shadow comparison, and publication-head rollback drill. Local static/preflight/canary + simulations and guardrail tests are complete, but these production executions remain SSC.7 + operational gates and must complete before the legacy read/compiler path is removed. diff --git a/knowledge-fs/.harness/docs/consolidated-iteration-plan.md b/knowledge-fs/.harness/docs/consolidated-iteration-plan.md index 3b470542009..8c93b3fba11 100644 --- a/knowledge-fs/.harness/docs/consolidated-iteration-plan.md +++ b/knowledge-fs/.harness/docs/consolidated-iteration-plan.md @@ -1,7 +1,7 @@ # KnowledgeFS Master Iteration Plan > Created: 2026-06-24 -> Updated: 2026-08-07 +> Updated: 2026-08-13 > Source directory: `.harness/docs` > Status: current executable master plan > Rule: historical plans remain source records; this file is the first document to use @@ -28,6 +28,7 @@ and how each slice should be accepted and verified. | `.harness/docs/pageindex-research-retrieval-v2-iteration-plan.md` | Document selection, book-like layered Research navigation, Value node queue, interactive/durable policies, degradation, budgets, and human-golden findability. | Active execution plan. Automatic Golden Question generation is explicitly excluded. | | `.harness/docs/multimodal-knowledgefs-iteration-plan.md` | `DocumentMultimodalManifest`, table/image/code/page inventory, visual assets, thumbnails, VLM answer support, visual embeddings, Admin browser, evals. | Core functional capability implemented. Remaining work is external QA fixtures, provider conformance, and richer trace UX. | | `.harness/docs/image-query-retrieval-iteration-plan.md` | Image-as-query support: query-side image visual embedding (`inputType: "query"`), gateway image transport with typed degradation, query images in VLM answering, and query image-to-text expansion for deep/research. | Implemented 2026-08-07 for the backend-only scope. Retrieval-time recognition of document images remains explicitly excluded (belongs to ingest-side enrichment reindex). | +| `.harness/docs/semantic-document-compilation-restoration-plan.md` | Restores the pre-monorepo profile-aware LLM semantic chunker and adds layout recomposition, durable semantic receipts, unified outline/index/Graph derivation, diagnostics, and rollout gates. | Active P0 regression-restoration track. It precedes further outline-quality work because current parser boundaries leak into chunks, outlines, and Graph inputs. | | `.harness/docs/rag-platform-redesign-technical-selection.md` | Architecture source of truth and technology choices. | Updated to reflect this master plan, PageIndex-inspired outlines, native multimodal contracts, and visual indexing. | | Dify prototype `/datasets` | Product UX target for dataset list/detail, overview readiness, sources, documents, evidence, quality, settings, agent access, and pipeline surfaces. | Used as the Admin/product parity target before deeper quality-only iteration. | @@ -88,7 +89,8 @@ The desired mode is: | Queryable ingestion | Done | QI.1-QI.5: upload creates nodes, local compute runtime, local generator over nodes, evidence query smoke, Admin BFF upload smoke. | None for this plan. | | Durable local runtime | Done | DLR.1-DLR.15: PostgreSQL executor, DB repository bundle, migrations, `.env`, durable smoke, application packaging, app Compose guardrails, API/Admin images and smoke gates. | None for this plan. | | JuiceFS hardening | Done | JH.1-JH.7: manifests, commit ledger, artifact segments, consistency/cache contracts, sessions/leases, fsck/gc/status/stats, quota/projection hardening, Admin/MCP operator UX. | Keep docs/runbooks aligned when related behavior changes. | -| PageIndex outline | V2 backend verification 2026-08-06 | Deterministic schema/builder/repository/API/KnowledgeFS; document shortlist; root-to-leaf layered LLM lane; Value propagation and node queue; per-level/round replay-safe checkpoints; degradation/budget semantics; exact-generation human-golden findability and bounded repair queue. | Automatic Golden Question generation remains explicitly excluded until its product requirements are decided. | +| Semantic document compilation | Local implementation, regression, and rollout automation complete; production execution pending 2026-08-13 | SSC.0-SSC.6 restore profile-aware LLM semantic ranges, immutable source-derived text, durable generation receipts, unified outline/PageIndex/search/Graph derivation, and exact profile-migration replay. SSC.7 provides guarded static/preflight/canary/backfill/rollback commands and tests; the changed semantic chain is above 90% branch coverage. | Apply migration 0043 before workers, run production preflight/canary/backfill/shadow comparison/rollback, close the historical full-API branch gap, then retire the legacy final chunker. | +| PageIndex outline | V2 backend verification 2026-08-06 | Deterministic schema/builder/repository/API/KnowledgeFS; document shortlist; root-to-leaf layered LLM lane; Value propagation and node queue; per-level/round replay-safe checkpoints; degradation/budget semantics; exact-generation human-golden findability and bounded repair queue. | Rebase outline inputs onto restored semantic generations before additional outline-quality tuning. Automatic Golden Question generation remains explicitly excluded until its product requirements are decided. | | Multimodal KnowledgeFS | Mostly done | Manifest, metadata normalization, asset extraction, PDF rasterization, thumbnails, KnowledgeFS descriptors, VLM answer providers, visual embeddings, visual retrieval metrics, Admin browser, eval utilities. | External QA fixtures, provider conformance packs, richer trace drill-downs. | | Prototype product parity | Planned | Underlying APIs and data contracts exist in pieces across KnowledgeFS, SourceFS, EvidenceFS, retrieval, quality, and Admin. | Align Admin routes and workflow APIs with the prototype: dataset list/detail shell, sources, documents, evidence, quality, settings, agent access, and pipeline mode. | | Admin integration | Active | AIR.1-AIR.2 done: upload/readiness/citation paths and local/Compose upstream wiring repaired. | AIR.3/AIR.4: preview panel audit and outline/multimodal trace UX. | @@ -101,13 +103,14 @@ Work should proceed in this order unless a production regression appears: 1. Documentation alignment and planning source of truth. 2. Prototype product surface parity for the dataset workspace. -3. PageIndex outline quality hardening. -4. Multimodal functional completion. -5. Admin integration honesty and trace drill-downs. -6. API/code-health closure. -7. Research-mode completeness over outline + multimodal + graph evidence. -8. Evaluation governance and CI regression hardening. -9. Optional provider, deployment, and adapter expansion. +3. Restore semantic document compilation (SSC.0-SSC.7). +4. PageIndex outline quality hardening on semantic generations. +5. Multimodal functional completion. +6. Admin integration honesty and trace drill-downs. +7. API/code-health closure. +8. Research-mode completeness over outline + multimodal + graph evidence. +9. Evaluation governance and CI regression hardening. +10. Optional provider, deployment, and adapter expansion. Every slice should: diff --git a/knowledge-fs/.harness/docs/semantic-document-compilation-restoration-plan.md b/knowledge-fs/.harness/docs/semantic-document-compilation-restoration-plan.md new file mode 100644 index 00000000000..4fcf6bffd01 --- /dev/null +++ b/knowledge-fs/.harness/docs/semantic-document-compilation-restoration-plan.md @@ -0,0 +1,180 @@ +# Semantic Document Compilation Restoration Plan + +> Created: 2026-08-13 +> Status: Implementation and local regression complete; production rollout verification active +> Owner boundary: KnowledgeFS TypeScript ingestion/compiler, Dify Admin integration +> Historical implementation reference: standalone KnowledgeFS commit `b3aa9ce` + +## 1. Why this restoration exists + +The product contract requires document compilation to parse the source, reconstruct a trustworthy +reading order, ask the knowledge-space reasoning model for semantic boundaries, and derive the +outline, summaries, vector/full-text projections, and Graph facts from the same immutable semantic +generation. + +The monorepo currently persists the parser artifact and sends it directly to the deterministic +1,200-grapheme chunker. The reasoning model enriches an already-built outline and later extracts +Graph facts, so parser heading mistakes become hard chunk boundaries. The semantic chunking +implementation previously shipped in the standalone KnowledgeFS repository was not carried into +the monorepo migration. This track restores that behavior and extends it with layout recomposition +so forms, invoices, tables, and multi-column documents do not preserve false parser boundaries. + +## 2. Non-negotiable invariants + +1. Raw parser output remains immutable and auditable. +2. Normalized elements retain source element ids, pages, byte offsets, and bounding boxes. +3. The reasoning model selects source ranges; it never supplies authoritative chunk text. +4. Every eligible source unit is covered exactly once by leaf chunks. Gaps, overlap, reordering, + invalid ids, and over-limit output fail closed. +5. Tables/images stay atomic unless a bounded deterministic safety split is unavoidable. +6. The model selection, capability identity, prompt version, input fingerprint, output fingerprint, + and window manifest are frozen with the candidate publication. +7. A retry reuses or proves the complete generation; it cannot append duplicate nodes or Graph rows. +8. Outline, summary, PageIndex, dense/FTS projections, and Graph facts bind to one publication + generation and publish atomically. +9. Provider/configuration failures remain actionable. No silent fallback to legacy character + chunking is allowed for a profile that requires semantic compilation. +10. All model/network/memory/database work is bounded and batch-oriented. + +## 3. Target pipeline + +```text +source bytes + -> immutable ParseArtifact + -> deterministic LayoutRecompositionArtifact + -> frozen reasoning-model SemanticSegmentationPlan + -> fail-closed coverage/provenance validation + -> immutable semantic KnowledgeNode generation + receipt + -> outline/summary + PageIndex + -> dense/FTS/metadata projections + Graph facts + -> candidate evaluation + -> atomic publication head CAS +``` + +## 4. Execution slices + +| ID | Status | Slice | Required behavior | Regression gate | +|---|---|---|---|---| +| SSC.0 | Complete 2026-08-13 | Baseline and migration audit | Restore the historical design record, create redacted structured-document fixtures, map current compiler/publication contracts, and prove the existing deterministic path reproduces fragmented output. | Focused parser/chunker/compiler tests fail for the new semantic contract before implementation. | +| SSC.1 | Complete 2026-08-13 | Layout normalization and boundary recomposition | Reuse the parser's bounded coordinate normalization for vertical CJK/noise, keep canonical element order and tables, classify Unstructured heading confidence, and preserve complete source provenance without mutating the stored parse artifact. | Invoice/form, trusted/untrusted heading, native parser, table isolation, and bounded-element tests. Multi-column reading order remains the parser provider's responsibility and is not silently reordered downstream. | +| SSC.2 | Complete 2026-08-13 | LLM semantic plan | Restore the profile-aware semantic chunker, bounded windows/look-ahead, structured output, joint entity/relation extraction, terminal model identity verification, and deterministic text materialization. Extend its prompt input with normalized structural hints. | Provider-boundary tests for natural boundaries, Unicode, caps, invalid JSON, incomplete coverage, wrong model identity, response limits, and retry replay. | +| SSC.3 | Complete 2026-08-13 | Durable generation | Persist a compact immutable semantic generation receipt and complete node generation transactionally in bounded batches; add PostgreSQL/TiDB migration and schema/index guards. | Repository, migration replay, all-excluded, conflicting replay, batch, identity, and size-bound tests. | +| SSC.4 | Complete 2026-08-13 | Compiler integration | Freeze the reasoning profile at admission, run recomposition and segmentation before projections, resume safely from checkpoints, and keep the published generation readable until the candidate is complete. | Worker success/failure/resume, deletion fence, profile migration, publication CAS, and no-legacy-fallback tests. | +| SSC.5 | Complete 2026-08-13 | Unified derived artifacts | Build outline hierarchy/summaries, PageIndex, dense/FTS projections, and Graph facts from final semantic nodes. Joint facts are quality-controlled and replayed for embedding-only migrations without another LLM call. | Exact-generation outline/path/Graph tests, duplicate prevention, source-node provenance, and profile-migration regression fixtures. | +| SSC.6 | Complete 2026-08-13 | Product diagnostics | Preserve compilation stages and semantic provenance; expose actionable document failure text on status hover while keeping technical trace ids secondary. The document outline now consumes semantic-node section paths and summaries. | Existing Admin hover/component coverage plus API worker/outline provenance tests. | +| SSC.7 | Local automation complete; production execution pending | Rollout and cleanup | The semantic compiler and receipt are versioned and fail closed. Static migration evidence, read-only preflight, explicit-document canary, bounded backfill, task polling, outline verification, retrieval probing, and revision rollback are available through guarded operator commands. Rebuild existing documents through normal reindex/profile-migration candidate publication. Do not delete the legacy implementation until production comparison is accepted. | Full checks, contract lock, rollout-script tests, deployment smoke, sampled shadow comparison, reindex/rebuild idempotency, rollback, and mixed-version reads. | + +## 4.1 Execution checkpoint and rollout order + +The code path is implemented in dependency order. The remaining work is operational verification, +not an untracked compiler shortcut: + +1. **Final local gates (complete 2026-08-13)** — formatting, typecheck, focused and full package + tests, CI coverage, database migration registry, OpenAPI/contract lock, and build are complete. + The unrelated repository-wide lint baseline and informational API branch-coverage gap are + recorded in the change note. +2. **Deploy schema first** — apply `0043_semantic_generation_receipts` to PostgreSQL/TiDB before + workers that can persist semantic receipts are started. +3. **Deploy API and workers together** — the runtime fails startup when semantic compilation is + configured without the reasoning provider or synchronous Graph materializer. This prevents a + mixed deployment from silently reverting to fixed-size chunking. +4. **Canary new imports** — compare chunk coherence, outline localization, Graph provenance, + provider calls, latency, and retrieval recall on synthetic structured documents and redacted + operator samples. +5. **Rebuild existing documents** — use the normal candidate reindex/profile-migration path. + Embedding-only changes clone the immutable semantic node generation; reasoning changes build a + new semantic generation. Publication remains an atomic head CAS. +6. **Rollback** — keep the previous published projection set readable until the candidate passes + evaluation. Roll back the publication head; never mutate or partially append to the old node + generation. +7. **Legacy cleanup** — remove deterministic final chunking only after production canaries, + existing-document rebuilds, and rollback drills pass. Until then it remains readable for old + generations but is not a fallback for newly admitted semantic profiles. + +### Guarded rollout commands + +Every mutating command requires both `SEMANTIC_ROLLOUT_APPLY=1` and an exact, space-scoped +confirmation string. Responses and polling are bounded. Tokens are read from the environment and +are never printed. + +```bash +# Repository/migration-registry evidence only; no network or mutation. +pnpm --dir knowledge-fs semantic:rollout:static + +# Read-only health, settings, document and failed-reindex baseline. +SEMANTIC_ROLLOUT_SPACE_ID= \ +SEMANTIC_ROLLOUT_API_BASE= \ +SEMANTIC_ROLLOUT_AUTH_TOKEN= \ +pnpm --dir knowledge-fs semantic:rollout:preflight + +# Explicit-document canary. Add SEMANTIC_ROLLOUT_QUERY for a Research retrieval assertion. +SEMANTIC_ROLLOUT_SPACE_ID= \ +SEMANTIC_ROLLOUT_DOCUMENT_IDS=[,...] \ +SEMANTIC_ROLLOUT_APPLY=1 \ +SEMANTIC_ROLLOUT_CONFIRM=semantic:canary: \ +pnpm --dir knowledge-fs semantic:rollout:canary + +# Whole-space bounded backfill through the existing bulk-reindex/candidate publication path. +SEMANTIC_ROLLOUT_SPACE_ID= \ +SEMANTIC_ROLLOUT_APPLY=1 \ +SEMANTIC_ROLLOUT_CONFIRM=semantic:backfill: \ +pnpm --dir knowledge-fs semantic:rollout:backfill + +# Roll a logical document back to a known prior immutable revision and verify activation. +SEMANTIC_ROLLOUT_SPACE_ID= \ +SEMANTIC_ROLLOUT_ROLLBACK_DOCUMENT_ID= \ +SEMANTIC_ROLLOUT_ROLLBACK_REVISION= \ +SEMANTIC_ROLLOUT_APPLY=1 \ +SEMANTIC_ROLLOUT_CONFIRM=semantic:rollback: \ +pnpm --dir knowledge-fs semantic:rollout:rollback +``` + +Migration 0043 still must be applied by the deployment system before API/workers are rolled out; +the operator script verifies checked-in evidence but intentionally does not receive database +credentials or execute production DDL. + +## 5. Redacted fixture matrix + +The real customer invoice must never enter the repository. A generated fixture with equivalent +geometry and synthetic names/identifiers covers: + +- one-page Chinese invoice/form with vertical labels and a line-item table; +- legitimate native headings versus low-confidence PDF title classifications; +- two-column reading order and cross-page continuation; +- a table larger than one model window; +- scanned/OCR text with repeated page noise; +- an empty document and a single over-limit atomic element; +- long CJK/emoji grapheme boundaries; +- model output with gaps, overlap, reordered ids, invented ids, and altered text attempts. + +## 6. Acceptance thresholds + +- Eligible source coverage: exactly 100%. +- Unproven generated source text: 0 bytes. +- Duplicate leaf coverage: 0. +- Orphan one-character CJK layout fragments in the invoice fixture: 0. +- Same immutable request retry: no new provider call after a complete receipt exists. +- Same generation retry: byte-identical node ids/text/offsets/ACL/provenance. +- Graph entity/relation rows: generation-scoped, source-linked, and duplicate-free. +- Failed candidate publication: previous published generation remains queryable. +- Every provider request/response, node count, window count, receipt size, and SQL batch is bounded. +- Every CI-enforced coverage package remains at or above its repository threshold. The semantic + compilation change set is at 90.01% branch coverage. The API package is currently excluded from + the repository CI coverage task; its informational full-suite historical baseline is now 89.48% + and must reach 90% before legacy compiler cleanup. + +## 7. Verification cadence + +Each slice follows RED -> GREEN -> REFACTOR and records its result under `.harness/changes`. +Targeted package tests run after each behavior change. Before completion run: + +```text +pnpm --dir knowledge-fs check +pnpm --dir knowledge-fs build +pnpm --dir knowledge-fs lint +pnpm --dir knowledge-fs db:migrations:check +``` + +The monorepo KnowledgeFS contract lock is regenerated only after all tracked KnowledgeFS changes +and generated artifacts are final and staged. Any skipped gate and its reason must be recorded in +the change summary. diff --git a/knowledge-fs/apps/api/src/document-compilation-runtime-options.test.ts b/knowledge-fs/apps/api/src/document-compilation-runtime-options.test.ts index 00a8b7275e9..b3636d09321 100644 --- a/knowledge-fs/apps/api/src/document-compilation-runtime-options.test.ts +++ b/knowledge-fs/apps/api/src/document-compilation-runtime-options.test.ts @@ -60,6 +60,18 @@ describe("createApiDocumentCompilationRuntime", () => { parser: {} as never, repositories: {}, }), + ).toThrow("requires the Reasoning-model semantic chunker"); + + expect(() => + createApiDocumentCompilationRuntime({ + adapter, + compute: {} as never, + config, + embeddingResolver: undefined, + parser: {} as never, + repositories: {}, + semanticChunker: {} as never, + }), ).toThrow("requires the per-space plugin embedding resolver"); expect(() => @@ -71,6 +83,7 @@ describe("createApiDocumentCompilationRuntime", () => { modelCapabilityPreflight: {} as never, parser: {} as never, repositories: {}, + semanticChunker: {} as never, }), ).toThrow("requires the atomic initial profile activation repository"); @@ -83,6 +96,7 @@ describe("createApiDocumentCompilationRuntime", () => { initialProfileActivations: {} as never, parser: {} as never, repositories: {}, + semanticChunker: {} as never, }), ).toThrow("requires model capability preflight"); @@ -96,6 +110,7 @@ describe("createApiDocumentCompilationRuntime", () => { modelCapabilityPreflight: {} as never, parser: {} as never, repositories: {}, + semanticChunker: {} as never, }), ).toThrow("requires database repository: artifacts"); }); @@ -122,6 +137,7 @@ describe("createApiDocumentCompilationRuntime", () => { assets: required(gateway.documentAssets), attempts: required(databaseRepositories.documentCompilationAttempts), chunks: required(gateway.documentChunks), + graph: required(gateway.graphIndex), legacyBootstraps: required(databaseRepositories.legacySpacePublicationBootstraps), pageIndexUpgradeBackfills: required(databaseRepositories.pageIndexUpgradeBackfills), logicalDocuments: required(gateway.logicalDocuments), @@ -137,6 +153,7 @@ describe("createApiDocumentCompilationRuntime", () => { settings: required(gateway.documentSettings), tasks: required(gateway.documentProcessingTasks), }, + semanticChunker: {} as never, }); expect(assembly).toMatchObject({ diff --git a/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts b/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts index 9db6c95fee3..4ed49a54671 100644 --- a/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts +++ b/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts @@ -47,6 +47,7 @@ import { type ParseArtifactRepository, type ProjectionSetPublicationMemberRepository, type ProjectionSetPublicationRepository, + type SemanticChunker, createDatabaseDocumentCompilationCandidateValidator, createDatabaseDocumentCompilationIndexOverrideResolver, createDatabaseDocumentLogicalMutationReconciler, @@ -72,6 +73,7 @@ import { createDurableDocumentCompilationJobStateMachine, createFtsProjectionBuilder, createIncrementalReindexer, + createJointSemanticGraphMaterializer, createKnowledgeSpaceProfileMigrationRuntime, createLegacySpacePublicationBootstrapRuntime, createLegacySpacePublicationBootstrapService, @@ -173,6 +175,7 @@ export interface CreateApiDocumentCompilationRuntimeOptions { }) | undefined; readonly semanticMetrics?: DocumentSemanticEnrichmentOperationalMetrics | undefined; + readonly semanticChunker?: SemanticChunker | undefined; readonly visual?: | { readonly model: string; @@ -253,6 +256,7 @@ export function createApiDocumentCompilationRuntime({ repositories: partialRepositories, semantic, semanticMetrics, + semanticChunker, visual, }: CreateApiDocumentCompilationRuntimeOptions): ApiDocumentCompilationRuntimeAssembly | undefined { if (!config) { @@ -261,6 +265,9 @@ export function createApiDocumentCompilationRuntime({ if (!compute) { throw new Error("Document compilation runtime requires an in-process compute runtime"); } + if (!semanticChunker) { + throw new Error("Document compilation runtime requires the Reasoning-model semantic chunker"); + } if (!embeddingResolver) { throw new Error( "Document compilation runtime requires the per-space plugin embedding resolver", @@ -363,7 +370,7 @@ export function createApiDocumentCompilationRuntime({ }, publications: repositories.publications, versions: { - chunkerVersion: "knowledge-compute-chunker-v1", + chunkerVersion: "knowledge-llm-semantic-chunker-v1", indexVersion: "knowledge-index-v1", nodeSchemaVersion: 1, parserPolicyVersion: "configured-parser-v1", @@ -402,6 +409,7 @@ export function createApiDocumentCompilationRuntime({ maxProjectionBatchSize: embeddingBatchSize, nodes: repositories.nodes, projections: repositories.projections, + semanticChunker, ...(visual ? { visualBuilder: createVisualEmbeddingProjectionBuilder({ @@ -417,6 +425,19 @@ export function createApiDocumentCompilationRuntime({ maxNodes: maxDocumentNodes, maxSummaryChars: 2_000, }); + const jointSemanticGraph = + semanticChunker && repositories.graph + ? createJointSemanticGraphMaterializer({ + graph: repositories.graph, + maxEntitiesPerNode: semantic?.semanticEntityExtractionMaxEntitiesPerNode ?? 50, + maxNodesPerArtifact: maxDocumentNodes, + maxRelationsPerNode: semantic?.semanticRelationExtractionMaxRelationsPerNode ?? 50, + nodes: repositories.nodes, + }) + : undefined; + if (semanticChunker && !jointSemanticGraph) { + throw new Error("Semantic document compilation requires the graph repository"); + } if (profileMigration && !outlineSummaryEnhancer) { throw new Error( "Profile migration runtime requires the profile-aware PageIndex Summary enhancer", @@ -441,12 +462,14 @@ export function createApiDocumentCompilationRuntime({ outlineSummaryEnhancer, outlines: repositories.outlines, pageIndexBuild, + paths: repositories.paths, profiles: repositories.profiles, projections: { getMany: repositories.projections.getMany.bind(repositories.projections), }, publications: repositories.publications, reindexer, + ...(jointSemanticGraph ? { semanticGraph: jointSemanticGraph } : {}), snapshots: createDatabaseKnowledgeSpaceProfileMigrationCandidateSnapshotRepository({ database: adapter.database, maxMembers: maxCandidateComponents, @@ -516,6 +539,7 @@ export function createApiDocumentCompilationRuntime({ failureManagement: "caller", generateKnowledgePathId: randomUUID, jobs, + ...(jointSemanticGraph ? { jointSemanticGraph } : {}), indexOverrides: documentIndexOverrides, knowledgePaths: repositories.paths, ...(multimodal?.documentMultimodalImageVariantGenerator diff --git a/knowledge-fs/apps/api/src/index.ts b/knowledge-fs/apps/api/src/index.ts index b1b68a26269..701daadd2b2 100644 --- a/knowledge-fs/apps/api/src/index.ts +++ b/knowledge-fs/apps/api/src/index.ts @@ -21,6 +21,7 @@ import { createKnowledgeSpaceOutlineSummaryEnhancer, createLlmAnswerQueryGenerator, createLlmAutoRetrievalModeResolver, + createLlmSemanticChunker, createModelCapabilityPreflight, createPageIndexFindabilityEvaluator, createPageIndexLayeredTreeSearch, @@ -433,6 +434,10 @@ const documentOutlineSummaryEnhancer = createKnowledgeSpaceOutlineSummaryEnhance modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate, providerFactory: profileReasoningCapability.providerFactory, }); +const documentSemanticChunker = createLlmSemanticChunker({ + maxNodes: 20_000, + reasoningProviderFactory: profileReasoningCapability.providerFactory, +}); const relevanceTriageOptions = createApiRelevanceTriageOptions({ ...(repositoryOptions.documentAssets ? { documentAssets: repositoryOptions.documentAssets } : {}), ...(repositoryOptions.documentOutlines @@ -590,6 +595,7 @@ const documentCompilationRuntime = createApiDocumentCompilationRuntime({ semanticExtractionMaxConcurrency: ingestionModelRuntimeOptions.semanticExtractionMaxConcurrency, }, semanticMetrics: operationalMetrics.semanticEnrichment, + semanticChunker: documentSemanticChunker, ...(visualEmbeddingOptions ? { visual: { diff --git a/knowledge-fs/apps/api/src/migrate.test.ts b/knowledge-fs/apps/api/src/migrate.test.ts index 4d90186fcbb..4aab2102913 100644 --- a/knowledge-fs/apps/api/src/migrate.test.ts +++ b/knowledge-fs/apps/api/src/migrate.test.ts @@ -133,6 +133,8 @@ describe("runApiDatabaseMigrations", () => { "insert", "schema", "insert", + "schema", + "insert", ]); expect(migrationSql).toHaveLength(expectedPostgresMigrationIds.length); expect(migrationSql[2]).toContain("-- Migration id: 0003_projection_set_publications\n"); @@ -192,6 +194,10 @@ describe("runApiDatabaseMigrations", () => { expect(migrationSql[39]).toContain("-- Migration id: 0040_knowledge_space_metadata\n"); expect(migrationSql[40]).toContain("-- Migration id: 0041_logical_document_availability\n"); expect(migrationSql[41]).toContain("-- Migration id: 0042_workflow_failed_retrieval_capture\n"); + expect(migrationSql[42]).toContain("-- Migration id: 0043_semantic_generation_receipts\n"); + expect(migrationSql[42]).toContain( + 'CREATE TABLE IF NOT EXISTS "knowledge_node_generation_receipts"', + ); expect(closed).toBe(true); }); diff --git a/knowledge-fs/package.json b/knowledge-fs/package.json index 2f2e63f046f..e5110cadac1 100644 --- a/knowledge-fs/package.json +++ b/knowledge-fs/package.json @@ -12,7 +12,7 @@ "scripts": { "build": "turbo run build", "capability:export": "node --import tsx scripts/export-capability-v2-operations.mjs", - "check": "pnpm typecheck && pnpm test && pnpm openapi:export:test && pnpm test:coverage:ci && pnpm eval:regression && pnpm eval:phase4 && pnpm swagger:test && pnpm db:migrations:check && pnpm p9:bundle:test && pnpm ci:workflow:test && pnpm compose:middleware:config && pnpm compose:config && pnpm dify:compose:config && pnpm compose:middleware:test && pnpm compose:apps:test && pnpm docker:context:test && pnpm local:happy-path:test && pnpm docker:api:bundle-smoke:test && pnpm docker:admin:http-smoke:test && pnpm docker:apps:smoke:test", + "check": "pnpm typecheck && pnpm test && pnpm openapi:export:test && pnpm test:coverage:ci && pnpm eval:regression && pnpm eval:phase4 && pnpm swagger:test && pnpm db:migrations:check && pnpm p9:bundle:test && pnpm ci:workflow:test && pnpm compose:middleware:config && pnpm compose:config && pnpm dify:compose:config && pnpm compose:middleware:test && pnpm compose:apps:test && pnpm docker:context:test && pnpm local:happy-path:test && pnpm semantic:rollout:test && pnpm docker:api:bundle-smoke:test && pnpm docker:admin:http-smoke:test && pnpm docker:apps:smoke:test", "ci:workflow:test": "node --test scripts/github-actions-workflow.test.mjs scripts/secret-scan.test.mjs", "compose:config": "docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml config", "compose:apps:test": "node --test scripts/compose-apps.test.mjs", @@ -49,6 +49,12 @@ "p9:bundle:test": "node --test scripts/build-p9-removal-bundle.test.mjs", "security:dependencies": "node scripts/audit-backend-dependencies.mjs", "security:secrets": "node scripts/secret-scan.mjs", + "semantic:rollout:backfill": "SEMANTIC_ROLLOUT_MODE=backfill node scripts/semantic-compilation-rollout.mjs", + "semantic:rollout:canary": "SEMANTIC_ROLLOUT_MODE=canary node scripts/semantic-compilation-rollout.mjs", + "semantic:rollout:preflight": "SEMANTIC_ROLLOUT_MODE=preflight node scripts/semantic-compilation-rollout.mjs", + "semantic:rollout:rollback": "SEMANTIC_ROLLOUT_MODE=rollback node scripts/semantic-compilation-rollout.mjs", + "semantic:rollout:static": "node scripts/semantic-compilation-rollout.mjs", + "semantic:rollout:test": "node --test scripts/semantic-compilation-rollout.test.mjs", "swagger": "node tools/swagger/server.mjs", "swagger:test": "node --test tools/swagger/server.test.mjs", "test": "turbo run test", diff --git a/knowledge-fs/packages/adapters/src/migration-runner.test.ts b/knowledge-fs/packages/adapters/src/migration-runner.test.ts index 7fbd3819e77..e6709397b14 100644 --- a/knowledge-fs/packages/adapters/src/migration-runner.test.ts +++ b/knowledge-fs/packages/adapters/src/migration-runner.test.ts @@ -50,6 +50,7 @@ const documentSemanticEnrichmentMigrationId = "0039_document_semantic_enrichment const knowledgeSpaceMetadataMigrationId = "0040_knowledge_space_metadata"; const logicalDocumentAvailabilityMigrationId = "0041_logical_document_availability"; const workflowFailedRetrievalCaptureMigrationId = "0042_workflow_failed_retrieval_capture"; +const semanticGenerationReceiptsMigrationId = "0043_semantic_generation_receipts"; const migrationsAfterDurableDeletion = [ versionedSpaceProfilesMigrationId, profilePublicationBindingsMigrationId, @@ -76,6 +77,7 @@ const migrationsAfterDurableDeletion = [ knowledgeSpaceMetadataMigrationId, logicalDocumentAvailabilityMigrationId, workflowFailedRetrievalCaptureMigrationId, + semanticGenerationReceiptsMigrationId, ] as const; const migrationsAfterTidbBaselineRepair = [ spaceAccessControlMigrationId, diff --git a/knowledge-fs/packages/api/package.json b/knowledge-fs/packages/api/package.json index 8fb73304d78..ae993200ae6 100644 --- a/knowledge-fs/packages/api/package.json +++ b/knowledge-fs/packages/api/package.json @@ -21,6 +21,7 @@ "hono": "^4.12.25", "jose": "^5.10.0", "sharp": "^0.35.3", + "unicode-segmenter": "0.15.0", "zod": "^3.24.1" }, "devDependencies": { diff --git a/knowledge-fs/packages/api/src/document-compilation-worker.test.ts b/knowledge-fs/packages/api/src/document-compilation-worker.test.ts index 95fc9ccadf1..189121a9932 100644 --- a/knowledge-fs/packages/api/src/document-compilation-worker.test.ts +++ b/knowledge-fs/packages/api/src/document-compilation-worker.test.ts @@ -371,6 +371,12 @@ describe("createDocumentCompilationWorker lease integration", () => { const semanticAdmissions: unknown[] = []; let semanticCalls = 0; let smokeCalls = 0; + const outlines = createInMemoryDocumentOutlineRepository({ maxOutlines: 2 }); + const knowledgePaths = createInMemoryKnowledgePathRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxPaths: 10, + }); const worker = createDocumentCompilationWorker({ assets, candidateComposer: { @@ -412,13 +418,12 @@ describe("createDocumentCompilationWorker lease integration", () => { "018f0d60-7a49-7cc2-9c1b-5b36f18f6a36", "018f0d60-7a49-7cc2-9c1b-5b36f18f6a37", "018f0d60-7a49-7cc2-9c1b-5b36f18f6a38", + "018f0d60-7a49-7cc2-9c1b-5b36f18f6a3b", + "018f0d60-7a49-7cc2-9c1b-5b36f18f6a3c", + "018f0d60-7a49-7cc2-9c1b-5b36f18f6a3d", ]), jobs: compilationJobs, - knowledgePaths: createInMemoryKnowledgePathRepository({ - maxBatchSize: 10, - maxListLimit: 10, - maxPaths: 10, - }), + knowledgePaths, multimodalManifests: createInMemoryDocumentMultimodalManifestRepository({ maxManifests: 2, }), @@ -428,7 +433,7 @@ describe("createDocumentCompilationWorker lease integration", () => { maxNodes: 10, maxSummaryChars: 200, }), - outlines: createInMemoryDocumentOutlineRepository({ maxOutlines: 2 }), + outlines, pageIndexBuild: { materializeBuilding: async ({ outline }) => { pageIndexBuildCalls += 1; @@ -460,6 +465,19 @@ describe("createDocumentCompilationWorker lease integration", () => { artifact: input.parseArtifact, nodeIds: ["018f0d60-7a49-7cc2-9c1b-5b36f18f6a39"], nodesCreated: 1, + outlineArtifact: ParseArtifactSchema.parse({ + ...input.parseArtifact, + elements: [ + { + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a39", + metadata: { semanticSectionSummary: "发票身份、购买方和金额信息。" }, + sectionPath: ["电子发票", "购买方与金额"], + text: "发票号码、购买方与价税合计", + type: "paragraph", + }, + ], + metadata: { semanticCompilation: { source: "llm-semantic-v1" } }, + }), projectionIds: ["018f0d60-7a49-7cc2-9c1b-5b36f18f6a3a"], projectionsCreated: 1, status: "rebuilt", @@ -471,6 +489,21 @@ describe("createDocumentCompilationWorker lease integration", () => { semanticAdmissions.push(input); }, }, + jointSemanticGraph: { + materialize: async () => { + semanticCalls += 1; + return { + entitiesExtracted: 1, + graphEntityIds: ["018f0d60-7a49-7cc2-9c1b-5b36f18f6a3b"], + graphEntitiesIndexed: 1, + graphRelationIds: ["018f0d60-7a49-7cc2-9c1b-5b36f18f6a3c"], + graphRelationsIndexed: 1, + nodesScanned: 1, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + }; + }, + }, semanticPostProcessor: { process: async () => { semanticCalls += 1; @@ -509,8 +542,18 @@ describe("createDocumentCompilationWorker lease integration", () => { expect.objectContaining({ componentReceipt: { documentOutlines: [expect.objectContaining({ generationId })], - graphEntities: [], - graphRelations: [], + graphEntities: [ + { + componentKey: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a3b", + generationId, + }, + ], + graphRelations: [ + { + componentKey: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a3c", + generationId, + }, + ], indexProjections: [ { componentKey: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a3a", @@ -528,18 +571,33 @@ describe("createDocumentCompilationWorker lease integration", () => { expect(smokeCalls).toBe(0); expect(mutableEmbeddingReads).toBe(0); expect(pageIndexBuildCalls).toBe(0); - expect(semanticAdmissions).toEqual([ + expect(semanticAdmissions).toEqual([]); + expect(semanticCalls).toBe(1); + expect(reindexInputs[0]).toEqual( expect.objectContaining({ - documentAssetId: asset.id, - parseArtifactId: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a02", - publicationGenerationId: generationId, + enableGraph: true, + language: "zh-CN", retrievalProfile: expect.objectContaining({ revision: 4 }), + skipDense: true, }), - ]); - expect(semanticCalls).toBe(0); - expect(reindexInputs[0]).toEqual(expect.objectContaining({ language: "zh-CN" })); + ); expect(reindexInputs[0]).not.toHaveProperty("denseModel"); expect(reindexInputs[0]).not.toHaveProperty("embeddingProfile"); + await expect( + outlines.getByDocumentVersion({ + documentAssetId: asset.id, + publicationGenerationId: generationId, + version: asset.version, + }), + ).resolves.toMatchObject({ + metadata: expect.objectContaining({ builder: "semantic-knowledge-nodes" }), + nodes: [ + expect.objectContaining({ + sectionPath: ["电子发票"], + children: [expect.objectContaining({ sectionPath: ["电子发票", "购买方与金额"] })], + }), + ], + }); await expect( assets.get({ id: asset.id, knowledgeSpaceId: asset.knowledgeSpaceId }), ).resolves.toMatchObject({ parserStatus: "pending" }); @@ -1342,7 +1400,9 @@ describe("createDocumentCompilationWorker lease integration", () => { expect.objectContaining({ denseModel: frozenEmbeddingProfile.vectorSpaceId, embeddingProfile: frozenEmbeddingProfile, + enableGraph: true, projectionStatus: "building", + retrievalProfile: frozenRetrievalProfile, tenantId: "tenant-1", }), ]); diff --git a/knowledge-fs/packages/api/src/document-compilation-worker.ts b/knowledge-fs/packages/api/src/document-compilation-worker.ts index 842aa03d25c..5b2ede20637 100644 --- a/knowledge-fs/packages/api/src/document-compilation-worker.ts +++ b/knowledge-fs/packages/api/src/document-compilation-worker.ts @@ -49,6 +49,7 @@ import { type DocumentPdfRasterizer, rasterizeDocumentPdfMultimodalAssets, } from "./document-pdf-rasterizer"; +import type { JointSemanticGraphMaterializer } from "./document-semantic-enrichment-processor"; import { logDocumentUploadDiagnostic } from "./document-upload-diagnostics"; import type { IncrementalReindexer } from "./index-reindexer"; import type { KnowledgeFsOperationLeaseCoordinator } from "./knowledge-fs-operation-leases"; @@ -89,6 +90,7 @@ export interface DocumentCompilationWorkerOptions { readonly failureManagement?: "caller" | "worker" | undefined; readonly generateKnowledgePathId?: (() => string) | undefined; readonly jobs: DocumentCompilationJobStateMachine; + readonly jointSemanticGraph?: JointSemanticGraphMaterializer | undefined; readonly knowledgePaths?: KnowledgePathRepository | undefined; readonly multimodalImageVariantGenerator?: DocumentImageVariantGenerator | undefined; readonly multimodalLocalAssetAllowlist?: readonly string[] | undefined; @@ -220,6 +222,7 @@ export function createDocumentCompilationWorker({ failureManagement = "worker", generateKnowledgePathId, jobs, + jointSemanticGraph, knowledgePaths, multimodalImageVariantGenerator, multimodalLocalAssetAllowlist, @@ -418,6 +421,9 @@ export function createDocumentCompilationWorker({ let documentOutlineIds: readonly string[] = []; let knowledgePathIds: readonly string[] = []; let persistedManifest: DocumentMultimodalManifest; + const deferOutlineUntilSemanticNodes = Boolean( + publicationGenerationId && frozenRetrievalProfile, + ); if (resumeOutlineGeneration && publicationGenerationId) { const [persistedOutline, resumedManifest] = await Promise.all([ outlines?.getByDocumentVersion({ @@ -461,7 +467,7 @@ export function createDocumentCompilationWorker({ knowledgeSpaceId: input.knowledgeSpaceId, ...(publicationGenerationId ? { publicationGenerationId } : {}), }); - if (outlineBuilder && outlines) { + if (outlineBuilder && outlines && !deferOutlineUntilSemanticNodes) { const deterministicOutline = outlineBuilder.build({ knowledgeSpaceId: input.knowledgeSpaceId, parseArtifact: canonicalArtifact, @@ -504,8 +510,10 @@ export function createDocumentCompilationWorker({ } await assertWritable(); persistedManifest = await multimodalManifests.upsert(multimodalManifest); - await assertWritable(); - await jobs.advance(input.documentCompilationJobId, "outline_built"); + if (!deferOutlineUntilSemanticNodes) { + await assertWritable(); + await jobs.advance(input.documentCompilationJobId, "outline_built"); + } } const resolvedEmbedding = frozenEmbeddingProfile @@ -531,6 +539,7 @@ export function createDocumentCompilationWorker({ ? { excludedNodeOrdinals: documentIndexOverrides.excludedNodeOrdinals } : {}), ...(frozenEmbeddingProfile ? { embeddingProfile: frozenEmbeddingProfile } : {}), + enableGraph: documentIndexOverrides.enableGraph !== false, knowledgeSpaceId: input.knowledgeSpaceId, ...(documentIndexOverrides.language ? { language: documentIndexOverrides.language } @@ -541,11 +550,70 @@ export function createDocumentCompilationWorker({ publicationGenerationId || legacyStagedProjectionPublication ? "building" : "ready", projectionVersion: input.version, ...(publicationGenerationId ? { publicationGenerationId } : {}), + ...(frozenRetrievalProfile ? { retrievalProfile: frozenRetrievalProfile } : {}), ...(initialJob.stage === "outline_built" ? { resetFailedProjections: true } : {}), ...(signal ? { signal } : {}), + ...(frozenRetrievalProfile && !resolvedEmbedding ? { skipDense: true as const } : {}), tenantId: input.tenantId, ...(visualEmbeddingModel ? { visualModel: visualEmbeddingModel } : {}), }); + if ( + deferOutlineUntilSemanticNodes && + !resumeOutlineGeneration && + publicationGenerationId + ) { + if ( + reindexResult.status !== "rebuilt" || + !reindexResult.outlineArtifact || + !outlineBuilder || + !outlines + ) { + throw new Error( + "Generation-scoped semantic compilation requires a semantic outline artifact", + ); + } + const deterministicOutline = outlineBuilder.build({ + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifact: reindexResult.outlineArtifact, + publicationGenerationId, + }); + const outline = outlineSummaryEnhancer + ? await outlineSummaryEnhancer.enhance({ + outline: deterministicOutline, + parseArtifact: reindexResult.outlineArtifact, + retrievalProfile: frozenRetrievalProfile, + ...(signal ? { signal } : {}), + tenantId: input.tenantId, + }) + : deterministicOutline; + await assertWritable(); + const persistedOutline = await outlines.upsert(outline); + if (documentIndexOverrides.enablePageIndex !== false) { + await assertWritable(); + await pageIndexBuild?.materializeBuilding({ + builtAt: persistedOutline.updatedAt ?? persistedOutline.createdAt, + outline: persistedOutline, + tenantId: input.tenantId, + }); + } + documentOutlineIds = [persistedOutline.id]; + if (knowledgePaths && generateKnowledgePathId) { + await assertWritable(); + const persistedPaths = await knowledgePaths.upsertMany( + buildCompilationKnowledgePaths({ + asset: activeAsset, + generateId: generateKnowledgePathId, + manifest: persistedManifest, + outline: persistedOutline, + publicationGenerationId, + tenantId: input.tenantId, + }), + ); + knowledgePathIds = persistedPaths.map((path) => path.id); + } + await assertWritable(); + await jobs.advance(input.documentCompilationJobId, "outline_built"); + } await assertWritable(); if (legacyStagedProjectionPublication && reindexResult.status === "rebuilt") { stagedProjectionIds = [...(reindexResult.projectionIds ?? [])]; @@ -580,8 +648,28 @@ export function createDocumentCompilationWorker({ await jobs.advance(input.documentCompilationJobId, "nodes_generated"); let graphEntityIds: readonly string[] = []; let graphRelationIds: readonly string[] = []; + if ( + jointSemanticGraph && + publicationGenerationId && + frozenRetrievalProfile && + reindexResult.status === "rebuilt" && + documentIndexOverrides.enableGraph !== false + ) { + await assertWritable(); + const semanticResult = await jointSemanticGraph.materialize({ + createdAt: activeAsset.updatedAt ?? activeAsset.createdAt, + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifactId: canonicalArtifact.id, + publicationGenerationId, + retrievalProfile: frozenRetrievalProfile, + }); + await assertWritable(); + graphEntityIds = semanticResult.graphEntityIds; + graphRelationIds = semanticResult.graphRelationIds; + } if ( semanticEnrichmentAdmission && + !jointSemanticGraph && publicationGenerationId && frozenRetrievalProfile && reindexResult.status === "rebuilt" && diff --git a/knowledge-fs/packages/api/src/document-layout-recomposer.test.ts b/knowledge-fs/packages/api/src/document-layout-recomposer.test.ts new file mode 100644 index 00000000000..fb71d2009a0 --- /dev/null +++ b/knowledge-fs/packages/api/src/document-layout-recomposer.test.ts @@ -0,0 +1,172 @@ +import { type ParseArtifact, ParseArtifactSchema } from "@knowledge/core"; +import { describe, expect, it } from "vitest"; + +import { recomposeDocumentLayoutForSemanticSegmentation } from "./document-layout-recomposer"; + +describe("document layout recomposition for semantic segmentation", () => { + it("removes unproven PDF title boundaries while preserving every source element", () => { + const input = artifact({ + elements: [ + element("title", "电子发票（普通发票）", ["电子发票（普通发票）"], "title-1"), + element("paragraph", "发票号码：26322000000000000000", ["电子发票（普通发票）"], "p-1"), + element( + "title", + "名称：示例人工智能有限公司", + ["名称：示例人工智能有限公司"], + "false-title-1", + ), + element( + "paragraph", + "统一社会信用代码：91320506EXAMPLE01", + ["名称：示例人工智能有限公司"], + "p-2", + ), + element("title", "91320506EXAMPLE02", ["91320506EXAMPLE02"], "false-title-2"), + element("table", "餐饮服务 | 1 | 533.96 | 6% | 32.04", ["91320506EXAMPLE02"], "table-1"), + element("paragraph", "合计：566.00", ["91320506EXAMPLE02"], "p-3"), + ], + parser: "unstructured", + }); + + const result = recomposeDocumentLayoutForSemanticSegmentation(input); + + expect(result.artifact.elements.map(({ id, text, type }) => ({ id, text, type }))).toEqual( + input.elements.map(({ id, text, type }) => ({ id, text, type })), + ); + expect(result.artifact.elements.map((item) => item.sectionPath)).toEqual([ + [], + [], + [], + [], + [], + [], + [], + ]); + expect(result.artifact.elements[2]?.metadata.layoutRecomposition).toEqual({ + boundaryPolicy: "reasoning-model", + originalSectionPath: ["名称：示例人工智能有限公司"], + originalType: "title", + reason: "unstructured-heading-without-hierarchy-evidence", + schemaVersion: 1, + }); + expect(result.stats).toEqual({ + elementsRecomposed: 7, + modelDecidedHeadingBoundaries: 3, + trustedHeadingBoundaries: 0, + }); + expect(result.fingerprint).toMatch(/^sha256:[a-f0-9]{64}$/u); + }); + + it("preserves explicit Unstructured heading hierarchy and applies it to following content", () => { + const input = artifact({ + elements: [ + element("title", "第一章", ["第一章"], "chapter", { category_depth: 0 }), + element("heading", "范围", ["第一章", "范围"], "section", { + category_depth: 1, + parent_id: "chapter", + }), + element("paragraph", "适用范围正文。", ["第一章", "范围"], "paragraph"), + ], + parser: "unstructured", + }); + + const result = recomposeDocumentLayoutForSemanticSegmentation(input); + + expect(result.artifact.elements.map((item) => item.sectionPath)).toEqual([ + ["第一章"], + ["第一章", "范围"], + ["第一章", "范围"], + ]); + expect(result.stats).toEqual({ + elementsRecomposed: 3, + modelDecidedHeadingBoundaries: 0, + trustedHeadingBoundaries: 2, + }); + }); + + it("leaves native parser section boundaries unchanged", () => { + const input = artifact({ + elements: [ + element("heading", "安装", ["安装"], "heading"), + element("paragraph", "安装正文。", ["安装"], "paragraph"), + ], + parser: "native-markdown", + }); + + const result = recomposeDocumentLayoutForSemanticSegmentation(input); + + expect(result.artifact).toEqual(input); + expect(result.stats).toEqual({ + elementsRecomposed: 0, + modelDecidedHeadingBoundaries: 0, + trustedHeadingBoundaries: 0, + }); + }); + + it("rejects artifacts that exceed the bounded element count", () => { + const input = artifact({ + elements: [element("paragraph", "一", [], "one"), element("paragraph", "二", [], "two")], + parser: "unstructured", + }); + + expect(() => recomposeDocumentLayoutForSemanticSegmentation(input, { maxElements: 1 })).toThrow( + "Document layout recomposition exceeds maxElements=1", + ); + expect(() => recomposeDocumentLayoutForSemanticSegmentation(input, { maxElements: 0 })).toThrow( + "maxElements must be at least 1", + ); + }); + + it("trusts a non-empty parent id even when category depth is absent", () => { + const input = artifact({ + elements: [ + element("heading", "子章节", ["父章节", "子章节"], "child", { + parent_id: "parent", + }), + element("paragraph", "正文", ["父章节", "子章节"], "body"), + ], + parser: "unstructured", + }); + + const result = recomposeDocumentLayoutForSemanticSegmentation(input, { maxElements: 2 }); + expect(result.stats.trustedHeadingBoundaries).toBe(1); + expect(result.artifact.elements[1]?.sectionPath).toEqual(["父章节", "子章节"]); + }); +}); + +function artifact({ + elements, + parser, +}: { + readonly elements: ParseArtifact["elements"]; + readonly parser: ParseArtifact["parser"]; +}): ParseArtifact { + return ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "text", + createdAt: "2026-08-13T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42", + elements, + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + metadata: { parserVersion: `${parser}@test` }, + parser, + version: 1, + }); +} + +function element( + type: ParseArtifact["elements"][number]["type"], + text: string, + sectionPath: readonly string[], + id: string, + metadata: Record = {}, +): ParseArtifact["elements"][number] { + return { + id, + metadata, + pageNumber: 1, + sectionPath: [...sectionPath], + text, + type, + }; +} diff --git a/knowledge-fs/packages/api/src/document-layout-recomposer.ts b/knowledge-fs/packages/api/src/document-layout-recomposer.ts new file mode 100644 index 00000000000..323563f63c3 --- /dev/null +++ b/knowledge-fs/packages/api/src/document-layout-recomposer.ts @@ -0,0 +1,157 @@ +import { createHash } from "node:crypto"; + +import { type ParseArtifact, ParseArtifactSchema, stableJson } from "@knowledge/core"; + +const DEFAULT_MAX_ELEMENTS = 20_000; +const LAYOUT_RECOMPOSITION_SCHEMA_VERSION = 1; + +export interface DocumentLayoutRecompositionOptions { + readonly maxElements?: number | undefined; +} + +export interface DocumentLayoutRecompositionStats { + readonly elementsRecomposed: number; + readonly modelDecidedHeadingBoundaries: number; + readonly trustedHeadingBoundaries: number; +} + +export interface DocumentLayoutRecompositionResult { + readonly artifact: ParseArtifact; + readonly fingerprint: string; + readonly stats: DocumentLayoutRecompositionStats; +} + +/** + * Produces the bounded parser view consumed by semantic segmentation. + * + * Native Markdown/HTML headings are authored structure and remain authoritative. Unstructured + * `Title`/`Heading` labels are only hard boundaries when the provider supplied explicit hierarchy + * evidence (`category_depth` or `parent_id`). Other labels remain visible to the reasoning model as + * title/heading units, but cannot fragment its input merely because a layout classifier guessed + * that a company name, tax id, or form field was a heading. + * + * Element order, text, ids, pages, and source metadata are never rewritten. This keeps canonical + * byte offsets stable while making boundary confidence explicit and replayable. + */ +export function recomposeDocumentLayoutForSemanticSegmentation( + input: ParseArtifact, + { maxElements = DEFAULT_MAX_ELEMENTS }: DocumentLayoutRecompositionOptions = {}, +): DocumentLayoutRecompositionResult { + if (!Number.isSafeInteger(maxElements) || maxElements < 1) { + throw new Error("Document layout recomposition maxElements must be at least 1"); + } + + const artifact = ParseArtifactSchema.parse(input); + if (artifact.elements.length > maxElements) { + throw new Error(`Document layout recomposition exceeds maxElements=${maxElements}`); + } + + if (artifact.parser !== "unstructured") { + const unchanged = ParseArtifactSchema.parse(artifact); + return { + artifact: unchanged, + fingerprint: layoutRecompositionFingerprint(unchanged), + stats: { + elementsRecomposed: 0, + modelDecidedHeadingBoundaries: 0, + trustedHeadingBoundaries: 0, + }, + }; + } + + let currentTrustedPath: string[] = []; + let modelDecidedHeadingBoundaries = 0; + let trustedHeadingBoundaries = 0; + const elements = artifact.elements.map((element) => { + const heading = element.type === "title" || element.type === "heading"; + if (heading && hasExplicitHierarchyEvidence(element.metadata)) { + currentTrustedPath = [...element.sectionPath]; + trustedHeadingBoundaries += 1; + return { + ...element, + metadata: { ...element.metadata }, + sectionPath: [...currentTrustedPath], + }; + } + + if (heading) { + modelDecidedHeadingBoundaries += 1; + return { + ...element, + metadata: { + ...element.metadata, + layoutRecomposition: { + boundaryPolicy: "reasoning-model", + originalSectionPath: [...element.sectionPath], + originalType: element.type, + reason: "unstructured-heading-without-hierarchy-evidence", + schemaVersion: LAYOUT_RECOMPOSITION_SCHEMA_VERSION, + }, + }, + sectionPath: [...currentTrustedPath], + }; + } + + const sectionPathChanged = !sameStrings(element.sectionPath, currentTrustedPath); + return { + ...element, + metadata: { + ...element.metadata, + ...(sectionPathChanged + ? { + layoutRecomposition: { + boundaryPolicy: "reasoning-model", + originalSectionPath: [...element.sectionPath], + reason: "inherited-untrusted-heading-boundary", + schemaVersion: LAYOUT_RECOMPOSITION_SCHEMA_VERSION, + }, + } + : {}), + }, + sectionPath: [...currentTrustedPath], + }; + }); + const recomposed = ParseArtifactSchema.parse({ ...artifact, elements }); + + return { + artifact: recomposed, + fingerprint: layoutRecompositionFingerprint(recomposed), + stats: { + elementsRecomposed: recomposed.elements.length, + modelDecidedHeadingBoundaries, + trustedHeadingBoundaries, + }, + }; +} + +function hasExplicitHierarchyEvidence(metadata: Readonly>): boolean { + const categoryDepth = metadata.category_depth; + const parentId = metadata.parent_id; + return ( + (typeof categoryDepth === "number" && Number.isInteger(categoryDepth) && categoryDepth >= 0) || + (typeof parentId === "string" && parentId.trim().length > 0) + ); +} + +function layoutRecompositionFingerprint(artifact: ParseArtifact): string { + return `sha256:${createHash("sha256") + .update( + stableJson({ + artifactHash: artifact.artifactHash, + elements: artifact.elements.map((element) => ({ + id: element.id, + metadata: element.metadata, + pageNumber: element.pageNumber, + sectionPath: element.sectionPath, + text: element.text, + type: element.type, + })), + schemaVersion: LAYOUT_RECOMPOSITION_SCHEMA_VERSION, + }), + ) + .digest("hex")}`; +} + +function sameStrings(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} diff --git a/knowledge-fs/packages/api/src/document-outline-builder.test.ts b/knowledge-fs/packages/api/src/document-outline-builder.test.ts index 3991bc46a7a..355a6c95d94 100644 --- a/knowledge-fs/packages/api/src/document-outline-builder.test.ts +++ b/knowledge-fs/packages/api/src/document-outline-builder.test.ts @@ -97,6 +97,48 @@ describe("document outline builder", () => { expect(outline.nodes[0]?.children[0]?.summary).toContain("dense, full-text, and graph"); }); + it("preserves semantic-node lineage and model section summaries", () => { + const builder = createDocumentOutlineBuilder({ + generateId: sequenceIds([ + "018f0d60-7a49-7cc2-9c1b-5b36f18f2c50", + "018f0d60-7a49-7cc2-9c1b-5b36f18f2c51", + "018f0d60-7a49-7cc2-9c1b-5b36f18f2c52", + ]), + maxElements: 20, + maxNodes: 10, + maxSummaryChars: 120, + now: () => createdAt, + }); + const sourceNodeId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2d01"; + const semanticArtifact: ParseArtifact = { + ...parseArtifact([ + { + id: sourceNodeId, + metadata: { + semanticSectionSummary: "发票身份、购买方和价税合计。", + sourceKnowledgeNodeId: sourceNodeId, + }, + sectionPath: ["电子发票", "购买方与金额"], + text: "发票号码、购买方、价税合计和开票人", + type: "paragraph", + }, + ]), + metadata: { + parserVersion: "native-markdown@1", + semanticCompilation: { source: "llm-semantic-v1" }, + }, + }; + + const outline = builder.build({ knowledgeSpaceId, parseArtifact: semanticArtifact }); + + expect(outline.metadata).toMatchObject({ builder: "semantic-knowledge-nodes" }); + expect(outline.nodes[0]?.children[0]).toMatchObject({ + sectionPath: ["电子发票", "购买方与金额"], + sourceNodeIds: [sourceNodeId], + summary: "发票身份、购买方和价税合计。", + }); + }); + it("falls back to a document node when parse output has no heading structure", () => { const builder = createDocumentOutlineBuilder({ generateId: sequenceIds([ diff --git a/knowledge-fs/packages/api/src/document-outline-builder.ts b/knowledge-fs/packages/api/src/document-outline-builder.ts index c51868873d3..e22f8d4d628 100644 --- a/knowledge-fs/packages/api/src/document-outline-builder.ts +++ b/knowledge-fs/packages/api/src/document-outline-builder.ts @@ -156,7 +156,9 @@ export function createDocumentOutlineBuilder({ id: buildGenerateId(), knowledgeSpaceId, metadata: { - builder: "deterministic-parse-artifact", + builder: artifact.metadata.semanticCompilation + ? "semantic-knowledge-nodes" + : "deterministic-parse-artifact", contentType: artifact.contentType, parser: artifact.parser, parserVersion: artifact.metadata.parserVersion, @@ -373,8 +375,22 @@ function applySpanToDraft(draft: OutlineNodeDraft, span: ElementSpan): void { draft.sourceElementIds.push(span.element.id); } + const sourceKnowledgeNodeId = span.element.metadata.sourceKnowledgeNodeId; + if ( + typeof sourceKnowledgeNodeId === "string" && + sourceKnowledgeNodeId.trim() && + !draft.sourceNodeIds.includes(sourceKnowledgeNodeId) + ) { + draft.sourceNodeIds.push(sourceKnowledgeNodeId); + } + if (span.element.type !== "heading" && span.element.type !== "title") { - draft.summaryTexts.push(span.text); + const semanticSectionSummary = span.element.metadata.semanticSectionSummary; + draft.summaryTexts.push( + typeof semanticSectionSummary === "string" && semanticSectionSummary.trim() + ? semanticSectionSummary.trim() + : span.text, + ); } if (!draft.titleLocation && (span.element.type === "heading" || span.element.type === "title")) { diff --git a/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.test.ts b/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.test.ts index cd925af1549..43ed3765db3 100644 --- a/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.test.ts +++ b/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.test.ts @@ -1,13 +1,22 @@ -import { type KnowledgeNode, KnowledgeNodeSchema } from "@knowledge/core"; +import { + type KnowledgeNode, + KnowledgeNodeSchema, + type KnowledgeSpaceRetrievalProfile, + ParseArtifactSchema, +} from "@knowledge/core"; import { describe, expect, it } from "vitest"; -import { createDocumentSemanticEnrichmentProcessor } from "./document-semantic-enrichment-processor"; +import { + createDocumentSemanticEnrichmentProcessor, + createJointSemanticGraphMaterializer, +} from "./document-semantic-enrichment-processor"; import { createInMemoryDocumentSemanticEnrichmentRepository, createInMemoryDocumentSemanticExtractionCheckpointRepository, } from "./document-semantic-enrichment-repository"; import { createInMemoryGraphIndexRepository } from "./graph-index-repository"; import { createInMemoryKnowledgeNodeRepository } from "./knowledge-node-repository"; +import { createLlmSemanticChunker } from "./llm-semantic-chunker"; const tenantId = "tenant-1"; const knowledgeSpaceId = uuid(1); @@ -17,6 +26,162 @@ const publicationGenerationId = uuid(4); const createdAt = "2026-08-09T10:00:00.000Z"; describe("createDocumentSemanticEnrichmentProcessor", () => { + it("validates processor and joint materializer bounds", () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 1, + }); + const graph = createInMemoryGraphIndexRepository({ + maxBatchSize: 1, + maxEntities: 1, + maxRelations: 1, + }); + const baseProcessor = { + checkpoints: createInMemoryDocumentSemanticExtractionCheckpointRepository(), + graph, + maxConcurrentBatches: 1, + maxEntitiesPerNode: 1, + maxNodesPerArtifact: 1, + maxOutputTokens: 1, + maxRelationsPerNode: 1, + nodes, + providerBatchSize: 1, + providerFactory: () => semanticProvider(), + }; + for (const [name, override] of [ + ["maxConcurrentBatches", { maxConcurrentBatches: 0 }], + ["maxEntitiesPerNode", { maxEntitiesPerNode: 0 }], + ["maxNodesPerArtifact", { maxNodesPerArtifact: 0 }], + ["maxOutputTokens", { maxOutputTokens: 0 }], + ["maxRelationsPerNode", { maxRelationsPerNode: 0 }], + ["providerBatchSize", { providerBatchSize: 0 }], + ] as const) { + expect(() => + createDocumentSemanticEnrichmentProcessor({ ...baseProcessor, ...override }), + ).toThrow(`${name} must be at least 1`); + } + + const baseMaterializer = { + graph, + maxEntitiesPerNode: 1, + maxNodesPerArtifact: 1, + maxRelationsPerNode: 1, + nodes, + }; + for (const [name, override] of [ + ["maxEntitiesPerNode", { maxEntitiesPerNode: 0 }], + ["maxNodesPerArtifact", { maxNodesPerArtifact: 0 }], + ["maxRelationsPerNode", { maxRelationsPerNode: 0 }], + ] as const) { + expect(() => + createJointSemanticGraphMaterializer({ ...baseMaterializer, ...override }), + ).toThrow(`${name} must be at least 1`); + } + }); + + it("returns an empty result without constructing a provider", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 1, + }); + const graph = createInMemoryGraphIndexRepository({ + maxBatchSize: 1, + maxEntities: 1, + maxRelations: 1, + }); + let providerFactoryCalls = 0; + const processor = createDocumentSemanticEnrichmentProcessor({ + checkpoints: createInMemoryDocumentSemanticExtractionCheckpointRepository(), + graph, + maxConcurrentBatches: 1, + maxEntitiesPerNode: 1, + maxNodesPerArtifact: 1, + maxOutputTokens: 1, + maxRelationsPerNode: 1, + nodes, + providerBatchSize: 1, + providerFactory: () => { + providerFactoryCalls += 1; + return semanticProvider(); + }, + }); + const expected = { + entitiesExtracted: 0, + graphEntityIds: [], + graphEntitiesIndexed: 0, + graphRelationIds: [], + graphRelationsIndexed: 0, + nodesScanned: 0, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + }; + + await expect(processor.process(await semanticJob())).resolves.toEqual(expected); + await expect( + createJointSemanticGraphMaterializer({ + graph, + maxEntitiesPerNode: 1, + maxNodesPerArtifact: 1, + maxRelationsPerNode: 1, + nodes, + }).materialize({ + createdAt, + knowledgeSpaceId, + parseArtifactId, + publicationGenerationId, + retrievalProfile: (await semanticJob()).retrievalProfile, + }), + ).resolves.toEqual(expected); + expect(providerFactoryCalls).toBe(0); + }); + + it("rejects an artifact page that exceeds the configured node bound", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 2, + maxListLimit: 2, + maxNodes: 2, + }); + await nodes.createMany([knowledgeNode(0), knowledgeNode(1)]); + const graph = createInMemoryGraphIndexRepository({ + maxBatchSize: 2, + maxEntities: 2, + maxRelations: 2, + }); + const processor = createDocumentSemanticEnrichmentProcessor({ + checkpoints: createInMemoryDocumentSemanticExtractionCheckpointRepository(), + graph, + maxConcurrentBatches: 1, + maxEntitiesPerNode: 1, + maxNodesPerArtifact: 1, + maxOutputTokens: 1, + maxRelationsPerNode: 1, + nodes, + providerBatchSize: 1, + providerFactory: () => semanticProvider(), + }); + + await expect(processor.process(await semanticJob())).rejects.toThrow( + "node count exceeds maxNodesPerArtifact=1", + ); + await expect( + createJointSemanticGraphMaterializer({ + graph, + maxEntitiesPerNode: 1, + maxNodesPerArtifact: 1, + maxRelationsPerNode: 1, + nodes, + }).materialize({ + createdAt, + knowledgeSpaceId, + parseArtifactId, + publicationGenerationId, + retrievalProfile: (await semanticJob()).retrievalProfile, + }), + ).rejects.toThrow("node count exceeds maxNodesPerArtifact=1"); + }); + it("batches 80 nodes into at most 20 requests, checkpoints them, and preserves published nodes", async () => { const nodes = createInMemoryKnowledgeNodeRepository({ maxBatchSize: 100, @@ -132,6 +297,7 @@ describe("createDocumentSemanticEnrichmentProcessor", () => { }); await nodes.createMany([knowledgeNode(0)]); const provider = semanticProvider({ entityCount: 1 }); + let gatedRequests = 0; const processor = createDocumentSemanticEnrichmentProcessor({ checkpoints: createInMemoryDocumentSemanticExtractionCheckpointRepository(), graph: createInMemoryGraphIndexRepository({ @@ -144,6 +310,12 @@ describe("createDocumentSemanticEnrichmentProcessor", () => { maxNodesPerArtifact: 10, maxOutputTokens: 1_500, maxRelationsPerNode: 8, + modelRequestGate: { + run: async (request) => { + gatedRequests += 1; + return request(); + }, + }, nodes, now: () => createdAt, providerBatchSize: 8, @@ -156,6 +328,253 @@ describe("createDocumentSemanticEnrichmentProcessor", () => { }); expect(provider.entityCalls).toBe(1); expect(provider.relationCalls).toBe(0); + expect(gatedRequests).toBe(1); + }); + + it("fails closed when a checkpoint repository drops a completed batch", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 1, + }); + await nodes.createMany([knowledgeNode(0)]); + const processor = createDocumentSemanticEnrichmentProcessor({ + checkpoints: { + getMany: async () => [], + putMany: async () => [], + }, + graph: createInMemoryGraphIndexRepository({ + maxBatchSize: 2, + maxEntities: 2, + maxRelations: 2, + }), + maxConcurrentBatches: 1, + maxEntitiesPerNode: 2, + maxNodesPerArtifact: 1, + maxOutputTokens: 1_500, + maxRelationsPerNode: 2, + nodes, + providerBatchSize: 1, + providerFactory: () => semanticProvider({ entityCount: 1 }), + }); + + await expect(processor.process(await semanticJob())).rejects.toThrow( + "checkpoint is incomplete", + ); + }); + + it("rejects mixed, legacy, and frozen-model-mismatched semantic generations", async () => { + const graph = createInMemoryGraphIndexRepository({ + maxBatchSize: 10, + maxEntities: 10, + maxRelations: 10, + }); + const job = await semanticJob(); + const validJoint = await jointKnowledgeNode(job.retrievalProfile); + const mixedNodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxNodes: 10, + }); + await mixedNodes.createMany([validJoint, knowledgeNode(1)]); + const processorFor = (nodes: ReturnType) => + createDocumentSemanticEnrichmentProcessor({ + checkpoints: createInMemoryDocumentSemanticExtractionCheckpointRepository(), + graph, + maxConcurrentBatches: 1, + maxEntitiesPerNode: 8, + maxNodesPerArtifact: 10, + maxOutputTokens: 1_500, + maxRelationsPerNode: 8, + nodes, + providerBatchSize: 8, + providerFactory: () => semanticProvider(), + }); + + await expect(processorFor(mixedNodes).process(job)).rejects.toThrow( + "refuses a mixed joint/legacy node generation", + ); + + const legacyNodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxNodes: 10, + }); + await legacyNodes.createMany([knowledgeNode(0)]); + await expect( + createJointSemanticGraphMaterializer({ + graph, + maxEntitiesPerNode: 8, + maxNodesPerArtifact: 10, + maxRelationsPerNode: 8, + nodes: legacyNodes, + }).materialize({ + createdAt, + knowledgeSpaceId, + parseArtifactId, + publicationGenerationId, + retrievalProfile: job.retrievalProfile, + }), + ).rejects.toThrow("refuses a legacy or invalid node generation"); + + const otherProfile: KnowledgeSpaceRetrievalProfile = { + ...job.retrievalProfile, + reasoningModel: { model: "other", pluginId: "other-plugin", provider: "other-provider" }, + }; + const mismatchedJoint = await jointKnowledgeNode(otherProfile); + const mismatchedNodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxNodes: 10, + }); + await mismatchedNodes.createMany([mismatchedJoint]); + await expect(processorFor(mismatchedNodes).process(job)).rejects.toThrow( + "joint metadata does not match the frozen reasoning model", + ); + await expect( + createJointSemanticGraphMaterializer({ + graph, + maxEntitiesPerNode: 8, + maxNodesPerArtifact: 10, + maxRelationsPerNode: 8, + nodes: mismatchedNodes, + }).materialize({ + createdAt, + knowledgeSpaceId, + parseArtifactId, + publicationGenerationId, + retrievalProfile: job.retrievalProfile, + }), + ).rejects.toThrow("metadata does not match the frozen reasoning model"); + }); + + it("indexes joint semantic-chunk metadata without a second model request", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxNodes: 10, + }); + let chunkingCalls = 0; + const semanticNodes = await createLlmSemanticChunker({ + now: () => createdAt, + reasoningProviderFactory: () => ({ + kind: "plugin-daemon", + async *stream(input) { + chunkingCalls += 1; + const user = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(user?.content ?? "{}") as { + units: Array<{ id: string }>; + }; + yield { + delta: JSON.stringify({ + chunks: [ + { + endUnitId: payload.units.at(-1)?.id, + entities: [ + { confidence: 0.98, id: "acme", text: "Acme", type: "organization" }, + { confidence: 0.97, id: "policy", text: "policy", type: "policy" }, + ], + relations: [ + { + confidence: 0.96, + objectEntityId: "policy", + subjectEntityId: "acme", + type: "references", + }, + ], + sectionPath: ["Guide", "Renewal"], + sectionSummary: "Acme renewal policy.", + startUnitId: payload.units[0]?.id, + }, + ], + }), + type: "delta" as const, + }; + yield { + finishReason: "stop", + metadata: { model: input.model, provider: "plugin-daemon" }, + type: "done" as const, + }; + }, + }), + }).chunk({ + knowledgeSpaceId, + parseArtifact: ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "text", + createdAt, + documentAssetId, + elements: [ + { + id: "element-1", + metadata: {}, + sectionPath: ["Guide"], + text: "Acme follows the renewal policy.", + type: "paragraph", + }, + ], + id: parseArtifactId, + metadata: {}, + parser: "native-markdown", + version: 1, + }), + publicationGenerationId, + retrievalProfile: (await semanticJob()).retrievalProfile, + tenantId, + }); + await nodes.createMany(semanticNodes); + let enrichmentFactoryCalls = 0; + const graph = createInMemoryGraphIndexRepository({ + maxBatchSize: 10, + maxEntities: 10, + maxRelations: 10, + }); + const processor = createDocumentSemanticEnrichmentProcessor({ + checkpoints: createInMemoryDocumentSemanticExtractionCheckpointRepository(), + graph, + maxConcurrentBatches: 1, + maxEntitiesPerNode: 8, + maxNodesPerArtifact: 10, + maxOutputTokens: 1_500, + maxRelationsPerNode: 8, + nodes, + now: () => createdAt, + providerBatchSize: 8, + providerFactory: () => { + enrichmentFactoryCalls += 1; + throw new Error("joint semantic nodes must not invoke enrichment provider"); + }, + }); + + await expect(processor.process(await semanticJob())).resolves.toMatchObject({ + entitiesExtracted: 2, + graphEntitiesIndexed: 2, + graphRelationsIndexed: 1, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + }); + expect(chunkingCalls).toBe(1); + expect(enrichmentFactoryCalls).toBe(0); + await expect( + createJointSemanticGraphMaterializer({ + graph, + maxEntitiesPerNode: 8, + maxNodesPerArtifact: 10, + maxRelationsPerNode: 8, + nodes, + now: () => createdAt, + }).materialize({ + createdAt, + knowledgeSpaceId, + parseArtifactId, + publicationGenerationId, + retrievalProfile: (await semanticJob()).retrievalProfile, + }), + ).resolves.toMatchObject({ + graphEntityIds: [expect.any(String), expect.any(String)], + graphRelationIds: [expect.any(String)], + semanticProviderCalls: 0, + }); }); }); @@ -219,6 +638,78 @@ function semanticProvider(options: { entityCount?: number; failEntityCall?: numb return provider; } +async function jointKnowledgeNode( + retrievalProfile: KnowledgeSpaceRetrievalProfile, +): Promise { + const nodes = await createLlmSemanticChunker({ + now: () => createdAt, + reasoningProviderFactory: () => ({ + kind: "plugin-daemon", + async *stream(input) { + const user = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(user?.content ?? "{}") as { + units: Array<{ id: string }>; + }; + yield { + delta: JSON.stringify({ + chunks: [ + { + endUnitId: payload.units.at(-1)?.id, + entities: [ + { confidence: 0.98, id: "acme", text: "Acme", type: "organization" }, + { confidence: 0.97, id: "policy", text: "policy", type: "policy" }, + ], + relations: [ + { + confidence: 0.96, + objectEntityId: "policy", + subjectEntityId: "acme", + type: "references", + }, + ], + startUnitId: payload.units[0]?.id, + }, + ], + }), + type: "delta" as const, + }; + yield { + finishReason: "stop", + metadata: { model: input.model, provider: "plugin-daemon" }, + type: "done" as const, + }; + }, + }), + }).chunk({ + knowledgeSpaceId, + parseArtifact: ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "text", + createdAt, + documentAssetId, + elements: [ + { + id: "joint-element", + metadata: {}, + sectionPath: ["Guide"], + text: "Acme follows the renewal policy.", + type: "paragraph", + }, + ], + id: parseArtifactId, + metadata: {}, + parser: "native-markdown", + version: 1, + }), + publicationGenerationId, + retrievalProfile, + tenantId, + }); + const node = nodes[0]; + if (!node) throw new Error("joint semantic node fixture is empty"); + return node; +} + async function semanticJob() { const repository = createInMemoryDocumentSemanticEnrichmentRepository({ generateLeaseToken: () => uuid(99), diff --git a/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.ts b/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.ts index 24e0d0a4c9f..dd2d391f7a3 100644 --- a/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.ts +++ b/knowledge-fs/packages/api/src/document-semantic-enrichment-processor.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { type KnowledgeNode, type KnowledgeSpaceModelSelection, + type KnowledgeSpaceRetrievalProfile, PublicationGenerationIdSchema, stableJson, } from "@knowledge/core"; @@ -21,7 +22,7 @@ import { import { createExtractionQualityControlFlow } from "./extraction-quality-control-flow"; import type { GraphIndexRepository } from "./graph-index-repository"; import { createGraphIndexWriter } from "./graph-index-writer"; -import { cloneJsonObject } from "./json-utils"; +import { cloneJsonObject, isPlainObject } from "./json-utils"; import { type KnowledgeNodeRepository, cloneKnowledgeNode, @@ -35,6 +36,7 @@ import { type RelationExtractionTextProvider, createLlmRelationExtractionProvider, } from "./llm-relation-extraction-provider"; +import { hasValidLlmSemanticJointExtraction } from "./llm-semantic-chunker"; import { createRelationExtractionFlow } from "./relation-extraction-flow"; export type DocumentSemanticEnrichmentTextProvider = EntityExtractionTextProvider & @@ -42,7 +44,9 @@ export type DocumentSemanticEnrichmentTextProvider = EntityExtractionTextProvide export interface DocumentSemanticEnrichmentProcessorResult { readonly entitiesExtracted: number; + readonly graphEntityIds: readonly string[]; readonly graphEntitiesIndexed: number; + readonly graphRelationIds: readonly string[]; readonly graphRelationsIndexed: number; readonly nodesScanned: number; readonly semanticProviderCalls: number; @@ -53,6 +57,25 @@ export interface DocumentSemanticEnrichmentProcessor { process(job: DocumentSemanticEnrichmentJob): Promise; } +export interface JointSemanticGraphMaterializer { + materialize(input: { + readonly createdAt: string; + readonly knowledgeSpaceId: string; + readonly parseArtifactId: string; + readonly publicationGenerationId: string; + readonly retrievalProfile: KnowledgeSpaceRetrievalProfile; + }): Promise; +} + +export interface JointSemanticGraphMaterializerOptions { + readonly graph: GraphIndexRepository; + readonly maxEntitiesPerNode: number; + readonly maxNodesPerArtifact: number; + readonly maxRelationsPerNode: number; + readonly nodes: Pick; + readonly now?: (() => string) | undefined; +} + export interface DocumentSemanticEnrichmentProcessorOptions { readonly checkpoints: DocumentSemanticExtractionCheckpointRepository; readonly graph: GraphIndexRepository; @@ -121,7 +144,9 @@ export function createDocumentSemanticEnrichmentProcessor({ if (page.items.length === 0) { return { entitiesExtracted: 0, + graphEntityIds: [], graphEntitiesIndexed: 0, + graphRelationIds: [], graphRelationsIndexed: 0, nodesScanned: 0, semanticProviderCalls: 0, @@ -133,6 +158,35 @@ export function createDocumentSemanticEnrichmentProcessor({ if (!selection) { throw new Error("Document semantic enrichment requires a frozen reasoning model"); } + const originalNodes = page.items.map(cloneKnowledgeNode); + const jointSemanticNodes = originalNodes.filter(hasValidLlmSemanticJointExtraction); + if (jointSemanticNodes.length > 0) { + if (jointSemanticNodes.length !== originalNodes.length) { + throw new Error( + "Document semantic enrichment refuses a mixed joint/legacy node generation", + ); + } + if ( + jointSemanticNodes.some( + (node) => stableJson(jointSemanticModelSelection(node)) !== stableJson(selection), + ) + ) { + throw new Error( + "Document semantic enrichment joint metadata does not match the frozen reasoning model", + ); + } + return indexPreparedSemanticNodes({ + graph, + job, + maxEntitiesPerNode, + maxNodesPerArtifact, + maxRelationsPerNode, + nodes: jointSemanticNodes, + now, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + }); + } let semanticProviderCalls = 0; const resolvedProvider = providerFactory(selection); const provider: DocumentSemanticEnrichmentTextProvider = { @@ -159,7 +213,6 @@ export function createDocumentSemanticEnrichmentProcessor({ publicationGenerationId: generationId, tenantId: job.tenantId, }; - const originalNodes = page.items.map(cloneKnowledgeNode); const entityCheckpoints = await completeEntityCheckpoints({ checkpoints, maxConcurrentBatches, @@ -186,57 +239,175 @@ export function createDocumentSemanticEnrichmentProcessor({ tenantId: job.tenantId, }); const relationNodes = applyStageCheckpoints(entityNodes, relationCheckpoints); - const qualityRepository = await temporaryNodeRepository(relationNodes); - const controlled = await createExtractionQualityControlFlow({ - maxBatchSize: maxNodesPerArtifact, - maxEligibleEntitiesPerNode: maxEntitiesPerNode, - maxEligibleRelationsPerNode: maxRelationsPerNode, - nodes: qualityRepository, - now, - }).apply({ - knowledgeSpaceId: job.knowledgeSpaceId, - nodeIds: relationNodes.map((node) => node.id), - publicationGenerationId: generationId, - }); - if (controlled.missingNodeIds.length > 0) { - throw new Error("Document semantic enrichment quality stage lost immutable nodes"); - } - const indexed = await createGraphIndexWriter({ - extractionVersion: 1, - graph, - maxBatchSize: maxNodesPerArtifact, - nodes: qualityRepository, - // A generation retry must reproduce byte-identical immutable graph rows. - now: () => job.createdAt, - }).indexNodes({ - knowledgeSpaceId: job.knowledgeSpaceId, - nodes: controlled.controlledNodes, - publicationGenerationId: generationId, - }); - if (indexed.missingNodeIds.length > 0) { - throw new Error("Document semantic enrichment graph stage lost immutable nodes"); - } - const eligibleRelationNodes = entityNodes.filter( (node) => extractedEntitiesFromNodeMetadata(node).length >= 2, ).length; - return { - entitiesExtracted: controlled.controlledNodes.reduce( - (sum, node) => sum + extractedEntitiesFromNodeMetadata(node).length, - 0, - ), - graphEntitiesIndexed: indexed.stats.entitiesIndexed, - graphRelationsIndexed: indexed.stats.relationsIndexed, - nodesScanned: originalNodes.length, + return indexPreparedSemanticNodes({ + graph, + job, + maxEntitiesPerNode, + maxNodesPerArtifact, + maxRelationsPerNode, + nodes: relationNodes, + now, semanticProviderCalls, semanticProviderCallsMaximum: Math.ceil(originalNodes.length / providerBatchSize) + Math.ceil(eligibleRelationNodes / providerBatchSize), - }; + }); }, }; } +/** Materializes only the joint facts already frozen by semantic chunking; it never calls an LLM. */ +export function createJointSemanticGraphMaterializer({ + graph, + maxEntitiesPerNode, + maxNodesPerArtifact, + maxRelationsPerNode, + nodes, + now = () => new Date().toISOString(), +}: JointSemanticGraphMaterializerOptions): JointSemanticGraphMaterializer { + for (const [name, value] of Object.entries({ + maxEntitiesPerNode, + maxNodesPerArtifact, + maxRelationsPerNode, + })) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`Joint semantic Graph ${name} must be at least 1`); + } + } + return { + materialize: async (input) => { + const generationId = PublicationGenerationIdSchema.parse(input.publicationGenerationId); + const page = await nodes.listByArtifact({ + knowledgeSpaceId: input.knowledgeSpaceId, + limit: maxNodesPerArtifact, + parseArtifactId: input.parseArtifactId, + publicationGenerationId: generationId, + }); + if (page.nextCursor) { + throw new Error( + `Joint semantic Graph node count exceeds maxNodesPerArtifact=${maxNodesPerArtifact}`, + ); + } + if (page.items.length === 0) { + return { + entitiesExtracted: 0, + graphEntityIds: [], + graphEntitiesIndexed: 0, + graphRelationIds: [], + graphRelationsIndexed: 0, + nodesScanned: 0, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + }; + } + const prepared = page.items.map(cloneKnowledgeNode); + if (prepared.some((node) => !hasValidLlmSemanticJointExtraction(node))) { + throw new Error("Joint semantic Graph refuses a legacy or invalid node generation"); + } + const selection = input.retrievalProfile.reasoningModel; + if ( + prepared.some( + (node) => stableJson(jointSemanticModelSelection(node)) !== stableJson(selection), + ) + ) { + throw new Error("Joint semantic Graph metadata does not match the frozen reasoning model"); + } + return indexPreparedSemanticNodes({ + graph, + job: { + createdAt: input.createdAt, + knowledgeSpaceId: input.knowledgeSpaceId, + publicationGenerationId: generationId, + }, + maxEntitiesPerNode, + maxNodesPerArtifact, + maxRelationsPerNode, + nodes: prepared, + now, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + }); + }, + }; +} + +function jointSemanticModelSelection(node: KnowledgeNode): unknown { + const semantic = node.metadata.semanticChunking; + return isPlainObject(semantic) ? semantic.modelSelection : undefined; +} + +async function indexPreparedSemanticNodes({ + graph, + job, + maxEntitiesPerNode, + maxNodesPerArtifact, + maxRelationsPerNode, + nodes, + now, + semanticProviderCalls, + semanticProviderCallsMaximum, +}: { + readonly graph: GraphIndexRepository; + readonly job: Pick< + DocumentSemanticEnrichmentJob, + "createdAt" | "knowledgeSpaceId" | "publicationGenerationId" + >; + readonly maxEntitiesPerNode: number; + readonly maxNodesPerArtifact: number; + readonly maxRelationsPerNode: number; + readonly nodes: readonly KnowledgeNode[]; + readonly now: () => string; + readonly semanticProviderCalls: number; + readonly semanticProviderCallsMaximum: number; +}): Promise { + const generationId = PublicationGenerationIdSchema.parse(job.publicationGenerationId); + const qualityRepository = await temporaryNodeRepository(nodes); + const controlled = await createExtractionQualityControlFlow({ + maxBatchSize: maxNodesPerArtifact, + maxEligibleEntitiesPerNode: maxEntitiesPerNode, + maxEligibleRelationsPerNode: maxRelationsPerNode, + nodes: qualityRepository, + now, + }).apply({ + knowledgeSpaceId: job.knowledgeSpaceId, + nodeIds: nodes.map((node) => node.id), + publicationGenerationId: generationId, + }); + if (controlled.missingNodeIds.length > 0) { + throw new Error("Document semantic enrichment quality stage lost immutable nodes"); + } + const indexed = await createGraphIndexWriter({ + extractionVersion: 1, + graph, + maxBatchSize: maxNodesPerArtifact, + nodes: qualityRepository, + now: () => job.createdAt, + }).indexNodes({ + knowledgeSpaceId: job.knowledgeSpaceId, + nodes: controlled.controlledNodes, + publicationGenerationId: generationId, + }); + if (indexed.missingNodeIds.length > 0) { + throw new Error("Document semantic enrichment graph stage lost immutable nodes"); + } + return { + entitiesExtracted: controlled.controlledNodes.reduce( + (sum, node) => sum + extractedEntitiesFromNodeMetadata(node).length, + 0, + ), + graphEntityIds: indexed.entities.map((entity) => entity.id), + graphEntitiesIndexed: indexed.stats.entitiesIndexed, + graphRelationIds: indexed.relations.map((relation) => relation.id), + graphRelationsIndexed: indexed.stats.relationsIndexed, + nodesScanned: nodes.length, + semanticProviderCalls, + semanticProviderCallsMaximum, + }; +} + async function completeEntityCheckpoints(input: { readonly checkpoints: DocumentSemanticExtractionCheckpointRepository; readonly maxConcurrentBatches: number; diff --git a/knowledge-fs/packages/api/src/document-semantic-enrichment-runtime.test.ts b/knowledge-fs/packages/api/src/document-semantic-enrichment-runtime.test.ts index 1b5cbda4d54..cdeccd286d3 100644 --- a/knowledge-fs/packages/api/src/document-semantic-enrichment-runtime.test.ts +++ b/knowledge-fs/packages/api/src/document-semantic-enrichment-runtime.test.ts @@ -58,7 +58,9 @@ describe("createDocumentSemanticEnrichmentRuntime", () => { const processor = { process: vi.fn(async () => ({ entitiesExtracted: 4, + graphEntityIds: [uuid(31), uuid(32), uuid(33)], graphEntitiesIndexed: 3, + graphRelationIds: [uuid(34)], graphRelationsIndexed: 1, nodesScanned: 8, semanticProviderCalls: 2, diff --git a/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts b/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts new file mode 100644 index 00000000000..3c64842e090 --- /dev/null +++ b/knowledge-fs/packages/api/src/index-reindexer-semantic.test.ts @@ -0,0 +1,752 @@ +import type { ComputeRuntime } from "@knowledge/compute"; +import { KnowledgeNodeSchema, ParseArtifactSchema } from "@knowledge/core"; +import { describe, expect, it } from "vitest"; + +import { createIncrementalReindexer } from "./index-reindexer"; +import { createInMemoryKnowledgeNodeRepository } from "./knowledge-node-repository"; +import { createLlmSemanticChunker } from "./llm-semantic-chunker"; +import { createInMemoryParseArtifactRepository } from "./parse-artifact-repository"; + +const KNOWLEDGE_SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c40"; +const DOCUMENT_ASSET_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c41"; +const PARSE_ARTIFACT_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42"; +const GENERATION_A = "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca1"; +const GENERATION_B = "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca2"; + +function parseArtifact() { + return ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "text", + createdAt: "2026-08-13T12:00:00.000Z", + documentAssetId: DOCUMENT_ASSET_ID, + elements: [ + { + id: "element-1", + sectionPath: ["Invoice"], + sourceLocation: { endOffset: 18, startOffset: 0 }, + text: "Invoice buyer amount", + type: "paragraph", + }, + ], + id: PARSE_ARTIFACT_ID, + metadata: {}, + parser: "native-markdown", + version: 1, + }); +} + +function computeRuntime(onChunk?: () => void): ComputeRuntime { + return { + chunkParseArtifact: (input) => { + onChunk?.(); + return [ + KnowledgeNodeSchema.parse({ + artifactHash: input.parseArtifact.artifactHash, + documentAssetId: input.parseArtifact.documentAssetId, + endOffset: 18, + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2d42", + kind: "chunk", + knowledgeSpaceId: input.knowledgeSpaceId, + metadata: { chunkIndex: 0, elementIds: ["element-1"] }, + parseArtifactId: input.parseArtifact.id, + permissionScope: input.permissionScope ? [...input.permissionScope] : undefined, + sourceLocation: { endOffset: 18, sectionPath: ["Invoice"], startOffset: 0 }, + startOffset: 0, + text: "Invoice buyer amount", + }), + ]; + }, + countApproxTokens: () => 1, + countTokens: () => 1, + diffText: () => ({ operations: [], stats: { delete: 0, equal: 0, insert: 0 } }), + packEvidence: () => ({ context: "", items: [], omitted: [], tokenBudget: 1, usedTokens: 0 }), + rrfFuse: () => [], + }; +} + +function retrievalProfile() { + return { + defaultMode: "research" as const, + reasoningModel: { + model: "reasoning-v1", + pluginId: "reasoning-plugin", + provider: "plugin-daemon", + }, + rerank: { enabled: false as const }, + revision: 1, + scoreThreshold: { enabled: false as const, stage: "mode-final" as const }, + topK: 10, + }; +} + +function semanticChunker(onCall: () => void) { + return createLlmSemanticChunker({ + maxChunkChars: 512, + maxWindowChars: 600, + now: () => "2026-08-13T12:00:00.000Z", + promptVersion: "semantic-reindex-v1", + reasoningProviderFactory: () => ({ + kind: "plugin-daemon", + async *stream(input) { + onCall(); + const user = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(user?.content ?? "{}") as { + units: Array<{ id: string }>; + }; + yield { + delta: JSON.stringify({ + chunks: [ + { + endUnitId: payload.units.at(-1)?.id, + entities: [ + { + confidence: 0.99, + id: "invoice", + text: "Invoice", + type: "policy", + }, + ], + relations: [], + sectionPath: ["Invoice", "Buyer and amount"], + sectionSummary: "Invoice identity, buyer, and amount details.", + startUnitId: payload.units[0]?.id, + }, + ], + }), + type: "delta" as const, + }; + yield { + finishReason: "stop", + metadata: { model: input.model, provider: "plugin-daemon" }, + type: "done" as const, + }; + }, + }), + }); +} + +function echoSemanticChunker(onCall: () => void) { + return createLlmSemanticChunker({ + maxChunkChars: 512, + maxWindowChars: 600, + now: () => "2026-08-13T12:00:00.000Z", + promptVersion: "semantic-reindex-v1", + reasoningProviderFactory: () => ({ + kind: "plugin-daemon", + async *stream(input) { + onCall(); + const user = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(user?.content ?? "{}") as { + sectionPath: string[]; + units: Array<{ id: string; type: string }>; + }; + yield { + delta: JSON.stringify({ + chunks: [ + { + endUnitId: payload.units.at(-1)?.id, + entities: [], + relations: [], + sectionPath: payload.sectionPath, + ...(payload.units[0]?.type === "paragraph" + ? { sectionSummary: "Semantic paragraph summary." } + : {}), + startUnitId: payload.units[0]?.id, + }, + ], + }), + type: "delta" as const, + }; + yield { + finishReason: "stop", + metadata: { model: input.model, provider: "plugin-daemon" }, + type: "done" as const, + }; + }, + }), + }); +} + +function richParseArtifact() { + return ParseArtifactSchema.parse({ + artifactHash: "b".repeat(64), + contentType: "structured", + createdAt: "2026-08-13T12:00:00.000Z", + documentAssetId: DOCUMENT_ASSET_ID, + elements: [ + { + id: "paragraph", + metadata: {}, + pageNumber: 1, + sectionPath: ["Rich"], + text: "Paragraph content.", + type: "paragraph", + }, + { + id: "table", + metadata: { title: "Amounts" }, + pageNumber: 2, + sectionPath: ["Rich"], + text: "Item | Amount", + type: "table", + }, + { + id: "image", + metadata: { caption: "Receipt" }, + sectionPath: ["Rich"], + text: "Receipt image", + type: "image", + }, + ], + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c49", + metadata: {}, + parser: "native-structured", + version: 1, + }); +} + +function compactCompletionSemanticChunker(onCall: () => void) { + return createLlmSemanticChunker({ + maxChunkChars: 512, + maxWindowChars: 600, + reasoningProviderFactory: () => ({ + async *stream(input) { + onCall(); + const user = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(user?.content ?? "{}") as { + units: Array<{ id: string }>; + }; + yield { + delta: JSON.stringify({ + chunks: payload.units.map((unit) => ({ + endUnitId: unit.id, + entities: [], + relations: [], + startUnitId: unit.id, + })), + }), + type: "delta" as const, + }; + yield { type: "done" as const }; + }, + }), + }); +} + +describe("incremental reindexer semantic generations", () => { + it("uses the frozen reasoning model and replays the durable generation without another call", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 1, + maxNodes: 4, + }); + let llmCalls = 0; + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }), + compute: { + ...computeRuntime(), + chunkParseArtifact: () => { + throw new Error("deterministic chunker must not run"); + }, + }, + maxNodeReplayPageSize: 1, + maxNodes: 4, + nodes, + semanticChunker: semanticChunker(() => llmCalls++), + }); + const input = { + chunkConfig: { maxChunkChars: 512 }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + } as const; + + const first = await reindexer.reindex(input); + const replay = await reindexer.reindex(input); + + expect(first).toMatchObject({ nodesCreated: 1, status: "rebuilt" }); + expect(first).toMatchObject({ + outlineArtifact: { + elements: [ + expect.objectContaining({ + sectionPath: ["Invoice", "Buyer and amount"], + text: "Invoice buyer amount", + }), + ], + metadata: expect.objectContaining({ semanticCompilation: expect.any(Object) }), + }, + }); + expect(replay).toMatchObject({ + nodeIds: first.status === "rebuilt" ? first.nodeIds : undefined, + nodesCreated: 1, + status: "rebuilt", + }); + expect(llmCalls).toBe(1); + await expect( + nodes.getGenerationReceipt?.({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifactId: PARSE_ARTIFACT_ID, + publicationGenerationId: GENERATION_A, + }), + ).resolves.toMatchObject({ + documentChunkCount: 1, + storedNodeCount: 1, + windowManifest: [expect.objectContaining({ windowId: "window-000000" })], + }); + }); + + it("persists and replays a fully excluded semantic result", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 1, + maxNodes: 4, + }); + let llmCalls = 0; + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }), + compute: computeRuntime(), + maxNodeReplayPageSize: 1, + maxNodes: 4, + nodes, + semanticChunker: semanticChunker(() => llmCalls++), + }); + const input = { + chunkConfig: { maxChunkChars: 512 }, + excludedNodeOrdinals: [0], + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + } as const; + + await expect(reindexer.reindex(input)).resolves.toMatchObject({ + nodeIds: [], + nodesCreated: 0, + status: "rebuilt", + }); + await expect(reindexer.reindex(input)).resolves.toMatchObject({ nodesCreated: 0 }); + expect(llmCalls).toBe(1); + }); + + it("clones an existing semantic node generation for projection-only migration", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }); + let deterministicCalls = 0; + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }), + compute: computeRuntime(() => deterministicCalls++), + maxNodeReplayPageSize: 1, + maxNodes: 4, + nodes, + }); + + const source = await reindexer.reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + }); + const target = await reindexer.reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 2, + publicationGenerationId: GENERATION_B, + reuseNodeGenerationId: GENERATION_A, + }); + const targetReplay = await reindexer.reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 2, + publicationGenerationId: GENERATION_B, + reuseNodeGenerationId: GENERATION_A, + }); + + expect(source).toMatchObject({ nodesCreated: 1 }); + expect(target).toMatchObject({ nodesCreated: 1 }); + expect(targetReplay).toMatchObject({ + nodeIds: target.status === "rebuilt" ? target.nodeIds : undefined, + nodesCreated: 1, + }); + expect(deterministicCalls).toBe(1); + }); + + it("requires tenant and durable receipt capabilities for profile-scoped semantic generations", async () => { + const durableNodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }); + const baseOptions = { + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }), + compute: computeRuntime(), + maxNodes: 4, + semanticChunker: semanticChunker(() => undefined), + }; + await expect( + createIncrementalReindexer({ ...baseOptions, nodes: durableNodes }).reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + }), + ).rejects.toThrow("tenantId is required"); + + const { + completeGenerationAtomically: _completeGenerationAtomically, + getGenerationReceipt: _getGenerationReceipt, + ...nodesWithoutReceipts + } = durableNodes; + await expect( + createIncrementalReindexer({ ...baseOptions, nodes: nodesWithoutReceipts }).reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + }), + ).rejects.toThrow("requires durable generation receipts"); + }); + + it("persists complete semantic options and builds a typed outline across layout elements", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxNodes: 10, + }); + let llmCalls = 0; + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }), + compute: computeRuntime(), + maxNodes: 10, + nodes, + semanticChunker: echoSemanticChunker(() => llmCalls++), + }); + const input = { + chunkConfig: { + maxChunkChars: 512, + maxNodes: 10, + maxWindowChars: 600, + overlapChars: 0, + }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + language: "zh-CN", + parseArtifact: richParseArtifact(), + permissionScope: ["tenant:one"], + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + } as const; + + const first = await reindexer.reindex(input); + const replay = await reindexer.reindex(input); + + expect(first).toMatchObject({ + nodesCreated: 3, + outlineArtifact: { + elements: [ + expect.objectContaining({ + metadata: expect.objectContaining({ + semanticSectionSummary: "Semantic paragraph summary.", + }), + pageNumber: 1, + type: "paragraph", + }), + expect.objectContaining({ pageNumber: 2, type: "table" }), + expect.objectContaining({ type: "image" }), + ], + }, + status: "rebuilt", + }); + expect( + first.status === "rebuilt" ? first.outlineArtifact?.elements[2]?.pageNumber : 0, + ).toBeUndefined(); + expect(replay).toMatchObject({ nodesCreated: 3, status: "rebuilt" }); + expect(llmCalls).toBe(3); + await expect( + nodes.getGenerationReceipt?.({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifactId: richParseArtifact().id, + publicationGenerationId: GENERATION_A, + }), + ).resolves.toMatchObject({ + language: "zh-CN", + permissionScope: ["tenant:one"], + semanticConfig: { maxNodes: 10, overlapChars: 0 }, + }); + }); + + it("supports semantic compilation without a publication generation", async () => { + let llmCalls = 0; + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 2 }), + compute: computeRuntime(), + maxNodes: 4, + nodes: createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }), + semanticChunker: echoSemanticChunker(() => llmCalls++), + }); + + await expect( + reindexer.reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + }), + ).resolves.toMatchObject({ nodesCreated: 1, outlineArtifact: expect.any(Object) }); + expect(llmCalls).toBe(1); + }); + + it("fails closed when durable semantic receipt capabilities are incomplete", async () => { + const durableNodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }); + const baseInput = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + } as const; + await expect( + createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 2 }), + compute: computeRuntime(), + maxNodes: 4, + nodes: durableNodes, + semanticChunker: { + chunk: async () => [], + }, + }).reindex(baseInput), + ).rejects.toThrow("must expose replay defaults"); + + const nodesWithoutAtomicPersistence = { + ...durableNodes, + completeGenerationAtomically: async () => undefined, + } as unknown as typeof durableNodes; + await expect( + createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 2 }), + compute: computeRuntime(), + maxNodes: 4, + nodes: nodesWithoutAtomicPersistence, + semanticChunker: echoSemanticChunker(() => undefined), + }).reindex(baseInput), + ).rejects.toThrow("requires atomic semantic generation receipts"); + }); + + it("rejects semantic exclusions outside the canonical upper bound before model invocation", async () => { + let llmCalls = 0; + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 2 }), + compute: computeRuntime(), + maxNodes: 4, + nodes: createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }), + semanticChunker: echoSemanticChunker(() => llmCalls++), + }); + + await expect( + reindexer.reindex({ + excludedNodeOrdinals: [-1], + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + }), + ).rejects.toThrow("exclusions exceed the canonical chunk upper bound"); + expect(llmCalls).toBe(0); + }); + + it("builds a multi-chunk receipt with a compact optional completion identity", async () => { + const nodes = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }); + let llmCalls = 0; + const source = ParseArtifactSchema.parse({ + ...parseArtifact(), + artifactHash: "c".repeat(64), + elements: [ + { + id: "two-sentences", + metadata: {}, + sectionPath: ["Receipt"], + text: "First sentence. Second sentence.", + type: "paragraph", + }, + ], + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c50", + }); + const reindexer = createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 2 }), + compute: computeRuntime(), + maxNodes: 4, + nodes, + semanticChunker: compactCompletionSemanticChunker(() => llmCalls++), + }); + + await expect( + reindexer.reindex({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: source, + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + }), + ).resolves.toMatchObject({ nodesCreated: 2 }); + expect(llmCalls).toBe(1); + await expect( + nodes.getGenerationReceipt?.({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifactId: source.id, + publicationGenerationId: GENERATION_A, + }), + ).resolves.toMatchObject({ + completionCatalog: [{ fingerprint: expect.stringMatching(/^sha256:/u) }], + documentChunkCount: 2, + windowManifest: [expect.objectContaining({ chunkRanges: expect.any(Array) })], + }); + }); + + it("fails closed when a semantic chunker returns corrupt receipt markers", async () => { + const baseInput = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact(), + projectionVersion: 1, + publicationGenerationId: GENERATION_A, + retrievalProfile: retrievalProfile(), + tenantId: "tenant-1", + } as const; + const cases: Array<{ + error: string; + mutate: ( + node: ReturnType, + ) => ReturnType; + }> = [ + { + error: "cannot build semantic window receipt from node marker", + mutate: (node) => + KnowledgeNodeSchema.parse({ + ...node, + metadata: { ...node.metadata, semanticChunking: null }, + }), + }, + { + error: "cannot build semantic window receipt from node marker", + mutate: (node) => { + const marker = node.metadata.semanticChunking as Record; + return KnowledgeNodeSchema.parse({ + ...node, + metadata: { ...node.metadata, semanticChunking: { ...marker, unitRange: null } }, + }); + }, + }, + { + error: "cannot build semantic window receipt from node marker", + mutate: (node) => { + const marker = node.metadata.semanticChunking as Record; + return KnowledgeNodeSchema.parse({ + ...node, + metadata: { ...node.metadata, semanticChunking: { ...marker, windowId: " " } }, + }); + }, + }, + { + error: "cannot build semantic window receipt from node marker", + mutate: (node) => { + const marker = node.metadata.semanticChunking as Record; + return KnowledgeNodeSchema.parse({ + ...node, + metadata: { + ...node.metadata, + semanticChunking: { ...marker, inputFingerprint: "invalid" }, + }, + }); + }, + }, + { + error: "cannot build semantic window receipt from node marker", + mutate: (node) => + KnowledgeNodeSchema.parse({ + ...node, + metadata: { ...node.metadata, chunkIndex: 1 }, + }), + }, + { + error: "semantic completion identity is missing", + mutate: (node) => { + const marker = node.metadata.semanticChunking as Record; + return KnowledgeNodeSchema.parse({ + ...node, + metadata: { ...node.metadata, semanticChunking: { ...marker, completion: null } }, + }); + }, + }, + { + error: "semantic completion actualModel is invalid", + mutate: (node) => { + const marker = node.metadata.semanticChunking as Record; + const completion = marker.completion as Record; + return KnowledgeNodeSchema.parse({ + ...node, + metadata: { + ...node.metadata, + semanticChunking: { + ...marker, + completion: { ...completion, actual: { model: " " } }, + }, + }, + }); + }, + }, + ]; + + for (const testCase of cases) { + const base = echoSemanticChunker(() => undefined); + const corruptingChunker = { + ...base, + chunk: async (input: Parameters[0]) => + (await base.chunk(input)).map(testCase.mutate), + }; + await expect( + createIncrementalReindexer({ + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 2 }), + compute: computeRuntime(), + maxNodes: 4, + nodes: createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 4, + maxListLimit: 4, + maxNodes: 4, + }), + semanticChunker: corruptingChunker, + }).reindex(baseInput), + ).rejects.toThrow(testCase.error); + } + }); +}); diff --git a/knowledge-fs/packages/api/src/index-reindexer.test.ts b/knowledge-fs/packages/api/src/index-reindexer.test.ts index 080367bc1d5..9fa7ddcc1a8 100644 --- a/knowledge-fs/packages/api/src/index-reindexer.test.ts +++ b/knowledge-fs/packages/api/src/index-reindexer.test.ts @@ -775,6 +775,60 @@ describe("incremental reindexer", () => { nodes, }), ).toThrow("Incremental reindexer maxNodes must be at least 1"); + expect(() => + createIncrementalReindexer({ + artifacts, + compute, + maxNodes: 4, + maxProjectionBatchSize: 0, + nodes, + }), + ).toThrow("maxProjectionBatchSize must be at least 1"); + expect(() => + createIncrementalReindexer({ + artifacts, + compute, + maxNodeReplayPageSize: 0, + maxNodes: 4, + nodes, + }), + ).toThrow("maxNodeReplayPageSize must be at least 1"); + + const validating = createIncrementalReindexer({ artifacts, compute, maxNodes: 4, nodes }); + const validInput = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: parseArtifact({ artifactHash: "8".repeat(64) }), + projectionVersion: 1, + } as const; + await expect(validating.reindex({ ...validInput, knowledgeSpaceId: " " })).rejects.toThrow( + "knowledgeSpaceId is required", + ); + await expect(validating.reindex({ ...validInput, projectionVersion: 0 })).rejects.toThrow( + "projectionVersion must be a positive integer", + ); + await expect( + validating.reindex({ ...validInput, reuseNodeGenerationId: PUBLICATION_GENERATION_ID }), + ).rejects.toThrow("reuseNodeGenerationId requires publicationGenerationId"); + await expect( + validating.reindex({ + ...validInput, + publicationGenerationId: PUBLICATION_GENERATION_ID, + reuseNodeGenerationId: PUBLICATION_GENERATION_ID, + }), + ).rejects.toThrow("source and target node generations must be different"); + await expect( + validating.reindex({ ...validInput, denseModel: "dense", skipDense: true }), + ).rejects.toThrow("skipDense cannot include dense model configuration"); + await expect( + validating.reindex({ ...validInput, skipVisual: true, visualModel: "clip" }), + ).rejects.toThrow("skipVisual cannot include visual model configuration"); + await expect( + validating.reindex({ + ...validInput, + publicationGenerationId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c53", + reuseNodeGenerationId: PUBLICATION_GENERATION_ID, + }), + ).rejects.toThrow("could not load source generation nodes"); let denseBuilds = 0; await expect( diff --git a/knowledge-fs/packages/api/src/index-reindexer.ts b/knowledge-fs/packages/api/src/index-reindexer.ts index b349e6068fc..3c76baae091 100644 --- a/knowledge-fs/packages/api/src/index-reindexer.ts +++ b/knowledge-fs/packages/api/src/index-reindexer.ts @@ -1,11 +1,16 @@ +import { createHash } from "node:crypto"; + import type { ChunkConfig, ComputeRuntime } from "@knowledge/compute"; import { type IndexProjection, type KnowledgeNode, type KnowledgeSpaceEmbeddingProfile, + type KnowledgeSpaceRetrievalProfile, + KnowledgeSpaceRetrievalProfileSchema, type ParseArtifact, ParseArtifactSchema, PublicationGenerationIdSchema, + stableJson, } from "@knowledge/core"; import { deterministicChildId } from "./api-shared-utils"; @@ -19,12 +24,31 @@ import { import type { IndexProjectionRepository } from "./index-projection-repository"; import { isPlainObject } from "./json-utils"; import type { KnowledgeFsOperationLeaseCoordinator } from "./knowledge-fs-operation-leases"; -import { type KnowledgeNodeRepository, cloneKnowledgeNode } from "./knowledge-node-repository"; +import { + type KnowledgeNodeGenerationCompletionReceipt, + type KnowledgeNodeGenerationReceipt, + type KnowledgeNodeGenerationUnitRangeReceipt, + type KnowledgeNodeGenerationWindowReceipt, + type KnowledgeNodeRepository, + type KnowledgeNodeSemanticGenerationConfig, + cloneKnowledgeNode, +} from "./knowledge-node-repository"; +import { + type SemanticChunker, + assertValidLlmSemanticGenerationReplay, + assertValidLlmSemanticWindowManifestReplay, + preflightLlmSemanticWindows, +} from "./llm-semantic-chunker"; import { type ParseArtifactLookupInput, type ParseArtifactRepository, cloneParseArtifact, } from "./parse-artifact-repository"; +import { + MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES, + llmSemanticCompletionFingerprint, + maximumKnowledgeNodeGenerationReceiptSerializedBytes, +} from "./semantic-generation-receipt"; export interface IncrementalReindexInput { readonly chunkConfig?: ChunkConfig | undefined; @@ -33,6 +57,8 @@ export interface IncrementalReindexInput { readonly denseModel?: string | undefined; /** Immutable profile captured before the reindex started. */ readonly embeddingProfile?: KnowledgeSpaceEmbeddingProfile | undefined; + /** Whether Graph facts from this semantic generation will be materialized by the caller. */ + readonly enableGraph?: boolean | undefined; readonly knowledgeSpaceId: string; /** Optional normalized BCP-47 document language persisted into every generated node. */ readonly language?: string | undefined; @@ -43,6 +69,14 @@ export interface IncrementalReindexInput { readonly publicationGenerationId?: string | undefined; /** Removes failed projections from an unpublished generation before rebuilding a retry. */ readonly resetFailedProjections?: boolean | undefined; + /** Clone already-published chunks from this generation when only projections are migrating. */ + readonly reuseNodeGenerationId?: string | undefined; + /** Immutable per-space reasoning profile captured by the durable compilation attempt. */ + readonly retrievalProfile?: KnowledgeSpaceRetrievalProfile | undefined; + /** Explicit FTS-only build for a published retrieval profile with no active embedding profile. */ + readonly skipDense?: true | undefined; + /** Preserve existing visual projections instead of rebuilding them in a text-only migration. */ + readonly skipVisual?: true | undefined; readonly signal?: AbortSignal | undefined; readonly tenantId?: string | undefined; readonly visualModel?: string | undefined; @@ -60,6 +94,8 @@ export type IncrementalReindexResult = readonly artifact: ParseArtifact; readonly nodeIds?: readonly string[] | undefined; readonly nodesCreated: number; + /** Ephemeral semantic-node projection used to build the published outline and PageIndex. */ + readonly outlineArtifact?: ParseArtifact | undefined; readonly projectionIds?: readonly string[] | undefined; readonly projectionsCreated: number; readonly status: "rebuilt"; @@ -84,10 +120,13 @@ export interface IncrementalReindexerOptions { readonly denseBuilder?: DenseVectorProjectionBuilder | undefined; readonly ftsBuilder?: FtsProjectionBuilder | undefined; readonly maxNodes: number; + /** Repository-safe page size used when replaying immutable generation nodes. */ + readonly maxNodeReplayPageSize?: number | undefined; readonly maxProjectionBatchSize?: number | undefined; readonly nodes: KnowledgeNodeRepository; readonly operationLeases?: KnowledgeFsOperationLeaseCoordinator | undefined; readonly projections?: IndexProjectionRepository | undefined; + readonly semanticChunker?: SemanticChunker | undefined; readonly visualBuilder?: VisualEmbeddingProjectionBuilder | undefined; } @@ -97,10 +136,12 @@ export function createIncrementalReindexer({ denseBuilder, ftsBuilder, maxNodes, + maxNodeReplayPageSize, maxProjectionBatchSize, nodes, operationLeases, projections, + semanticChunker, visualBuilder, }: IncrementalReindexerOptions): IncrementalReindexer { if (!Number.isInteger(maxNodes) || maxNodes < 1) { @@ -108,11 +149,16 @@ export function createIncrementalReindexer({ } const projectionBatchSize = maxProjectionBatchSize ?? maxNodes; + const nodeReplayPageSize = maxNodeReplayPageSize ?? Math.min(maxNodes, 100); if (!Number.isInteger(projectionBatchSize) || projectionBatchSize < 1) { throw new Error("Incremental reindexer maxProjectionBatchSize must be at least 1"); } + if (!Number.isInteger(nodeReplayPageSize) || nodeReplayPageSize < 1) { + throw new Error("Incremental reindexer maxNodeReplayPageSize must be at least 1"); + } + const canUpdateProjectionStatuses = projections?.updateStatusByIds !== undefined; const updateProjectionStatus = async ({ @@ -175,7 +221,11 @@ export function createIncrementalReindexer({ } : {}), reindex: async (input) => { - validateIncrementalReindexInput(input, { visualBuilder }); + validateIncrementalReindexInput(input, { + nodes, + semanticChunker, + visualBuilder, + }); const parseArtifact = cloneParseArtifact(ParseArtifactSchema.parse(input.parseArtifact)); const publicationGenerationId = input.publicationGenerationId === undefined @@ -191,44 +241,241 @@ export function createIncrementalReindexer({ "Incremental reindexer requires a projection repository to reset failed projections", ); } + const retrievalProfile = input.retrievalProfile + ? KnowledgeSpaceRetrievalProfileSchema.parse(input.retrievalProfile) + : undefined; const reindex = async (): Promise => { input.signal?.throwIfAborted(); const storedArtifact = await artifacts.create(parseArtifact); input.signal?.throwIfAborted(); const excludedNodeOrdinals = new Set(input.excludedNodeOrdinals ?? []); - const chunkedNodes = compute - .chunkParseArtifact({ - ...(input.chunkConfig ? { config: input.chunkConfig } : {}), - knowledgeSpaceId: input.knowledgeSpaceId, + const semanticReceiptRequest = + publicationGenerationId && semanticChunker && retrievalProfile + ? semanticGenerationReceiptRequest({ + chunkConfig: input.chunkConfig, + excludedNodeOrdinals, + knowledgeSpaceId: input.knowledgeSpaceId, + language: input.language, + maxNodes, + modelSelection: retrievalProfile.reasoningModel, + parseArtifact: storedArtifact, + permissionScope: input.permissionScope ?? [], + publicationGenerationId, + semanticChunker, + }) + : undefined; + const generationReceipt = semanticReceiptRequest + ? await nodes.getGenerationReceipt?.({ + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifactId: storedArtifact.id, + publicationGenerationId: semanticReceiptRequest.publicationGenerationId, + }) + : null; + // Durable candidate retries must not call a non-deterministic semantic model again after + // generation-scoped nodes have been persisted. Those rows are immutable and already carry + // the exact chunk boundaries and joint extraction response used by the candidate. + const replayedNodes = publicationGenerationId + ? await listGenerationArtifactNodes({ + knowledgeSpaceId: input.knowledgeSpaceId, + maxNodes, + pageSize: nodeReplayPageSize, + nodes, + parseArtifactId: storedArtifact.id, + publicationGenerationId, + }) + : []; + const sourceGenerationNodes = input.reuseNodeGenerationId + ? await listGenerationArtifactNodes({ + knowledgeSpaceId: input.knowledgeSpaceId, + maxNodes, + pageSize: nodeReplayPageSize, + nodes, + parseArtifactId: storedArtifact.id, + publicationGenerationId: input.reuseNodeGenerationId, + }) + : []; + if (generationReceipt && semanticReceiptRequest) { + assertSemanticGenerationReceiptReplay({ + expected: semanticReceiptRequest, + nodes: replayedNodes, parseArtifact: storedArtifact, - ...(input.permissionScope ? { permissionScope: [...input.permissionScope] } : {}), - }) - .filter((_, ordinal) => !excludedNodeOrdinals.has(ordinal)) - .map((node) => - cloneKnowledgeNode( - publicationGenerationId - ? { - ...node, - id: deterministicChildId(publicationGenerationId, `knowledge-node:${node.id}`), - ...(input.language - ? { metadata: { ...node.metadata, language: input.language } } - : {}), - publicationGenerationId, - } - : input.language - ? { ...node, metadata: { ...node.metadata, language: input.language } } - : node, - ), + receipt: generationReceipt, + }); + } + if (input.reuseNodeGenerationId && sourceGenerationNodes.length === 0) { + throw new Error( + "Incremental reindexer could not load source generation nodes for projection-only migration", ); + } + const reusedSourceNodes = + publicationGenerationId && sourceGenerationNodes.length > 0 + ? sourceGenerationNodes.map((node) => + cloneKnowledgeNode({ + ...node, + id: deterministicChildId( + publicationGenerationId, + `knowledge-node-reuse:${node.id}`, + ), + publicationGenerationId, + }), + ) + : []; + if (replayedNodes.length > 0) { + if (reusedSourceNodes.length > 0) { + assertExactReusedGenerationReplay(replayedNodes, reusedSourceNodes); + } else if (semanticChunker && retrievalProfile) { + const replayMaxChunkChars = + input.chunkConfig?.maxChunkChars ?? semanticChunker.replayDefaults?.maxChunkChars; + const replayMaxWindowChars = semanticChunker.replayDefaults + ? Math.max( + semanticChunker.replayDefaults.maxWindowChars, + replayMaxChunkChars ?? semanticChunker.replayDefaults.maxChunkChars, + ) + : undefined; + assertValidLlmSemanticGenerationReplay({ + config: { + ...(replayMaxChunkChars === undefined + ? {} + : { maxChunkChars: replayMaxChunkChars }), + ...(replayMaxWindowChars === undefined + ? {} + : { maxWindowChars: replayMaxWindowChars }), + maxNodes: input.chunkConfig?.maxNodes ?? maxNodes, + ...(input.chunkConfig?.overlapChars === undefined + ? {} + : { overlapChars: input.chunkConfig.overlapChars }), + }, + excludedNodeOrdinals, + ...(input.language ? { language: input.language } : {}), + modelSelection: retrievalProfile.reasoningModel, + nodes: replayedNodes, + parseArtifact: storedArtifact, + permissionScope: input.permissionScope ?? [], + ...(semanticChunker.replayDefaults?.promptVersion + ? { promptVersion: semanticChunker.replayDefaults.promptVersion } + : {}), + publicationGenerationId, + }); + } + } + if ( + semanticReceiptRequest && + !generationReceipt && + replayedNodes.length === 0 && + reusedSourceNodes.length === 0 + ) { + const preflight = preflightLlmSemanticWindows({ + config: { + maxChunkChars: semanticReceiptRequest.semanticConfig.maxChunkChars, + maxNodes: semanticReceiptRequest.semanticConfig.maxNodes, + maxWindowChars: semanticReceiptRequest.semanticConfig.maxWindowChars, + overlapChars: semanticReceiptRequest.semanticConfig.overlapChars, + }, + parseArtifact: storedArtifact, + }); + assertSemanticGenerationReceiptAdmission({ + maximumChunkCount: Math.min( + preflight.unitCount, + semanticReceiptRequest.semanticConfig.maxNodes, + ), + maximumWindowCount: preflight.maximumWindowCount, + request: semanticReceiptRequest, + }); + } + const generatedNodes = + generationReceipt || replayedNodes.length > 0 || reusedSourceNodes.length > 0 + ? [] + : semanticChunker && retrievalProfile + ? await semanticChunker.chunk({ + config: { + ...(input.chunkConfig?.maxChunkChars !== undefined + ? { maxChunkChars: input.chunkConfig.maxChunkChars } + : {}), + maxNodes: input.chunkConfig?.maxNodes ?? maxNodes, + ...(input.chunkConfig?.overlapChars !== undefined + ? { overlapChars: input.chunkConfig.overlapChars } + : {}), + }, + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifact: storedArtifact, + ...(input.permissionScope ? { permissionScope: [...input.permissionScope] } : {}), + ...(publicationGenerationId ? { publicationGenerationId } : {}), + retrievalProfile, + ...(input.tenantId ? { tenantId: input.tenantId } : {}), + }) + : compute.chunkParseArtifact({ + ...(input.chunkConfig ? { config: input.chunkConfig } : {}), + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifact: storedArtifact, + ...(input.permissionScope ? { permissionScope: [...input.permissionScope] } : {}), + }); + input.signal?.throwIfAborted(); + const chunkedNodes = + replayedNodes.length > 0 + ? replayedNodes + : reusedSourceNodes.length > 0 + ? reusedSourceNodes + : generatedNodes + .filter((_, ordinal) => !excludedNodeOrdinals.has(ordinal)) + .map((node) => + cloneKnowledgeNode( + publicationGenerationId + ? { + ...node, + id: + node.publicationGenerationId === publicationGenerationId + ? node.id + : deterministicChildId( + publicationGenerationId, + `knowledge-node:${node.id}`, + ), + ...(input.language + ? { metadata: { ...node.metadata, language: input.language } } + : {}), + publicationGenerationId, + } + : input.language + ? { ...node, metadata: { ...node.metadata, language: input.language } } + : node, + ), + ); + + const generationReceiptToPersist = + semanticReceiptRequest && + !generationReceipt && + replayedNodes.length === 0 && + reusedSourceNodes.length === 0 + ? createSemanticGenerationReceipt({ + generatedNodes, + request: semanticReceiptRequest, + storedNodes: chunkedNodes, + }) + : undefined; if (chunkedNodes.length > maxNodes) { throw new Error(`Incremental reindexer node count exceeds maxNodes=${maxNodes}`); } const storedNodes = - chunkedNodes.length > 0 - ? await nodes.upsertMany(chunkedNodes.map(cloneKnowledgeNode)) - : []; + replayedNodes.length > 0 + ? replayedNodes.map(cloneKnowledgeNode) + : generationReceiptToPersist + ? (( + await nodes.completeGenerationAtomically?.({ + nodes: chunkedNodes.map(cloneKnowledgeNode), + receipt: generationReceiptToPersist, + }) + )?.nodes ?? + (() => { + throw new Error( + "Incremental reindexer requires atomic semantic generation receipts", + ); + })()) + : chunkedNodes.length > 0 + ? publicationGenerationId && nodes.upsertGenerationAtomically + ? await nodes.upsertGenerationAtomically(chunkedNodes.map(cloneKnowledgeNode)) + : await nodes.upsertMany(chunkedNodes.map(cloneKnowledgeNode)) + : []; input.signal?.throwIfAborted(); if (input.resetFailedProjections && projections) { for (const nodeBatch of chunkNodes(storedNodes, projectionBatchSize)) { @@ -320,6 +567,9 @@ export function createIncrementalReindexer({ artifact: cloneParseArtifact(storedArtifact), nodeIds: storedNodes.map((node) => node.id), nodesCreated: storedNodes.length, + ...(semanticChunker && retrievalProfile + ? { outlineArtifact: semanticOutlineArtifact(storedArtifact, storedNodes) } + : {}), projectionIds: [...projectionIds], projectionsCreated: projectionIds.length, status: "rebuilt", @@ -345,6 +595,508 @@ export function createIncrementalReindexer({ }; } +function semanticOutlineArtifact( + parseArtifact: ParseArtifact, + nodes: readonly KnowledgeNode[], +): ParseArtifact { + return ParseArtifactSchema.parse({ + ...cloneParseArtifact(parseArtifact), + elements: nodes.map((node) => { + const semantic = isPlainObject(node.metadata.semanticChunking) + ? node.metadata.semanticChunking + : undefined; + const section = semantic && isPlainObject(semantic.section) ? semantic.section : undefined; + const summary = section && typeof section.summary === "string" ? section.summary : undefined; + return { + id: node.id, + metadata: { + ...(summary ? { semanticSectionSummary: summary } : {}), + sourceKnowledgeNodeId: node.id, + }, + ...(node.sourceLocation.pageNumber === undefined + ? {} + : { pageNumber: node.sourceLocation.pageNumber }), + sectionPath: [...node.sourceLocation.sectionPath], + sourceLocation: { + endOffset: node.endOffset, + startOffset: node.startOffset, + }, + text: node.text, + type: node.kind === "table" ? "table" : node.kind === "image" ? "image" : "paragraph", + }; + }), + metadata: { + ...parseArtifact.metadata, + semanticCompilation: { + nodeCount: nodes.length, + source: "llm-semantic-v1", + }, + }, + }); +} + +interface SemanticGenerationReceiptRequest { + readonly artifactHash: string; + readonly documentAssetId: string; + readonly excludedNodeOrdinals: readonly number[]; + readonly knowledgeSpaceId: string; + readonly language?: string | undefined; + readonly modelSelection: KnowledgeSpaceRetrievalProfile["reasoningModel"]; + readonly parseArtifactId: string; + readonly permissionScope: readonly string[]; + readonly publicationGenerationId: string; + readonly requestFingerprint: string; + readonly semanticConfig: KnowledgeNodeSemanticGenerationConfig; +} + +function semanticGenerationReceiptRequest({ + chunkConfig, + excludedNodeOrdinals, + knowledgeSpaceId, + language, + maxNodes, + modelSelection, + parseArtifact, + permissionScope, + publicationGenerationId, + semanticChunker, +}: { + readonly chunkConfig?: ChunkConfig | undefined; + readonly excludedNodeOrdinals: ReadonlySet; + readonly knowledgeSpaceId: string; + readonly language?: string | undefined; + readonly maxNodes: number; + readonly modelSelection: KnowledgeSpaceRetrievalProfile["reasoningModel"]; + readonly parseArtifact: ParseArtifact; + readonly permissionScope: readonly string[]; + readonly publicationGenerationId: string; + readonly semanticChunker: SemanticChunker; +}): SemanticGenerationReceiptRequest { + if (!semanticChunker.replayDefaults) { + throw new Error( + "Incremental reindexer semantic chunker must expose replay defaults for durable receipts", + ); + } + const maxChunkChars = chunkConfig?.maxChunkChars ?? semanticChunker.replayDefaults.maxChunkChars; + const semanticConfig: KnowledgeNodeSemanticGenerationConfig = { + maxChunkChars, + maxNodes: chunkConfig?.maxNodes ?? maxNodes, + maxWindowChars: Math.max(semanticChunker.replayDefaults.maxWindowChars, maxChunkChars), + overlapChars: chunkConfig?.overlapChars ?? 0, + promptVersion: semanticChunker.replayDefaults.promptVersion, + }; + const request = { + artifactHash: parseArtifact.artifactHash, + documentAssetId: parseArtifact.documentAssetId, + excludedNodeOrdinals: [...excludedNodeOrdinals].sort((left, right) => left - right), + knowledgeSpaceId, + ...(language ? { language } : {}), + modelSelection, + parseArtifactId: parseArtifact.id, + permissionScope: [...permissionScope], + publicationGenerationId, + semanticConfig, + }; + if (!knowledgeSpaceId.trim()) { + throw new Error("Incremental reindexer semantic receipt knowledgeSpaceId is required"); + } + return { + ...request, + requestFingerprint: sha256StableJson(request), + }; +} + +function createSemanticGenerationReceipt({ + generatedNodes, + request, + storedNodes, +}: { + readonly generatedNodes: readonly KnowledgeNode[]; + readonly request: SemanticGenerationReceiptRequest; + readonly storedNodes: readonly KnowledgeNode[]; +}): KnowledgeNodeGenerationReceipt { + const { completionCatalog, windowManifest } = semanticGenerationWindowManifest(generatedNodes); + const responseFingerprint = sha256StableJson({ completionCatalog, windowManifest }); + return { + artifactHash: request.artifactHash, + completionCatalog, + documentAssetId: request.documentAssetId, + documentChunkCount: generatedNodes.length, + excludedNodeOrdinals: [...request.excludedNodeOrdinals], + knowledgeSpaceId: request.knowledgeSpaceId, + ...(request.language ? { language: request.language } : {}), + modelSelection: request.modelSelection, + parseArtifactId: request.parseArtifactId, + permissionScope: [...request.permissionScope], + promptResponseFingerprint: sha256StableJson({ + requestFingerprint: request.requestFingerprint, + responseFingerprint, + }), + publicationGenerationId: request.publicationGenerationId, + requestFingerprint: request.requestFingerprint, + responseFingerprint, + schemaVersion: 1, + semanticConfig: request.semanticConfig, + storedNodeCount: storedNodes.length, + storedResponseFingerprint: semanticNodesResponseFingerprint(storedNodes), + windowManifest, + }; +} + +function assertSemanticGenerationReceiptAdmission({ + maximumChunkCount, + maximumWindowCount, + request, +}: { + readonly maximumChunkCount: number; + readonly maximumWindowCount: number; + readonly request: SemanticGenerationReceiptRequest; +}): void { + if ( + request.excludedNodeOrdinals.some( + (ordinal) => !Number.isSafeInteger(ordinal) || ordinal < 0 || ordinal >= maximumChunkCount, + ) + ) { + throw new Error( + "Incremental reindexer semantic generation exclusions exceed the canonical chunk upper bound", + ); + } + const emptyFingerprint = `sha256:${"0".repeat(64)}`; + const emptyReceipt: KnowledgeNodeGenerationReceipt = { + artifactHash: request.artifactHash, + completionCatalog: [], + documentAssetId: request.documentAssetId, + documentChunkCount: maximumChunkCount, + excludedNodeOrdinals: [...request.excludedNodeOrdinals], + knowledgeSpaceId: request.knowledgeSpaceId, + ...(request.language ? { language: request.language } : {}), + modelSelection: request.modelSelection, + parseArtifactId: request.parseArtifactId, + permissionScope: [...request.permissionScope], + promptResponseFingerprint: emptyFingerprint, + publicationGenerationId: request.publicationGenerationId, + requestFingerprint: request.requestFingerprint, + responseFingerprint: emptyFingerprint, + schemaVersion: 1, + semanticConfig: request.semanticConfig, + storedNodeCount: maximumChunkCount, + storedResponseFingerprint: emptyFingerprint, + windowManifest: [], + }; + const maximumBytes = maximumKnowledgeNodeGenerationReceiptSerializedBytes({ + emptyReceipt, + maximumChunkCount, + maximumWindowCount, + }); + if (maximumBytes > MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES) { + throw new Error( + `Incremental reindexer semantic generation receipt admission exceeds maxBytes=${MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES} (upperBoundBytes=${maximumBytes})`, + ); + } +} + +function assertSemanticGenerationReceiptReplay({ + expected, + nodes, + parseArtifact, + receipt, +}: { + readonly expected: SemanticGenerationReceiptRequest; + readonly nodes: readonly KnowledgeNode[]; + readonly parseArtifact: ParseArtifact; + readonly receipt: KnowledgeNodeGenerationReceipt; +}): void { + const expectedIdentity = { + artifactHash: expected.artifactHash, + documentAssetId: expected.documentAssetId, + excludedNodeOrdinals: expected.excludedNodeOrdinals, + knowledgeSpaceId: expected.knowledgeSpaceId, + ...(expected.language ? { language: expected.language } : {}), + modelSelection: expected.modelSelection, + parseArtifactId: expected.parseArtifactId, + permissionScope: expected.permissionScope, + publicationGenerationId: expected.publicationGenerationId, + requestFingerprint: expected.requestFingerprint, + semanticConfig: expected.semanticConfig, + }; + const receiptIdentity = { + artifactHash: receipt.artifactHash, + documentAssetId: receipt.documentAssetId, + excludedNodeOrdinals: receipt.excludedNodeOrdinals, + knowledgeSpaceId: receipt.knowledgeSpaceId, + ...(receipt.language ? { language: receipt.language } : {}), + modelSelection: receipt.modelSelection, + parseArtifactId: receipt.parseArtifactId, + permissionScope: receipt.permissionScope, + publicationGenerationId: receipt.publicationGenerationId, + requestFingerprint: receipt.requestFingerprint, + semanticConfig: receipt.semanticConfig, + }; + if (stableJson(expectedIdentity) !== stableJson(receiptIdentity)) { + throw new Error("LLM semantic replay validation failed: generation receipt request mismatch"); + } + const responseFingerprint = sha256StableJson({ + completionCatalog: receipt.completionCatalog, + windowManifest: receipt.windowManifest, + }); + const promptResponseFingerprint = sha256StableJson({ + requestFingerprint: receipt.requestFingerprint, + responseFingerprint, + }); + if ( + receipt.responseFingerprint !== responseFingerprint || + receipt.promptResponseFingerprint !== promptResponseFingerprint + ) { + throw new Error( + "LLM semantic replay validation failed: generation receipt provenance mismatch", + ); + } + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: receipt.completionCatalog, + config: { + maxChunkChars: receipt.semanticConfig.maxChunkChars, + maxNodes: receipt.semanticConfig.maxNodes, + maxWindowChars: receipt.semanticConfig.maxWindowChars, + overlapChars: receipt.semanticConfig.overlapChars, + }, + documentChunkCount: receipt.documentChunkCount, + modelSelection: receipt.modelSelection, + parseArtifact, + promptVersion: receipt.semanticConfig.promptVersion, + windowManifest: receipt.windowManifest, + }); + if ( + receipt.storedNodeCount !== nodes.length || + receipt.storedResponseFingerprint !== semanticNodesResponseFingerprint(nodes) + ) { + throw new Error( + "LLM semantic replay validation failed: generation receipt stored-node mismatch", + ); + } + const manifestChunks = new Map( + receipt.windowManifest.flatMap((window) => + window.chunkRanges.map( + (unitRange, offset) => + [window.firstChunkIndex + offset, { unitRange, windowId: window.windowId }] as const, + ), + ), + ); + for (const node of nodes) { + const chunkIndex = node.metadata.chunkIndex; + const expectedChunk = + Number.isSafeInteger(chunkIndex) && manifestChunks.get(chunkIndex as number); + const semanticMarker = node.metadata.semanticChunking; + const unitRange = isPlainObject(semanticMarker) + ? semanticMarkerUnitRangeReceipt(semanticMarker.unitRange) + : undefined; + const markerWindowId = isPlainObject(semanticMarker) ? semanticMarker.windowId : undefined; + if ( + !expectedChunk || + !unitRange || + markerWindowId !== expectedChunk.windowId || + stableJson(unitRange) !== stableJson(expectedChunk.unitRange) + ) { + throw new Error("LLM semantic replay validation failed: generation receipt node mismatch"); + } + } +} + +function semanticGenerationWindowManifest(nodes: readonly KnowledgeNode[]): { + readonly completionCatalog: KnowledgeNodeGenerationCompletionReceipt[]; + readonly windowManifest: KnowledgeNodeGenerationWindowReceipt[]; +} { + const completionCatalog: KnowledgeNodeGenerationCompletionReceipt[] = []; + const completionIndexes = new Map(); + const windows = new Map< + string, + { + readonly payloads: Readonly>[]; + readonly receipt: KnowledgeNodeGenerationWindowReceipt; + } + >(); + const ordered = [...nodes].sort( + (left, right) => + Number(left.metadata.chunkIndex) - Number(right.metadata.chunkIndex) || + left.id.localeCompare(right.id), + ); + for (const [ordinal, node] of ordered.entries()) { + const marker = node.metadata.semanticChunking; + const unitRange = isPlainObject(marker) + ? semanticMarkerUnitRangeReceipt(marker.unitRange) + : undefined; + const coreUnitRange = isPlainObject(marker) + ? semanticMarkerUnitRangeReceipt(marker.windowCoreUnitRange) + : undefined; + const committedUnitRange = isPlainObject(marker) + ? semanticMarkerUnitRangeReceipt(marker.windowCommittedUnitRange) + : undefined; + const lookAheadUnitRange = isPlainObject(marker) + ? semanticMarkerUnitRangeReceipt(marker.windowLookAheadUnitRange) + : undefined; + const windowId = isPlainObject(marker) ? marker.windowId : undefined; + const inputFingerprint = isPlainObject(marker) ? marker.inputFingerprint : undefined; + const chunkIndex = node.metadata.chunkIndex; + if ( + !isPlainObject(marker) || + !unitRange || + !coreUnitRange || + !committedUnitRange || + typeof windowId !== "string" || + !windowId.trim() || + typeof inputFingerprint !== "string" || + !/^sha256:[a-f0-9]{64}$/u.test(inputFingerprint) || + !Number.isSafeInteger(chunkIndex) || + chunkIndex !== ordinal + ) { + throw new Error( + "Incremental reindexer cannot build semantic window receipt from node marker", + ); + } + const completion = semanticGenerationCompletionReceipt(marker); + let completionIndex = completionIndexes.get(completion.fingerprint); + if (completionIndex === undefined) { + completionIndex = completionCatalog.length; + completionIndexes.set(completion.fingerprint, completionIndex); + completionCatalog.push(completion); + } else if (stableJson(completionCatalog[completionIndex]) !== stableJson(completion)) { + throw new Error("Incremental reindexer semantic completion fingerprint collision"); + } + const payload = semanticNodeResponsePayload(node); + const existing = windows.get(windowId); + if (existing) { + if ( + existing.receipt.inputFingerprint !== inputFingerprint || + existing.receipt.completionIndex !== completionIndex || + stableJson(existing.receipt.coreUnitRange) !== stableJson(coreUnitRange) || + stableJson(existing.receipt.committedUnitRange) !== stableJson(committedUnitRange) || + stableJson(existing.receipt.lookAheadUnitRange) !== stableJson(lookAheadUnitRange) || + existing.receipt.firstChunkIndex + existing.receipt.chunkRanges.length !== chunkIndex + ) { + throw new Error("Incremental reindexer semantic window provenance is inconsistent"); + } + const chunkRanges = [...existing.receipt.chunkRanges, unitRange]; + const payloads = [...existing.payloads, payload]; + windows.set(windowId, { + payloads, + receipt: { + ...existing.receipt, + chunkRanges, + responseFingerprint: sha256StableJson(payloads), + }, + }); + continue; + } + windows.set(windowId, { + payloads: [payload], + receipt: { + chunkRanges: [unitRange], + committedUnitRange, + completionIndex, + coreUnitRange, + firstChunkIndex: chunkIndex as number, + inputFingerprint, + ...(lookAheadUnitRange ? { lookAheadUnitRange } : {}), + responseFingerprint: sha256StableJson([payload]), + windowId, + }, + }); + } + return { + completionCatalog, + windowManifest: [...windows.values()].map(({ receipt }) => receipt), + }; +} + +function semanticGenerationCompletionReceipt( + marker: Readonly>, +): KnowledgeNodeGenerationCompletionReceipt { + const completion = marker.completion; + const actual = isPlainObject(completion) ? completion.actual : undefined; + if (!isPlainObject(actual)) { + throw new Error("Incremental reindexer semantic completion identity is missing"); + } + const identity = { + ...optionalSemanticCompletionField(actual.model, "actualModel"), + ...optionalSemanticCompletionField(actual.provider, "actualProvider"), + ...optionalSemanticCompletionField(actual.finishReason, "finishReason"), + ...optionalSemanticCompletionField(marker.provider, "transportProvider"), + }; + return { + fingerprint: llmSemanticCompletionFingerprint(identity), + ...identity, + }; +} + +function optionalSemanticCompletionField( + value: unknown, + field: "actualModel" | "actualProvider" | "finishReason" | "transportProvider", +): Partial> { + if (value === undefined) return {}; + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Incremental reindexer semantic completion ${field} is invalid`); + } + return { [field]: value.trim() } as Partial>; +} + +function semanticMarkerUnitRangeReceipt( + value: unknown, +): KnowledgeNodeGenerationUnitRangeReceipt | undefined { + if (!isPlainObject(value)) return undefined; + const startUnitId = value.startUnitId; + const endUnitId = value.endUnitId; + return typeof startUnitId === "string" && + startUnitId.trim() && + typeof endUnitId === "string" && + endUnitId.trim() + ? [startUnitId.trim(), endUnitId.trim()] + : undefined; +} + +function semanticNodesResponseFingerprint(nodes: readonly KnowledgeNode[]): string { + return sha256StableJson( + [...nodes] + .sort( + (left, right) => + Number(left.metadata.chunkIndex) - Number(right.metadata.chunkIndex) || + left.id.localeCompare(right.id), + ) + .map(semanticNodeResponsePayload), + ); +} + +function semanticNodeResponsePayload(node: KnowledgeNode): Readonly> { + return { + chunkIndex: node.metadata.chunkIndex, + endOffset: node.endOffset, + entityExtraction: node.metadata.entityExtraction, + extractedEntities: extractionResponseWithoutQuality(node.metadata.extractedEntities), + extractedRelations: extractionResponseWithoutQuality(node.metadata.extractedRelations), + id: node.id, + kind: node.kind, + relationExtraction: node.metadata.relationExtraction, + semanticChunking: node.metadata.semanticChunking, + sourceLocation: node.sourceLocation, + startOffset: node.startOffset, + text: node.text, + }; +} + +function extractionResponseWithoutQuality(value: unknown): unknown { + if (!Array.isArray(value)) return value; + return value.map((item) => { + if (!isPlainObject(item)) return item; + const { quality: _quality, ...response } = JSON.parse(JSON.stringify(item)) as Record< + string, + unknown + >; + return response; + }); +} + +function sha256StableJson(value: unknown): string { + return `sha256:${createHash("sha256").update(stableJson(value)).digest("hex")}`; +} + function chunkNodes(nodes: readonly KnowledgeNode[], size: number) { const chunks: KnowledgeNode[][] = []; @@ -355,6 +1107,42 @@ function chunkNodes(nodes: readonly KnowledgeNode[], size: number) { return chunks; } +async function listGenerationArtifactNodes({ + knowledgeSpaceId, + maxNodes, + nodes, + pageSize, + parseArtifactId, + publicationGenerationId, +}: { + readonly knowledgeSpaceId: string; + readonly maxNodes: number; + readonly nodes: KnowledgeNodeRepository; + readonly pageSize: number; + readonly parseArtifactId: string; + readonly publicationGenerationId: string; +}): Promise { + const collected: KnowledgeNode[] = []; + let cursor: Awaited>["nextCursor"]; + + do { + const page = await nodes.listByArtifact({ + ...(cursor ? { cursor } : {}), + knowledgeSpaceId, + limit: Math.min(pageSize, maxNodes - collected.length), + parseArtifactId, + publicationGenerationId, + }); + collected.push(...page.items.map(cloneKnowledgeNode)); + cursor = page.nextCursor; + if (cursor && collected.length >= maxNodes) { + throw new Error(`Incremental reindexer node count exceeds maxNodes=${maxNodes}`); + } + } while (cursor); + + return collected; +} + function chunkStrings(values: readonly string[], size: number): string[][] { const chunks: string[][] = []; @@ -365,6 +1153,20 @@ function chunkStrings(values: readonly string[], size: number): string[][] { return chunks; } +function assertExactReusedGenerationReplay( + replayedNodes: readonly KnowledgeNode[], + expectedNodes: readonly KnowledgeNode[], +): void { + if ( + replayedNodes.length !== expectedNodes.length || + replayedNodes.some((node, index) => stableJson(node) !== stableJson(expectedNodes[index])) + ) { + throw new Error( + "Incremental reindexer found an incomplete projection-only node generation replay", + ); + } +} + function validateReindexProjectionDimensions( projections: readonly IndexProjection[], observedVectorSpaces: Map, @@ -409,7 +1211,11 @@ function validateReindexProjectionDimensions( function validateIncrementalReindexInput( input: IncrementalReindexInput, - { visualBuilder }: Pick, + { + nodes, + semanticChunker, + visualBuilder, + }: Pick, ): void { if (!input.knowledgeSpaceId.trim()) { throw new Error("Incremental reindexer knowledgeSpaceId is required"); @@ -427,9 +1233,45 @@ function validateIncrementalReindexInput( PublicationGenerationIdSchema.parse(input.publicationGenerationId); } - if (visualBuilder && !input.visualModel?.trim()) { + if (input.reuseNodeGenerationId !== undefined) { + PublicationGenerationIdSchema.parse(input.reuseNodeGenerationId); + if (!input.publicationGenerationId) { + throw new Error( + "Incremental reindexer reuseNodeGenerationId requires publicationGenerationId", + ); + } + if (input.reuseNodeGenerationId === input.publicationGenerationId) { + throw new Error("Incremental reindexer source and target node generations must be different"); + } + } + + if (input.skipDense && (input.denseModel?.trim() || input.embeddingProfile)) { + throw new Error("Incremental reindexer skipDense cannot include dense model configuration"); + } + + if (input.skipVisual && input.visualModel?.trim()) { + throw new Error("Incremental reindexer skipVisual cannot include visual model configuration"); + } + + if (visualBuilder && !input.skipVisual && !input.visualModel?.trim()) { throw new Error( "Incremental reindexer visualModel is required when visualBuilder is configured", ); } + + if (semanticChunker && input.retrievalProfile && !input.tenantId?.trim()) { + throw new Error( + "Incremental reindexer tenantId is required for profile-scoped semantic chunking", + ); + } + if ( + semanticChunker && + input.retrievalProfile && + input.publicationGenerationId && + (!nodes.completeGenerationAtomically || !nodes.getGenerationReceipt) + ) { + throw new Error( + "Incremental reindexer generation-scoped semantic chunking requires durable generation receipts", + ); + } } diff --git a/knowledge-fs/packages/api/src/index.ts b/knowledge-fs/packages/api/src/index.ts index 3026054e7d0..2f70f8b7821 100644 --- a/knowledge-fs/packages/api/src/index.ts +++ b/knowledge-fs/packages/api/src/index.ts @@ -112,6 +112,7 @@ export * from "./document-multimodal-manifest-builder"; export * from "./document-multimodal-manifest-repository"; export * from "./document-pdf-rasterizer"; export * from "./document-outline-builder"; +export * from "./document-layout-recomposer"; export * from "./document-outline-evaluation"; export * from "./document-outline-repository"; export * from "./document-offsets"; @@ -238,6 +239,7 @@ export * from "./llm-community-summary-provider"; export * from "./llm-entity-extraction-provider"; export * from "./llm-multimodal-answer-provider"; export * from "./llm-relation-extraction-provider"; +export * from "./llm-semantic-chunker"; export * from "./knowledge-fs-errors"; export * from "./knowledge-fs-handlers"; export * from "./knowledge-fs-command-registry"; diff --git a/knowledge-fs/packages/api/src/knowledge-node-generation-receipt-repository.test.ts b/knowledge-fs/packages/api/src/knowledge-node-generation-receipt-repository.test.ts new file mode 100644 index 00000000000..bcda7755a28 --- /dev/null +++ b/knowledge-fs/packages/api/src/knowledge-node-generation-receipt-repository.test.ts @@ -0,0 +1,504 @@ +import type { KnowledgeNode } from "@knowledge/core"; +import { KnowledgeNodeSchema } from "@knowledge/core"; +import { describe, expect, it } from "vitest"; + +import { + type KnowledgeNodeGenerationReceipt, + KnowledgeNodeGenerationReceiptConflictError, + createInMemoryKnowledgeNodeRepository, +} from "./knowledge-node-repository"; +import { + MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES, + knowledgeNodeGenerationReceiptSerializedBytes, + llmSemanticCompletionFingerprint, + maximumKnowledgeNodeGenerationReceiptSerializedBytes, +} from "./semantic-generation-receipt"; + +const KNOWLEDGE_SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c40"; +const DOCUMENT_ASSET_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c41"; +const PARSE_ARTIFACT_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42"; +const PUBLICATION_GENERATION_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca1"; + +interface ReceiptPatch { + readonly path: readonly string[]; + readonly value: unknown; +} + +function knowledgeNode(chunkIndex: number): KnowledgeNode { + const startOffset = chunkIndex * 20; + return KnowledgeNodeSchema.parse({ + artifactHash: "a".repeat(64), + documentAssetId: DOCUMENT_ASSET_ID, + endOffset: startOffset + 12, + id: `018f0d60-7a49-7cc2-9c1b-${(0x5b36f18f8a00 + chunkIndex).toString(16).padStart(12, "0")}`, + kind: "chunk", + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + metadata: { chunkIndex }, + parseArtifactId: PARSE_ARTIFACT_ID, + permissionScope: ["tenant:tenant-1"], + publicationGenerationId: PUBLICATION_GENERATION_ID, + sourceLocation: { endOffset: startOffset + 12, sectionPath: [], startOffset }, + startOffset, + text: `chunk ${chunkIndex}`, + }); +} + +function semanticGenerationReceipt( + overrides: Partial = {}, +): KnowledgeNodeGenerationReceipt { + const completion = { + actualModel: "reasoning-v1", + actualProvider: "plugin-daemon", + finishReason: "stop", + transportProvider: "plugin-daemon", + }; + return { + artifactHash: "a".repeat(64), + completionCatalog: [ + { ...completion, fingerprint: llmSemanticCompletionFingerprint(completion) }, + ], + documentAssetId: DOCUMENT_ASSET_ID, + documentChunkCount: 1, + excludedNodeOrdinals: [0], + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + modelSelection: { + model: "reasoning-v1", + pluginId: "reasoning-plugin", + provider: "plugin-daemon", + }, + parseArtifactId: PARSE_ARTIFACT_ID, + permissionScope: ["tenant:tenant-1"], + promptResponseFingerprint: `sha256:${"b".repeat(64)}`, + publicationGenerationId: PUBLICATION_GENERATION_ID, + requestFingerprint: `sha256:${"c".repeat(64)}`, + responseFingerprint: `sha256:${"d".repeat(64)}`, + schemaVersion: 1, + semanticConfig: { + maxChunkChars: 1_200, + maxNodes: 20_000, + maxWindowChars: 4_800, + overlapChars: 0, + promptVersion: "semantic-v2", + }, + storedNodeCount: 0, + storedResponseFingerprint: `sha256:${"e".repeat(64)}`, + windowManifest: [ + { + chunkRanges: [["u-000000-000000", "u-000000-000000"]], + committedUnitRange: ["u-000000-000000", "u-000000-000000"], + completionIndex: 0, + coreUnitRange: ["u-000000-000000", "u-000000-000000"], + firstChunkIndex: 0, + inputFingerprint: `sha256:${"1".repeat(64)}`, + responseFingerprint: `sha256:${"2".repeat(64)}`, + windowId: "window-000000", + }, + ], + ...overrides, + }; +} + +function patch(path: string, value: unknown): ReceiptPatch { + return { path: path.split("."), value }; +} + +function applyReceiptPatch(target: Record, input: ReceiptPatch): void { + const finalSegment = input.path.at(-1); + if (!finalSegment) throw new Error("Receipt patch path is required"); + let cursor: unknown = target; + for (const segment of input.path.slice(0, -1)) { + cursor = Array.isArray(cursor) + ? cursor[Number(segment)] + : isMutableRecord(cursor) + ? cursor[segment] + : undefined; + if (cursor === undefined) throw new Error(`Receipt patch path is invalid: ${input.path}`); + } + if (Array.isArray(cursor)) { + cursor[Number(finalSegment)] = input.value; + return; + } + if (!isMutableRecord(cursor)) { + throw new Error(`Receipt patch target is invalid: ${input.path}`); + } + cursor[finalSegment] = input.value; +} + +function isMutableRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +describe("KnowledgeNode semantic generation receipts", () => { + it("persists an all-excluded generation and replays it without node rows", async () => { + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 2, + }); + const receipt = semanticGenerationReceipt(); + + await expect( + repository.completeGenerationAtomically?.({ nodes: [], receipt }), + ).resolves.toEqual({ nodes: [], receipt }); + await expect( + repository.getGenerationReceipt?.({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifactId: PARSE_ARTIFACT_ID, + publicationGenerationId: PUBLICATION_GENERATION_ID, + }), + ).resolves.toEqual(receipt); + await expect( + repository.completeGenerationAtomically?.({ nodes: [], receipt }), + ).resolves.toEqual({ nodes: [], receipt }); + }); + + it("rejects a conflicting replay for the same immutable generation", async () => { + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 2, + }); + await repository.completeGenerationAtomically?.({ + nodes: [], + receipt: semanticGenerationReceipt(), + }); + + await expect( + repository.completeGenerationAtomically?.({ + nodes: [], + receipt: semanticGenerationReceipt({ language: "zh-CN" }), + }), + ).rejects.toBeInstanceOf(KnowledgeNodeGenerationReceiptConflictError); + }); + + it("atomically persists a complete immutable generation beyond ordinary batch size", async () => { + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 2, + maxNodes: 2, + }); + const nodes = [knowledgeNode(0), knowledgeNode(1)]; + const baseWindow = semanticGenerationReceipt().windowManifest[0]; + if (!baseWindow) throw new Error("semantic receipt fixture requires one window"); + const receipt = semanticGenerationReceipt({ + documentChunkCount: 2, + excludedNodeOrdinals: [], + storedNodeCount: 2, + windowManifest: [ + { + ...baseWindow, + chunkRanges: [ + ["u-000000-000000", "u-000000-000000"], + ["u-000000-000001", "u-000000-000001"], + ], + }, + ], + }); + + await expect(repository.upsertMany(nodes)).rejects.toThrow("maxBatchSize=1"); + await expect(repository.completeGenerationAtomically?.({ nodes, receipt })).resolves.toEqual({ + nodes, + receipt, + }); + }); + + it("rejects receipts whose node count or identity does not match persisted nodes", async () => { + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 2, + }); + + await expect( + repository.completeGenerationAtomically?.({ + nodes: [knowledgeNode(0)], + receipt: semanticGenerationReceipt(), + }), + ).rejects.toThrow("storedNodeCount does not match nodes"); + + const storedReceipt = semanticGenerationReceipt({ + documentChunkCount: 1, + excludedNodeOrdinals: [], + storedNodeCount: 1, + }); + for (const node of [ + KnowledgeNodeSchema.parse({ ...knowledgeNode(0), artifactHash: "b".repeat(64) }), + KnowledgeNodeSchema.parse({ + ...knowledgeNode(0), + metadata: { ...knowledgeNode(0).metadata, chunkIndex: "invalid" }, + }), + ]) { + await expect( + repository.completeGenerationAtomically?.({ nodes: [node], receipt: storedReceipt }), + ).rejects.toThrow("identity does not match nodes"); + } + await expect( + repository.completeGenerationAtomically?.({ + nodes: [KnowledgeNodeSchema.parse({ ...knowledgeNode(0), metadata: { chunkIndex: 1 } })], + receipt: storedReceipt, + }), + ).rejects.toThrow("chunk indexes do not match nodes"); + }); + + it("round-trips minimal terminal identity and a manifest without look-ahead", async () => { + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 2, + }); + const baseWindow = semanticGenerationReceipt().windowManifest[0]; + if (!baseWindow) throw new Error("semantic receipt fixture requires one window"); + const completion = {}; + const receipt = semanticGenerationReceipt({ + completionCatalog: [ + { fingerprint: llmSemanticCompletionFingerprint(completion), ...completion }, + ], + windowManifest: [{ ...baseWindow, lookAheadUnitRange: undefined }], + }); + + await expect( + repository.completeGenerationAtomically?.({ nodes: [], receipt }), + ).resolves.toEqual({ nodes: [], receipt }); + await expect( + repository.getGenerationReceipt?.({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifactId: PARSE_ARTIFACT_ID, + publicationGenerationId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2ca2", + }), + ).resolves.toBeNull(); + }); + + it("enforces the durable receipt byte limit before persistence", async () => { + const base = semanticGenerationReceipt({ permissionScope: ["x"] }); + const baseBytes = knowledgeNodeGenerationReceiptSerializedBytes(base); + const oversized = semanticGenerationReceipt({ + permissionScope: [ + `x${"y".repeat(MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES + 1 - baseBytes)}`, + ], + }); + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 1, + maxListLimit: 1, + maxNodes: 2, + }); + + await expect( + repository.completeGenerationAtomically?.({ nodes: [], receipt: oversized }), + ).rejects.toThrow(`exceeds maxBytes=${MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES}`); + }); + + it("fails closed for malformed receipt envelopes, identities, windows, and ranges", async () => { + const repository = createInMemoryKnowledgeNodeRepository({ + maxBatchSize: 2, + maxListLimit: 2, + maxNodes: 4, + }); + const valid = semanticGenerationReceipt(); + const invalidCases: Array<{ + readonly expected: string; + readonly patches: readonly ReceiptPatch[]; + }> = [ + { expected: "schemaVersion must be 1", patches: [patch("schemaVersion", 2)] }, + { expected: "artifactHash is invalid", patches: [patch("artifactHash", "bad")] }, + { + expected: "maxChunkChars must be at least 1", + patches: [patch("semanticConfig.maxChunkChars", 0)], + }, + { + expected: "maxNodes must be at least 1", + patches: [patch("semanticConfig.maxNodes", 0)], + }, + { + expected: "maxWindowChars must be at least 1", + patches: [patch("semanticConfig.maxWindowChars", 0)], + }, + { + expected: "overlapChars is invalid", + patches: [patch("semanticConfig.overlapChars", -1)], + }, + { + expected: "overlapChars is invalid", + patches: [patch("semanticConfig.overlapChars", 1_200)], + }, + { + expected: "semantic config is invalid", + patches: [patch("semanticConfig.maxWindowChars", 1_199)], + }, + { + expected: "semantic config is invalid", + patches: [patch("semanticConfig.promptVersion", " ")], + }, + { + expected: "node counts are invalid", + patches: [patch("documentChunkCount", -1)], + }, + { + expected: "node counts are invalid", + patches: [patch("storedNodeCount", 2)], + }, + { + expected: "exclusions are invalid", + patches: [patch("excludedNodeOrdinals", [-1])], + }, + { + expected: "exclusions are invalid", + patches: [patch("excludedNodeOrdinals", [1])], + }, + { + expected: "exclusions are invalid", + patches: [patch("documentChunkCount", 2), patch("excludedNodeOrdinals", [1, 0])], + }, + { + expected: "fingerprint is invalid", + patches: [patch("requestFingerprint", "sha256:bad")], + }, + { + expected: "permissionScope is invalid", + patches: [patch("permissionScope", [" "])], + }, + { expected: "language is invalid", patches: [patch("language", " ")] }, + { + expected: "completion catalog is invalid", + patches: [patch("completionCatalog", "not-an-array")], + }, + { + expected: "completion identity is invalid", + patches: [patch("completionCatalog", [null])], + }, + { + expected: "actualModel is invalid", + patches: [patch("completionCatalog.0.actualModel", " ")], + }, + { + expected: "actualProvider is invalid", + patches: [patch("completionCatalog.0.actualProvider", "x".repeat(256))], + }, + { + expected: "finishReason is invalid", + patches: [patch("completionCatalog.0.finishReason", "x".repeat(65))], + }, + { + expected: "transportProvider is invalid", + patches: [patch("completionCatalog.0.transportProvider", 42)], + }, + { + expected: "completion identity is invalid", + patches: [patch("completionCatalog.0.fingerprint", `sha256:${"0".repeat(64)}`)], + }, + { + expected: "completion identity is invalid", + patches: [ + patch("completionCatalog", [valid.completionCatalog[0], valid.completionCatalog[0]]), + ], + }, + { + expected: "window manifest is incomplete", + patches: [patch("windowManifest", [])], + }, + { + expected: "window manifest is incomplete", + patches: [patch("completionCatalog", [])], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest", [null])], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest.0.windowId", "window-x")], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest.0.inputFingerprint", "bad")], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest.0.responseFingerprint", "bad")], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest.0.completionIndex", -1)], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest.0.firstChunkIndex", 1)], + }, + { + expected: "window manifest is invalid", + patches: [patch("windowManifest.0.chunkRanges", [])], + }, + { + expected: "window unit range is invalid", + patches: [patch("windowManifest.0.coreUnitRange", ["bad", "bad"])], + }, + { + expected: "window unit range is invalid", + patches: [patch("windowManifest.0.chunkRanges", [["u-000000-000000"]])], + }, + { + expected: "window chunks do not cover the document", + patches: [ + patch("documentChunkCount", 2), + patch("storedNodeCount", 1), + patch("excludedNodeOrdinals", [1]), + ], + }, + ]; + + for (const { expected, patches } of invalidCases) { + const receipt = structuredClone(valid) as unknown as Record; + for (const receiptPatch of patches) applyReceiptPatch(receipt, receiptPatch); + await expect( + repository.completeGenerationAtomically?.({ + nodes: [], + receipt: receipt as unknown as KnowledgeNodeGenerationReceipt, + }), + ).rejects.toThrow(expected); + } + }); + + it("computes exact empty and bounded receipt admission sizes", () => { + const empty = semanticGenerationReceipt({ + completionCatalog: [], + documentChunkCount: 0, + excludedNodeOrdinals: [], + storedNodeCount: 0, + windowManifest: [], + }); + + expect( + maximumKnowledgeNodeGenerationReceiptSerializedBytes({ + emptyReceipt: empty, + maximumChunkCount: 0, + maximumWindowCount: 0, + }), + ).toBe(knowledgeNodeGenerationReceiptSerializedBytes(empty)); + expect( + maximumKnowledgeNodeGenerationReceiptSerializedBytes({ + emptyReceipt: empty, + maximumChunkCount: 2, + maximumWindowCount: 1, + }), + ).toBeGreaterThan(knowledgeNodeGenerationReceiptSerializedBytes(empty)); + expect(() => + maximumKnowledgeNodeGenerationReceiptSerializedBytes({ + emptyReceipt: semanticGenerationReceipt(), + maximumChunkCount: 1, + maximumWindowCount: 1, + }), + ).toThrow("requires empty dynamic arrays"); + for (const [maximumChunkCount, maximumWindowCount] of [ + [-1, 0], + [0, -1], + [0, 1], + ] as const) { + expect(() => + maximumKnowledgeNodeGenerationReceiptSerializedBytes({ + emptyReceipt: empty, + maximumChunkCount, + maximumWindowCount, + }), + ).toThrow("admission bounds are invalid"); + } + }); +}); diff --git a/knowledge-fs/packages/api/src/knowledge-node-repository.ts b/knowledge-fs/packages/api/src/knowledge-node-repository.ts index 069f7cd8a78..3453b3ce219 100644 --- a/knowledge-fs/packages/api/src/knowledge-node-repository.ts +++ b/knowledge-fs/packages/api/src/knowledge-node-repository.ts @@ -7,8 +7,11 @@ import type { } from "@knowledge/core"; import { KnowledgeNodeSchema, + KnowledgeSpaceModelSelectionSchema, PUBLICATION_GENERATION_ID_SENTINEL, PublicationGenerationIdSchema, + UuidSchema, + stableJson, } from "@knowledge/core"; import { numberColumn, optionalStringColumn, stringColumn } from "./database-row-utils"; @@ -23,7 +26,36 @@ import { assertExactGenerationReplay, assertInMemoryGenerationNotPublished, } from "./generation-immutability"; -import { cloneJsonObject, jsonObjectColumn, jsonStringArrayColumn } from "./json-utils"; +import { + cloneJsonObject, + isPlainObject, + jsonObjectColumn, + jsonStringArrayColumn, +} from "./json-utils"; +import { + type KnowledgeNodeGenerationCompletionReceipt, + type KnowledgeNodeGenerationReceipt, + type KnowledgeNodeGenerationUnitRangeReceipt, + type KnowledgeNodeGenerationWindowReceipt, + type KnowledgeNodeSemanticGenerationConfig, + MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES, + MAX_LLM_SEMANTIC_COMPLETION_IDENTITIES, + MAX_LLM_SEMANTIC_FINISH_REASON_CODE_POINTS, + MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS, + MAX_LLM_SEMANTIC_UNIT_ID_CODE_POINTS, + MAX_LLM_SEMANTIC_WINDOWS, + MAX_LLM_SEMANTIC_WINDOW_ID_CODE_POINTS, + knowledgeNodeGenerationReceiptSerializedBytes, + llmSemanticCompletionFingerprint, +} from "./semantic-generation-receipt"; + +export type { + KnowledgeNodeGenerationCompletionReceipt, + KnowledgeNodeGenerationReceipt, + KnowledgeNodeGenerationUnitRangeReceipt, + KnowledgeNodeGenerationWindowReceipt, + KnowledgeNodeSemanticGenerationConfig, +} from "./semantic-generation-receipt"; export interface KnowledgeNodeCursor { readonly id: string; @@ -99,12 +131,34 @@ export interface UpdateKnowledgeNodeMetadataManyInput { readonly publicationGenerationId?: string | undefined; } +export interface KnowledgeNodeGenerationReceiptLookupInput { + readonly knowledgeSpaceId: string; + readonly parseArtifactId: string; + readonly publicationGenerationId: string; +} + +export interface CompleteKnowledgeNodeGenerationInput { + readonly nodes: readonly KnowledgeNode[]; + readonly receipt: KnowledgeNodeGenerationReceipt; +} + +export interface CompleteKnowledgeNodeGenerationResult { + readonly nodes: KnowledgeNode[]; + readonly receipt: KnowledgeNodeGenerationReceipt; +} + export interface KnowledgeNodeRepository { + completeGenerationAtomically?( + input: CompleteKnowledgeNodeGenerationInput, + ): Promise; createMany(nodes: readonly KnowledgeNode[]): Promise; deleteByDocumentAsset( input: DeleteKnowledgeNodesByDocumentAssetInput, ): Promise; get(input: KnowledgeNodeLookupInput): Promise; + getGenerationReceipt?( + input: KnowledgeNodeGenerationReceiptLookupInput, + ): Promise; getMany(input: GetManyKnowledgeNodesInput): Promise; /** * Reads immutable evidence references by their globally unique ids without selecting a @@ -117,6 +171,11 @@ export interface KnowledgeNodeRepository { ): Promise; listBySpace(input: ListKnowledgeNodesBySpaceInput): Promise; updateMetadataMany(input: UpdateKnowledgeNodeMetadataManyInput): Promise; + /** + * Persists one immutable publication generation in a single repository transaction while the + * implementation may split SQL statements into repository-safe batches. + */ + upsertGenerationAtomically?(nodes: readonly KnowledgeNode[]): Promise; upsertMany(nodes: readonly KnowledgeNode[]): Promise; } @@ -151,6 +210,12 @@ export class KnowledgeNodeLogicalConflictError extends Error { } } +export class KnowledgeNodeGenerationReceiptConflictError extends Error { + constructor() { + super("Knowledge node generation receipt conflicts with the immutable persisted receipt"); + } +} + export function createInMemoryKnowledgeNodeRepository({ maxBatchSize, maxListLimit, @@ -160,8 +225,83 @@ export function createInMemoryKnowledgeNodeRepository({ validateKnowledgeNodeRepositoryBounds({ maxBatchSize, maxListLimit, maxNodes }); const nodes = new Map(); + const generationReceipts = new Map(); + const upsertAtomically = (input: readonly KnowledgeNode[]): KnowledgeNode[] => { + const parsed = input.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); + validateKnowledgeNodeLogicalBatch(parsed); + const next = new Map(nodes); + const persisted: KnowledgeNode[] = []; + + for (const node of parsed) { + const existingById = next.get(node.id); + if (existingById && !hasSameKnowledgeNodeOwnership(existingById, node)) { + throw new KnowledgeNodeOwnershipConflictError(node.id); + } + if ( + existingById && + knowledgeNodeLogicalIdentity(existingById) !== knowledgeNodeLogicalIdentity(node) + ) { + throw new KnowledgeNodeLogicalConflictError(); + } + + const existingByLogicalIdentity = findKnowledgeNodeByLogicalIdentity(next.values(), node); + if ( + existingById && + existingByLogicalIdentity && + existingById.id !== existingByLogicalIdentity.id + ) { + throw new KnowledgeNodeLogicalConflictError(); + } + if ( + existingByLogicalIdentity && + !hasSameKnowledgeNodeOwnership(existingByLogicalIdentity, node) + ) { + throw new KnowledgeNodeOwnershipConflictError(node.id); + } + + const existing = existingById ?? existingByLogicalIdentity; + if (existing && node.publicationGenerationId) { + assertExactGenerationReplay({ + componentType: "knowledge-node", + incoming: node, + logicalKey: knowledgeNodeLogicalIdentity(node), + persisted: existing, + }); + persisted.push(existing); + continue; + } + const stored = existing ? { ...node, id: existing.id } : node; + next.set(stored.id, cloneKnowledgeNode(stored)); + persisted.push(stored); + } + + if (next.size > maxNodes) { + throw new KnowledgeNodeCapacityExceededError(maxNodes); + } + + nodes.clear(); + for (const [id, node] of next) { + nodes.set(id, node); + } + + return persisted.map(cloneKnowledgeNode); + }; return { + completeGenerationAtomically: async (input) => { + const receipt = validateKnowledgeNodeGenerationCompletion(input); + const key = knowledgeNodeGenerationReceiptKey(receipt); + const existingReceipt = generationReceipts.get(key); + if (existingReceipt && stableJson(existingReceipt) !== stableJson(receipt)) { + throw new KnowledgeNodeGenerationReceiptConflictError(); + } + const persistedNodes = input.nodes.length > 0 ? upsertAtomically(input.nodes) : []; + generationReceipts.set(key, cloneKnowledgeNodeGenerationReceipt(receipt)); + return { + nodes: persistedNodes, + receipt: cloneKnowledgeNodeGenerationReceipt(existingReceipt ?? receipt), + }; + }, createMany: async (input) => { validateKnowledgeNodeBatch(input, maxBatchSize); const parsed = input.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); @@ -213,66 +353,13 @@ export function createInMemoryKnowledgeNodeRepository({ return persisted.map(cloneKnowledgeNode); }, + upsertGenerationAtomically: async (input) => { + validateKnowledgeNodeGenerationBatch(input); + return upsertAtomically(input); + }, upsertMany: async (input) => { validateKnowledgeNodeBatch(input, maxBatchSize); - const parsed = input.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); - validateKnowledgeNodeLogicalBatch(parsed); - const next = new Map(nodes); - const persisted: KnowledgeNode[] = []; - - for (const node of parsed) { - const existingById = next.get(node.id); - if (existingById && !hasSameKnowledgeNodeOwnership(existingById, node)) { - throw new KnowledgeNodeOwnershipConflictError(node.id); - } - if ( - existingById && - knowledgeNodeLogicalIdentity(existingById) !== knowledgeNodeLogicalIdentity(node) - ) { - throw new KnowledgeNodeLogicalConflictError(); - } - - const existingByLogicalIdentity = findKnowledgeNodeByLogicalIdentity(next.values(), node); - if ( - existingById && - existingByLogicalIdentity && - existingById.id !== existingByLogicalIdentity.id - ) { - throw new KnowledgeNodeLogicalConflictError(); - } - if ( - existingByLogicalIdentity && - !hasSameKnowledgeNodeOwnership(existingByLogicalIdentity, node) - ) { - throw new KnowledgeNodeOwnershipConflictError(node.id); - } - - const existing = existingById ?? existingByLogicalIdentity; - if (existing && node.publicationGenerationId) { - assertExactGenerationReplay({ - componentType: "knowledge-node", - incoming: node, - logicalKey: knowledgeNodeLogicalIdentity(node), - persisted: existing, - }); - persisted.push(existing); - continue; - } - const stored = existing ? { ...node, id: existing.id } : node; - next.set(stored.id, cloneKnowledgeNode(stored)); - persisted.push(stored); - } - - if (next.size > maxNodes) { - throw new KnowledgeNodeCapacityExceededError(maxNodes); - } - - nodes.clear(); - for (const [id, node] of next) { - nodes.set(id, node); - } - - return persisted.map(cloneKnowledgeNode); + return upsertAtomically(input); }, deleteByDocumentAsset: async ({ documentAssetId, knowledgeSpaceId, maxNodes }) => { if (!Number.isInteger(maxNodes) || maxNodes < 1) { @@ -321,6 +408,11 @@ export function createInMemoryKnowledgeNodeRepository({ ? cloneKnowledgeNode(node) : null; }, + getGenerationReceipt: async (input) => { + const normalized = normalizeKnowledgeNodeGenerationReceiptLookup(input); + const receipt = generationReceipts.get(knowledgeNodeGenerationReceiptKey(normalized)); + return receipt ? cloneKnowledgeNodeGenerationReceipt(receipt) : null; + }, getMany: async ({ ids, knowledgeSpaceId, publicationGenerationId }) => { validateKnowledgeNodeBatchIds(ids, maxBatchSize); const generation = normalizeKnowledgeNodeGeneration(publicationGenerationId); @@ -448,8 +540,38 @@ export function createDatabaseKnowledgeNodeRepository({ maxNodes: Number.MAX_SAFE_INTEGER, }); const tableName = "knowledge_nodes"; + const receiptTableName = "knowledge_node_generation_receipts"; return { + completeGenerationAtomically: async (input) => { + const receipt = validateKnowledgeNodeGenerationCompletion(input); + const parsedNodes = input.nodes.map((node) => + cloneKnowledgeNode(KnowledgeNodeSchema.parse(node)), + ); + validateKnowledgeNodeLogicalBatch(parsedNodes); + + return database.transaction(async (transaction) => { + const persisted: KnowledgeNode[] = []; + for (const batch of chunkKnowledgeNodeBatch(parsedNodes, maxBatchSize)) { + persisted.push( + ...(await databaseWriteKnowledgeNodeGroups({ + database, + executor: transaction, + legacyMode: "upsert", + nodes: batch, + tableName, + })), + ); + } + const persistedReceipt = await databaseWriteKnowledgeNodeGenerationReceipt({ + database, + executor: transaction, + receipt, + tableName: receiptTableName, + }); + return { nodes: persisted, receipt: persistedReceipt }; + }); + }, createMany: async (input) => { validateKnowledgeNodeBatch(input, maxBatchSize); const nodes = input.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); @@ -473,6 +595,27 @@ export function createDatabaseKnowledgeNodeRepository({ }), ); }, + upsertGenerationAtomically: async (input) => { + validateKnowledgeNodeGenerationBatch(input); + const nodes = input.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); + validateKnowledgeNodeLogicalBatch(nodes); + + return database.transaction(async (transaction) => { + const persisted: KnowledgeNode[] = []; + for (const batch of chunkKnowledgeNodeBatch(nodes, maxBatchSize)) { + persisted.push( + ...(await databaseWriteKnowledgeNodeGroups({ + database, + executor: transaction, + legacyMode: "upsert", + nodes: batch, + tableName, + })), + ); + } + return persisted; + }); + }, upsertMany: async (input) => { validateKnowledgeNodeBatch(input, maxBatchSize); const nodes = input.map((node) => cloneKnowledgeNode(KnowledgeNodeSchema.parse(node))); @@ -589,6 +732,30 @@ export function createDatabaseKnowledgeNodeRepository({ return result.rows[0] ? mapKnowledgeNodeRow(result.rows[0]) : null; }, + getGenerationReceipt: async (input) => { + const normalized = normalizeKnowledgeNodeGenerationReceiptLookup(input); + const result = await database.execute({ + maxRows: 1, + operation: "select", + params: [ + normalized.knowledgeSpaceId, + normalized.publicationGenerationId, + normalized.parseArtifactId, + ], + sql: `SELECT * FROM ${quoteDatabaseIdentifier(database, receiptTableName)} WHERE ${quoteDatabaseIdentifier( + database, + "knowledge_space_id", + )} = ${databasePlaceholder(database, 1)} AND ${quoteDatabaseIdentifier( + database, + "publication_generation_id", + )} = ${databasePlaceholder(database, 2)} AND ${quoteDatabaseIdentifier( + database, + "parse_artifact_id", + )} = ${databasePlaceholder(database, 3)} LIMIT 1;`, + tableName: receiptTableName, + }); + return result.rows[0] ? mapKnowledgeNodeGenerationReceiptRow(result.rows[0]) : null; + }, getMany: async ({ ids, knowledgeSpaceId, publicationGenerationId }) => { return databaseKnowledgeNodeGetMany(database, tableName, maxBatchSize, { ids, @@ -801,6 +968,466 @@ export function knowledgeNodeCursor(node: KnowledgeNode): KnowledgeNodeCursor { }; } +function validateKnowledgeNodeGenerationCompletion({ + nodes, + receipt: inputReceipt, +}: CompleteKnowledgeNodeGenerationInput): KnowledgeNodeGenerationReceipt { + const receipt = validateKnowledgeNodeGenerationReceipt(inputReceipt); + if (nodes.length !== receipt.storedNodeCount) { + throw new Error("Knowledge node generation receipt storedNodeCount does not match nodes"); + } + const expectedIndexes = Array.from( + { length: receipt.documentChunkCount }, + (_, index) => index, + ).filter((index) => !receipt.excludedNodeOrdinals.includes(index)); + const actualIndexes: number[] = []; + for (const candidate of nodes) { + const node = KnowledgeNodeSchema.parse(candidate); + if ( + node.knowledgeSpaceId !== receipt.knowledgeSpaceId || + node.documentAssetId !== receipt.documentAssetId || + node.parseArtifactId !== receipt.parseArtifactId || + node.publicationGenerationId !== receipt.publicationGenerationId || + node.artifactHash !== receipt.artifactHash || + !Number.isSafeInteger(node.metadata.chunkIndex) + ) { + throw new Error("Knowledge node generation receipt identity does not match nodes"); + } + actualIndexes.push(node.metadata.chunkIndex as number); + } + actualIndexes.sort((left, right) => left - right); + if (stableJson(actualIndexes) !== stableJson(expectedIndexes)) { + throw new Error("Knowledge node generation receipt chunk indexes do not match nodes"); + } + return receipt; +} + +function validateKnowledgeNodeGenerationReceipt( + input: KnowledgeNodeGenerationReceipt, +): KnowledgeNodeGenerationReceipt { + assertKnowledgeNodeGenerationReceiptSize(input); + const knowledgeSpaceId = UuidSchema.parse(input.knowledgeSpaceId); + const publicationGenerationId = PublicationGenerationIdSchema.parse( + input.publicationGenerationId, + ); + const parseArtifactId = UuidSchema.parse(input.parseArtifactId); + const documentAssetId = UuidSchema.parse(input.documentAssetId); + if (input.schemaVersion !== 1) { + throw new Error("Knowledge node generation receipt schemaVersion must be 1"); + } + if (!/^[a-f0-9]{64}$/u.test(input.artifactHash)) { + throw new Error("Knowledge node generation receipt artifactHash is invalid"); + } + const semanticConfig = input.semanticConfig; + for (const [name, value] of [ + ["maxChunkChars", semanticConfig.maxChunkChars], + ["maxNodes", semanticConfig.maxNodes], + ["maxWindowChars", semanticConfig.maxWindowChars], + ] as const) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`Knowledge node generation receipt ${name} must be at least 1`); + } + } + if ( + !Number.isSafeInteger(semanticConfig.overlapChars) || + semanticConfig.overlapChars < 0 || + semanticConfig.overlapChars >= semanticConfig.maxChunkChars + ) { + throw new Error("Knowledge node generation receipt overlapChars is invalid"); + } + if ( + semanticConfig.maxWindowChars < semanticConfig.maxChunkChars || + !semanticConfig.promptVersion.trim() + ) { + throw new Error("Knowledge node generation receipt semantic config is invalid"); + } + if ( + !Number.isSafeInteger(input.documentChunkCount) || + input.documentChunkCount < 0 || + input.documentChunkCount > semanticConfig.maxNodes || + !Number.isSafeInteger(input.storedNodeCount) || + input.storedNodeCount < 0 || + input.storedNodeCount > input.documentChunkCount + ) { + throw new Error("Knowledge node generation receipt node counts are invalid"); + } + const excludedNodeOrdinals = [...input.excludedNodeOrdinals]; + if ( + excludedNodeOrdinals.some( + (ordinal) => + !Number.isSafeInteger(ordinal) || ordinal < 0 || ordinal >= input.documentChunkCount, + ) || + excludedNodeOrdinals.some((ordinal, index) => { + const previous = excludedNodeOrdinals[index - 1]; + return previous !== undefined && ordinal <= previous; + }) || + input.storedNodeCount !== input.documentChunkCount - excludedNodeOrdinals.length + ) { + throw new Error("Knowledge node generation receipt exclusions are invalid"); + } + for (const fingerprint of [ + input.promptResponseFingerprint, + input.requestFingerprint, + input.responseFingerprint, + input.storedResponseFingerprint, + ]) { + if (!/^sha256:[a-f0-9]{64}$/u.test(fingerprint)) { + throw new Error("Knowledge node generation receipt fingerprint is invalid"); + } + } + const permissionScope = [...input.permissionScope]; + if (permissionScope.some((scope) => typeof scope !== "string" || !scope.trim())) { + throw new Error("Knowledge node generation receipt permissionScope is invalid"); + } + if (input.language !== undefined && !input.language.trim()) { + throw new Error("Knowledge node generation receipt language is invalid"); + } + const modelSelection = KnowledgeSpaceModelSelectionSchema.parse(input.modelSelection); + const completionCatalog = validateKnowledgeNodeGenerationCompletionCatalog( + input.completionCatalog, + ); + const windowManifest = validateKnowledgeNodeGenerationWindowManifest( + input.windowManifest, + input.documentChunkCount, + semanticConfig.maxNodes, + completionCatalog.length, + ); + const receipt: KnowledgeNodeGenerationReceipt = { + artifactHash: input.artifactHash, + completionCatalog, + documentAssetId, + documentChunkCount: input.documentChunkCount, + excludedNodeOrdinals, + knowledgeSpaceId, + ...(input.language === undefined ? {} : { language: input.language }), + modelSelection, + parseArtifactId, + permissionScope, + promptResponseFingerprint: input.promptResponseFingerprint, + publicationGenerationId, + requestFingerprint: input.requestFingerprint, + responseFingerprint: input.responseFingerprint, + schemaVersion: 1, + semanticConfig: { + maxChunkChars: semanticConfig.maxChunkChars, + maxNodes: semanticConfig.maxNodes, + maxWindowChars: semanticConfig.maxWindowChars, + overlapChars: semanticConfig.overlapChars, + promptVersion: semanticConfig.promptVersion, + }, + storedNodeCount: input.storedNodeCount, + storedResponseFingerprint: input.storedResponseFingerprint, + windowManifest, + }; + assertKnowledgeNodeGenerationReceiptSize(receipt); + return receipt; +} + +function validateKnowledgeNodeGenerationCompletionCatalog( + input: readonly KnowledgeNodeGenerationCompletionReceipt[], +): KnowledgeNodeGenerationCompletionReceipt[] { + if (!Array.isArray(input) || input.length > MAX_LLM_SEMANTIC_COMPLETION_IDENTITIES) { + throw new Error("Knowledge node generation receipt completion catalog is invalid"); + } + const fingerprints = new Set(); + const identities = new Set(); + return input.map((candidate) => { + if (!isPlainObject(candidate)) { + throw new Error("Knowledge node generation receipt completion identity is invalid"); + } + const actualModel = optionalBoundedReceiptString( + candidate.actualModel, + "actualModel", + MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS, + ); + const actualProvider = optionalBoundedReceiptString( + candidate.actualProvider, + "actualProvider", + MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS, + ); + const finishReason = optionalBoundedReceiptString( + candidate.finishReason, + "finishReason", + MAX_LLM_SEMANTIC_FINISH_REASON_CODE_POINTS, + ); + const transportProvider = optionalBoundedReceiptString( + candidate.transportProvider, + "transportProvider", + MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS, + ); + const identity = { + ...(actualModel ? { actualModel } : {}), + ...(actualProvider ? { actualProvider } : {}), + ...(finishReason ? { finishReason } : {}), + ...(transportProvider ? { transportProvider } : {}), + }; + const fingerprint = llmSemanticCompletionFingerprint(identity); + if ( + candidate.fingerprint !== fingerprint || + fingerprints.has(fingerprint) || + identities.has(stableJson(identity)) + ) { + throw new Error("Knowledge node generation receipt completion identity is invalid"); + } + fingerprints.add(fingerprint); + identities.add(stableJson(identity)); + return { fingerprint, ...identity }; + }); +} + +function validateKnowledgeNodeGenerationWindowManifest( + input: readonly KnowledgeNodeGenerationWindowReceipt[], + documentChunkCount: number, + maxNodes: number, + completionCatalogLength: number, +): KnowledgeNodeGenerationWindowReceipt[] { + if ( + !Array.isArray(input) || + input.length > maxNodes || + input.length > MAX_LLM_SEMANTIC_WINDOWS || + (documentChunkCount > 0 && input.length === 0) || + (documentChunkCount === 0 && input.length > 0) || + (input.length > 0 && completionCatalogLength === 0) || + completionCatalogLength > input.length + ) { + throw new Error("Knowledge node generation receipt window manifest is incomplete"); + } + const windowIds = new Set(); + let nextChunkIndex = 0; + const windows = input.map((window, windowIndex) => { + if (window === null || typeof window !== "object" || Array.isArray(window)) { + throw new Error("Knowledge node generation receipt window manifest is invalid"); + } + if ( + !isValidReceiptWindowId(window.windowId) || + window.windowId !== `window-${windowIndex.toString().padStart(6, "0")}` || + windowIds.has(window.windowId) || + !/^sha256:[a-f0-9]{64}$/u.test(window.inputFingerprint) || + !/^sha256:[a-f0-9]{64}$/u.test(window.responseFingerprint) || + !Number.isSafeInteger(window.completionIndex) || + window.completionIndex < 0 || + window.completionIndex >= completionCatalogLength || + !Number.isSafeInteger(window.firstChunkIndex) || + window.firstChunkIndex !== nextChunkIndex || + !Array.isArray(window.chunkRanges) || + window.chunkRanges.length < 1 || + window.chunkRanges.length > maxNodes + ) { + throw new Error("Knowledge node generation receipt window manifest is invalid"); + } + windowIds.add(window.windowId); + const coreUnitRange = validateKnowledgeNodeGenerationUnitRange(window.coreUnitRange); + const committedUnitRange = validateKnowledgeNodeGenerationUnitRange(window.committedUnitRange); + const lookAheadUnitRange = + window.lookAheadUnitRange === undefined + ? undefined + : validateKnowledgeNodeGenerationUnitRange(window.lookAheadUnitRange); + const chunkRanges = window.chunkRanges.map(validateKnowledgeNodeGenerationUnitRange); + nextChunkIndex += chunkRanges.length; + return { + chunkRanges, + committedUnitRange, + completionIndex: window.completionIndex, + coreUnitRange, + firstChunkIndex: window.firstChunkIndex, + inputFingerprint: window.inputFingerprint, + ...(lookAheadUnitRange ? { lookAheadUnitRange } : {}), + responseFingerprint: window.responseFingerprint, + windowId: window.windowId, + }; + }); + if (nextChunkIndex !== documentChunkCount) { + throw new Error("Knowledge node generation receipt window chunks do not cover the document"); + } + return windows; +} + +function validateKnowledgeNodeGenerationUnitRange( + input: KnowledgeNodeGenerationUnitRangeReceipt, +): KnowledgeNodeGenerationUnitRangeReceipt { + if (!Array.isArray(input) || input.length !== 2 || !input.every(isValidReceiptUnitId)) { + throw new Error("Knowledge node generation receipt window unit range is invalid"); + } + return [input[0] as string, input[1] as string]; +} + +function isValidReceiptWindowId(value: unknown): value is string { + return ( + typeof value === "string" && + Array.from(value).length <= MAX_LLM_SEMANTIC_WINDOW_ID_CODE_POINTS && + /^window-\d{6,}$/u.test(value) + ); +} + +function isValidReceiptUnitId(value: unknown): value is string { + return ( + typeof value === "string" && + Array.from(value).length <= MAX_LLM_SEMANTIC_UNIT_ID_CODE_POINTS && + /^u-\d{6,}-\d{6,}$/u.test(value) + ); +} + +function optionalBoundedReceiptString( + value: unknown, + name: string, + maxCodePoints: number, +): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== "string" || !value.trim() || Array.from(value).length > maxCodePoints) { + throw new Error(`Knowledge node generation receipt ${name} is invalid`); + } + return value.trim(); +} + +function assertKnowledgeNodeGenerationReceiptSize(value: unknown): void { + if ( + knowledgeNodeGenerationReceiptSerializedBytes(value) > + MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES + ) { + throw new Error( + `Knowledge node generation receipt exceeds maxBytes=${MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES}`, + ); + } +} + +function normalizeKnowledgeNodeGenerationReceiptLookup( + input: KnowledgeNodeGenerationReceiptLookupInput, +): KnowledgeNodeGenerationReceiptLookupInput { + return { + knowledgeSpaceId: UuidSchema.parse(input.knowledgeSpaceId), + parseArtifactId: UuidSchema.parse(input.parseArtifactId), + publicationGenerationId: PublicationGenerationIdSchema.parse(input.publicationGenerationId), + }; +} + +function knowledgeNodeGenerationReceiptKey( + input: KnowledgeNodeGenerationReceiptLookupInput, +): string { + return stableJson([input.knowledgeSpaceId, input.publicationGenerationId, input.parseArtifactId]); +} + +function cloneKnowledgeNodeGenerationReceipt( + receipt: KnowledgeNodeGenerationReceipt, +): KnowledgeNodeGenerationReceipt { + return validateKnowledgeNodeGenerationReceipt( + JSON.parse(JSON.stringify(receipt)) as KnowledgeNodeGenerationReceipt, + ); +} + +async function databaseWriteKnowledgeNodeGenerationReceipt({ + database, + executor, + receipt, + tableName, +}: { + readonly database: DatabaseAdapter; + readonly executor: DatabaseExecutor; + readonly receipt: KnowledgeNodeGenerationReceipt; + readonly tableName: string; +}): Promise { + const columns = [ + "knowledge_space_id", + "publication_generation_id", + "parse_artifact_id", + "document_asset_id", + "artifact_hash", + "document_chunk_count", + "stored_node_count", + "request_fingerprint", + "response_fingerprint", + "prompt_response_fingerprint", + "receipt", + ] as const; + const params: readonly DatabaseQueryValue[] = [ + receipt.knowledgeSpaceId, + receipt.publicationGenerationId, + receipt.parseArtifactId, + receipt.documentAssetId, + receipt.artifactHash, + receipt.documentChunkCount, + receipt.storedNodeCount, + receipt.requestFingerprint, + receipt.responseFingerprint, + receipt.promptResponseFingerprint, + JSON.stringify(receipt), + ]; + const conflictSql = + database.dialect === "postgres" + ? " ON CONFLICT DO NOTHING RETURNING *" + : ` ON DUPLICATE KEY UPDATE ${quoteDatabaseIdentifier( + database, + "knowledge_space_id", + )} = ${quoteDatabaseIdentifier(database, "knowledge_space_id")}`; + const inserted = await executor.execute({ + maxRows: 1, + operation: "insert", + params, + sql: `INSERT INTO ${quoteDatabaseIdentifier(database, tableName)} (${columns + .map((column) => quoteDatabaseIdentifier(database, column)) + .join(", ")}) VALUES (${columns + .map((_, index) => + index === columns.length - 1 + ? jsonInsertPlaceholder(database, index + 1, "receipt") + : databasePlaceholder(database, index + 1), + ) + .join(", ")})${conflictSql};`, + tableName, + }); + const row = + inserted.rows[0] ?? + ( + await executor.execute({ + maxRows: 1, + operation: "select", + params: [ + receipt.knowledgeSpaceId, + receipt.publicationGenerationId, + receipt.parseArtifactId, + ], + sql: `SELECT * FROM ${quoteDatabaseIdentifier(database, tableName)} WHERE ${quoteDatabaseIdentifier( + database, + "knowledge_space_id", + )} = ${databasePlaceholder(database, 1)} AND ${quoteDatabaseIdentifier( + database, + "publication_generation_id", + )} = ${databasePlaceholder(database, 2)} AND ${quoteDatabaseIdentifier( + database, + "parse_artifact_id", + )} = ${databasePlaceholder(database, 3)} LIMIT 1;`, + tableName, + }) + ).rows[0]; + if (!row) { + throw new Error("Knowledge node generation receipt was not persisted"); + } + const persisted = mapKnowledgeNodeGenerationReceiptRow(row); + if (stableJson(persisted) !== stableJson(receipt)) { + throw new KnowledgeNodeGenerationReceiptConflictError(); + } + return persisted; +} + +function mapKnowledgeNodeGenerationReceiptRow(row: DatabaseRow): KnowledgeNodeGenerationReceipt { + const receipt = validateKnowledgeNodeGenerationReceipt( + jsonObjectColumn(row, "receipt") as unknown as KnowledgeNodeGenerationReceipt, + ); + if ( + stringColumn(row, "knowledge_space_id") !== receipt.knowledgeSpaceId || + stringColumn(row, "publication_generation_id") !== receipt.publicationGenerationId || + stringColumn(row, "parse_artifact_id") !== receipt.parseArtifactId || + stringColumn(row, "document_asset_id") !== receipt.documentAssetId || + stringColumn(row, "artifact_hash") !== receipt.artifactHash || + numberColumn(row, "document_chunk_count") !== receipt.documentChunkCount || + numberColumn(row, "stored_node_count") !== receipt.storedNodeCount || + stringColumn(row, "request_fingerprint") !== receipt.requestFingerprint || + stringColumn(row, "response_fingerprint") !== receipt.responseFingerprint || + stringColumn(row, "prompt_response_fingerprint") !== receipt.promptResponseFingerprint + ) { + throw new KnowledgeNodeGenerationReceiptConflictError(); + } + return receipt; +} + async function databaseWriteKnowledgeNodeGroups({ database, executor, @@ -1278,6 +1905,37 @@ function validateKnowledgeNodeBatch(nodes: readonly KnowledgeNode[], maxBatchSiz } } +function validateKnowledgeNodeGenerationBatch(nodes: readonly KnowledgeNode[]): void { + if (nodes.length < 1) { + throw new Error("Knowledge node generation batch must contain at least 1 node"); + } + const first = KnowledgeNodeSchema.parse(nodes[0]); + if (!first.publicationGenerationId) { + throw new Error("Knowledge node generation batch requires publicationGenerationId"); + } + for (const candidate of nodes) { + const node = KnowledgeNodeSchema.parse(candidate); + if ( + node.publicationGenerationId !== first.publicationGenerationId || + node.knowledgeSpaceId !== first.knowledgeSpaceId || + node.parseArtifactId !== first.parseArtifactId + ) { + throw new Error("Knowledge node generation batch must share one generation and artifact"); + } + } +} + +function chunkKnowledgeNodeBatch( + nodes: readonly KnowledgeNode[], + maxBatchSize: number, +): KnowledgeNode[][] { + const batches: KnowledgeNode[][] = []; + for (let start = 0; start < nodes.length; start += maxBatchSize) { + batches.push(nodes.slice(start, start + maxBatchSize)); + } + return batches; +} + function validateKnowledgeNodeLogicalBatch(nodes: readonly KnowledgeNode[]): void { const identities = new Set(); diff --git a/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.test.ts b/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.test.ts index 51f4e6f68bc..762d4d7c5e7 100644 --- a/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.test.ts +++ b/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.test.ts @@ -3,11 +3,14 @@ import type { DatabaseExecuteInput, DatabaseExecuteResult, DatabaseTransactionCallback, + DocumentAsset, IndexProjection, + KnowledgePath, } from "@knowledge/core"; import { describe, expect, it, vi } from "vitest"; import { deterministicChildId } from "./api-shared-utils"; +import { buildDocumentOutlineKnowledgePath } from "./document-knowledge-paths"; import type { KnowledgeSpaceProfileMigrationRun } from "./knowledge-space-profile-migration"; import { type ReplaceKnowledgeSpaceProfileMigrationCandidateSnapshotInput, @@ -334,12 +337,48 @@ describe("profile migration candidate builder", () => { pageIndexSummaryOutlineRebuilt: true, publicationStatus: "validating", }); + expect(fixture.reindex).toHaveBeenCalledOnce(); + expect(fixture.reindex).toHaveBeenCalledWith( + expect.objectContaining({ + denseModel: fixture.baseVectorSpaceId, + embeddingProfile: expect.objectContaining({ model: "embedding-v1" }), + retrievalProfile: expect.objectContaining({ + reasoningModel: expect.objectContaining({ model: "reasoning-v2" }), + }), + }), + ); + expect(fixture.buildOutline).toHaveBeenCalledWith( + expect.objectContaining({ + parseArtifact: expect.objectContaining({ + metadata: expect.objectContaining({ semanticCompilation: expect.any(Object) }), + }), + }), + ); expect(fixture.enhance).toHaveBeenCalledOnce(); expect(fixture.materialize).toHaveBeenCalledOnce(); + expect(fixture.materializeGraph).toHaveBeenCalledWith( + expect.objectContaining({ + createdAt: now, + publicationGenerationId: fixture.expectedGenerationId, + }), + ); expect(fixture.heartbeat.mock.calls.length).toBeGreaterThanOrEqual(5); expect(fixture.candidateMembers()).toEqual( expect.arrayContaining([ - expect.objectContaining({ componentKey: pathId, componentType: "knowledge-path" }), + expect.objectContaining({ + componentType: "knowledge-path", + generationId: fixture.expectedGenerationId, + }), + expect.objectContaining({ + componentKey: fixture.rebuiltGraphEntityId, + componentType: "graph-entity", + generationId: fixture.expectedGenerationId, + }), + expect.objectContaining({ + componentKey: fixture.rebuiltGraphRelationId, + componentType: "graph-relation", + generationId: fixture.expectedGenerationId, + }), expect.objectContaining({ componentKey: fixture.rebuiltOutlineId, componentType: "document-outline", @@ -376,12 +415,32 @@ describe("profile migration candidate builder", () => { publicationStatus: "validating", }); expect(fixture.reindex).toHaveBeenCalledOnce(); + expect(fixture.reindex).toHaveBeenCalledWith( + expect.objectContaining({ + reuseNodeGenerationId: baseGenerationId, + skipVisual: true, + retrievalProfile: expect.objectContaining({ + reasoningModel: expect.objectContaining({ model: "reasoning-v1" }), + }), + }), + ); + expect(fixture.materializeGraph).toHaveBeenCalledWith( + expect.objectContaining({ + createdAt: now, + publicationGenerationId: fixture.expectedGenerationId, + }), + ); expect(fixture.candidateMembers()).toEqual( expect.arrayContaining([ expect.objectContaining({ componentKey: pathId, componentType: "knowledge-path" }), expect.objectContaining({ componentKey: fixture.visualProjectionId }), expect.objectContaining({ componentKey: fixture.rebuiltFtsId }), expect.objectContaining({ componentKey: fixture.rebuiltDenseId }), + expect.objectContaining({ + componentKey: fixture.rebuiltGraphEntityId, + componentType: "graph-entity", + generationId: fixture.expectedGenerationId, + }), ]), ); expect( @@ -740,6 +799,9 @@ function builderFixture( const visualProjectionId = deterministicChildId(documentAssetId, "base-visual"); const rebuiltFtsId = deterministicChildId(runId, "rebuilt-fts"); const rebuiltDenseId = deterministicChildId(runId, "rebuilt-dense"); + const baseGraphEntityId = deterministicChildId(documentAssetId, "base-graph-entity"); + const rebuiltGraphEntityId = deterministicChildId(runId, "rebuilt-graph-entity"); + const rebuiltGraphRelationId = deterministicChildId(runId, "rebuilt-graph-relation"); const expectedGenerationId = deterministicChildId( runId, `profile-migration:${scope === "full-vector-space" ? "vector-space" : "page-index"}:${documentAssetId}`, @@ -771,33 +833,55 @@ function builderFixture( publicationGenerationId: baseGenerationId, version: 1, }; + const asset: DocumentAsset = { + createdAt: now, + filename: "synthetic-invoice.pdf", + id: documentAssetId, + knowledgeSpaceId: spaceId, + metadata: { permissionScope: ["read"] }, + mimeType: "application/pdf", + objectKey: "documents/synthetic-invoice.pdf", + parserStatus: "parsed", + sha256: digestA, + sizeBytes: 1_024, + version: 1, + }; + const storedPaths = new Map(); + const baseOutlinePath = { + ...buildDocumentOutlineKnowledgePath({ + asset, + id: pathId, + publicationGenerationId: baseGenerationId, + tenantId, + }), + id: pathId, + }; + storedPaths.set(baseGenerationId, [baseOutlinePath]); let rebuiltOutline: typeof baseOutline | undefined; let candidate: ProjectionSetPublication | undefined; let candidateMembers: readonly ProjectionSetPublicationMember[] = []; const projections = new Map(); - const baseProjectionMembers: ProjectionSetPublicationMember[] = []; - if (scope === "full-vector-space") { - const baseProjections = [ - projection(baseFtsId, baseGenerationId, "fts"), - projection(baseDenseId, baseGenerationId, "dense-vector", oldVectorSpaceId), - { - ...projection(visualProjectionId, baseGenerationId, "dense-vector", "visual-model"), - metadata: { - documentAssetId, - multimodal: { vectorSpace: "visual" }, - }, + const baseProjections = [ + projection(baseFtsId, baseGenerationId, "fts"), + projection(baseDenseId, baseGenerationId, "dense-vector", oldVectorSpaceId), + { + ...projection(visualProjectionId, baseGenerationId, "dense-vector", "visual-model"), + metadata: { + documentAssetId, + multimodal: { vectorSpace: "visual" }, }, - ]; - for (const item of baseProjections) projections.set(item.id, item); - baseProjectionMembers.push( - member("index-projection", baseFtsId, baseGenerationId), - member("index-projection", baseDenseId, baseGenerationId), - member("index-projection", visualProjectionId, baseGenerationId), - ); - } + }, + ]; + for (const item of baseProjections) projections.set(item.id, item); + const baseProjectionMembers: ProjectionSetPublicationMember[] = [ + member("index-projection", baseFtsId, baseGenerationId), + member("index-projection", baseDenseId, baseGenerationId), + member("index-projection", visualProjectionId, baseGenerationId), + ]; const baseMembers = [ member("document-outline", outlineId, baseGenerationId), member("knowledge-path", pathId, baseGenerationId), + member("graph-entity", baseGraphEntityId, baseGenerationId), ...baseProjectionMembers, ]; const heartbeat = vi.fn(async () => undefined); @@ -805,7 +889,28 @@ function builderFixture( ...outline, metadata: { summary: { model: "reasoning-v2" } }, })); + const buildOutline = vi.fn( + ({ publicationGenerationId }: { readonly publicationGenerationId?: string }) => { + rebuiltOutline = { + ...baseOutline, + id: rebuiltOutlineId, + metadata: {}, + publicationGenerationId: publicationGenerationId ?? baseGenerationId, + }; + return rebuiltOutline as never; + }, + ); const materialize = vi.fn(async () => ({ status: "building" }) as never); + const materializeGraph = vi.fn(async () => ({ + entitiesExtracted: 1, + graphEntityIds: [rebuiltGraphEntityId], + graphEntitiesIndexed: 1, + graphRelationIds: [rebuiltGraphRelationId], + graphRelationsIndexed: 1, + nodesScanned: 1, + semanticProviderCalls: 0, + semanticProviderCallsMaximum: 0, + })); const reindex = vi.fn( async ({ publicationGenerationId }: { readonly publicationGenerationId?: string }) => { if (options.incompleteReindexReceipt) { @@ -819,13 +924,26 @@ function builderFixture( } if (!publicationGenerationId) throw new Error("generation missing"); projections.set(rebuiltFtsId, projection(rebuiltFtsId, publicationGenerationId, "fts")); + const targetVectorSpaceId = + scope === "full-vector-space" ? newVectorSpaceId : oldVectorSpaceId; projections.set( rebuiltDenseId, - projection(rebuiltDenseId, publicationGenerationId, "dense-vector", newVectorSpaceId), + projection(rebuiltDenseId, publicationGenerationId, "dense-vector", targetVectorSpaceId), ); return { artifact: {} as never, + nodeIds: [deterministicChildId(publicationGenerationId, "semantic-node")], nodesCreated: 1, + outlineArtifact: { + artifactHash: digestA, + documentAssetId, + elements: [], + id: parseArtifactId, + metadata: { semanticCompilation: { source: "reasoning-v2" } }, + parseVersion: "semantic-outline-v1", + parser: "semantic", + version: 1, + } as never, projectionIds: [rebuiltFtsId, rebuiltDenseId], projectionsCreated: 2, status: "rebuilt" as const, @@ -881,8 +999,7 @@ function builderFixture( }) as never, }, assets: { - get: async () => - ({ id: documentAssetId, metadata: { permissionScope: ["read"] }, version: 1 }) as never, + get: async () => asset, }, maxDocuments: 10, maxMembers: 100, @@ -893,15 +1010,7 @@ function builderFixture( }, now: () => now, outlineBuilder: { - build: ({ publicationGenerationId }: { readonly publicationGenerationId?: string }) => { - rebuiltOutline = { - ...baseOutline, - id: rebuiltOutlineId, - metadata: {}, - publicationGenerationId: publicationGenerationId ?? baseGenerationId, - }; - return rebuiltOutline as never; - }, + build: buildOutline, } as never, outlineSummaryEnhancer: { enhance } as never, outlines: { @@ -919,59 +1028,106 @@ function builderFixture( hasCompleteBuild: async () => options.completePageIndex !== false, materializeBuilding: materialize, }, + paths: { + listPhysicalDescendants: async ({ publicationGenerationId }) => ({ + items: storedPaths.get(publicationGenerationId ?? "") ?? [], + }), + upsertMany: async (items) => { + for (const item of items) { + const generationId = item.publicationGenerationId ?? ""; + const generation = storedPaths.get(generationId) ?? []; + const existing = generation.findIndex((path) => path.id === item.id); + if (existing >= 0) generation[existing] = item; + else generation.push(item); + storedPaths.set(generationId, generation); + } + return [...items]; + }, + }, profiles: { - getRevision: async ({ kind }) => + getRevision: async ({ kind, revision }) => kind === "embedding" - ? ({ - id: "embedding-2", - revision: 2, - snapshot: { - dimension: 3072, - model: "embedding-v2", - pluginId: "plugin-embedding", - provider: "plugin-daemon", - revision: 2, - vectorSpaceId: newVectorSpaceId, - }, - snapshotDigest: digestB, - state: "candidate", - } as never) - : ({ - id: "retrieval-2", - revision: 2, - snapshot: { - defaultMode: "research", - reasoningModel: { - model: "reasoning-v2", - pluginId: "plugin-reasoning", + ? revision === 1 + ? ({ + id: "embedding-1", + revision: 1, + snapshot: { + dimension: 3072, + model: "embedding-v1", + pluginId: "plugin-embedding", provider: "plugin-daemon", + revision: 1, + vectorSpaceId: oldVectorSpaceId, }, - rerank: { enabled: false }, + snapshotDigest: digestA, + state: "active", + } as never) + : ({ + id: "embedding-2", revision: 2, - scoreThreshold: { enabled: false, stage: "mode-final" }, - topK: 12, - }, - snapshotDigest: digestB, - state: "candidate", - } as never), + snapshot: { + dimension: 3072, + model: "embedding-v2", + pluginId: "plugin-embedding", + provider: "plugin-daemon", + revision: 2, + vectorSpaceId: newVectorSpaceId, + }, + snapshotDigest: digestB, + state: "candidate", + } as never) + : revision === 1 + ? ({ + id: "retrieval-1", + revision: 1, + snapshot: { + defaultMode: "research", + reasoningModel: { + model: "reasoning-v1", + pluginId: "plugin-reasoning", + provider: "plugin-daemon", + }, + rerank: { enabled: false }, + revision: 1, + scoreThreshold: { enabled: false, stage: "mode-final" }, + topK: 12, + }, + snapshotDigest: digestA, + state: "active", + } as never) + : ({ + id: "retrieval-2", + revision: 2, + snapshot: { + defaultMode: "research", + reasoningModel: { + model: "reasoning-v2", + pluginId: "plugin-reasoning", + provider: "plugin-daemon", + }, + rerank: { enabled: false }, + revision: 2, + scoreThreshold: { enabled: false, stage: "mode-final" }, + topK: 12, + }, + snapshotDigest: digestB, + state: "candidate", + } as never), }, projections: { getMany: async ({ ids }) => ids.flatMap((id) => projections.get(id) ?? []), }, publications, reindexer: { reindex } as never, + semanticGraph: { materialize: materializeGraph }, snapshots: { replace }, }); const input = { - ...(scope === "full-vector-space" - ? { - baseEmbeddingProfile: { - id: "embedding-1", - revision: 1, - snapshotDigest: digestA, - }, - } - : {}), + baseEmbeddingProfile: { + id: "embedding-1", + revision: 1, + snapshotDigest: digestA, + }, basePublication: { fingerprint: baseFingerprint, headRevision: 3, @@ -992,6 +1148,8 @@ function builderFixture( }; return { baseDenseId, + baseVectorSpaceId: oldVectorSpaceId, + buildOutline, builder, candidateMembers: () => candidateMembers, candidateStatus: () => candidate?.status, @@ -1000,9 +1158,12 @@ function builderFixture( heartbeat, input, materialize, + materializeGraph, published: () => basePublication, rebuiltDenseId, rebuiltFtsId, + rebuiltGraphEntityId, + rebuiltGraphRelationId, rebuiltOutlineId, reindex, replace, @@ -1021,15 +1182,25 @@ describe("profile migration structural evaluator", () => { expect(result).toMatchObject({ passed: true }); }); - it("accepts a Research-only reasoning Summary/Outline/PageIndex rebuild without Graph or FTS", async () => { + it("accepts a reasoning rebuild with one semantic generation for path, outline, FTS, and dense", async () => { const rebuiltGenerationId = deterministicChildId( runId, `profile-migration:page-index:${documentAssetId}`, ); + const candidateMembers = [ + member("document-outline", outlineId, rebuiltGenerationId), + member("knowledge-path", pathId, rebuiltGenerationId), + member("index-projection", ftsId, rebuiltGenerationId), + member("index-projection", denseId, rebuiltGenerationId), + ]; const result = await evaluator({ baseMembers: [member("document-outline", outlineId, baseGenerationId)], - candidateMembers: [member("document-outline", outlineId, rebuiltGenerationId)], + candidateMembers, outlineGenerationId: rebuiltGenerationId, + projections: [ + projection(ftsId, rebuiltGenerationId, "fts"), + projection(denseId, rebuiltGenerationId, "dense-vector", vectorSpaceId), + ], summaryModel: "reasoning-v2", }).evaluate({ candidate: candidateResult(), @@ -1062,7 +1233,7 @@ describe("profile migration structural evaluator", () => { expect(result).toMatchObject({ passed: true }); }); - it("fails closed when a reasoning candidate drops a non-outline Graph/path member", async () => { + it("fails closed when a reasoning candidate drops a preserved multimodal member", async () => { const rebuiltGenerationId = deterministicChildId( runId, `profile-migration:page-index:${documentAssetId}`, @@ -1070,7 +1241,7 @@ describe("profile migration structural evaluator", () => { const result = await evaluator({ baseMembers: [ member("document-outline", outlineId, baseGenerationId), - member("knowledge-path", pathId, baseGenerationId), + member("multimodal-manifest", pathId, baseGenerationId), ], candidateMembers: [member("document-outline", outlineId, rebuiltGenerationId)], outlineGenerationId: rebuiltGenerationId, @@ -1108,21 +1279,21 @@ function evaluator(input: { }, pageIndexBuild: { hasCompleteBuild: async () => true }, profiles: { - getRevision: async ({ kind }) => + getRevision: async ({ kind, revision }) => kind === "embedding" ? ({ - id: "embedding-2", - revision: 1, + id: revision === 1 ? "embedding-1" : "embedding-2", + revision, snapshot: { dimension: 3072, - model: "embedding-v2", + model: revision === 1 ? "embedding-v1" : "embedding-v2", pluginId: "plugin-embedding", provider: "plugin-daemon", - revision: 1, + revision, vectorSpaceId, }, - snapshotDigest: digestB, - state: "candidate", + snapshotDigest: revision === 1 ? digestA : digestB, + state: revision === 1 ? "active" : "candidate", } as never) : ({ id: "retrieval-2", @@ -1156,10 +1327,11 @@ function migrationRun( return { accessChannel: "interactive", basePublication: { fingerprint: baseFingerprint, headRevision: 3, id: basePublicationId }, + baseEmbeddingProfile: { id: "embedding-1", revision: 1, snapshotDigest: digestA }, baseRetrievalProfile: { id: "retrieval-1", revision: 1, snapshotDigest: digestA }, candidateProfile: { id: changedKind === "embedding" ? "embedding-2" : "retrieval-2", - revision: changedKind === "embedding" ? 1 : 2, + revision: 2, snapshotDigest: digestB, }, candidatePublicationFingerprint: candidateResult().publicationFingerprint, diff --git a/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.ts b/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.ts index 24688c03191..8d0f56cc845 100644 --- a/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.ts +++ b/knowledge-fs/packages/api/src/knowledge-space-profile-migration-candidate-builder.ts @@ -4,7 +4,9 @@ import { type DatabaseAdapter, type DatabaseQueryValue, DateTimeSchema, + type DocumentOutline, type IndexProjection, + type KnowledgePath, type KnowledgeSpaceEmbeddingProfile, KnowledgeSpaceEmbeddingProfileSchema, type KnowledgeSpaceRetrievalProfile, @@ -19,12 +21,18 @@ import { import { deterministicChildId } from "./api-shared-utils"; import { databasePlaceholder, quoteDatabaseIdentifier } from "./database-sql-utils"; import type { DocumentAssetRepository } from "./document-asset-repository"; +import { + buildDocumentOutlineKnowledgePath, + buildDocumentSectionKnowledgePaths, +} from "./document-knowledge-paths"; import type { DocumentOutlineBuilder } from "./document-outline-builder"; import type { DocumentOutlineRepository } from "./document-outline-repository"; import type { DocumentOutlineSummaryEnhancer } from "./document-outline-summary-enhancer"; +import type { JointSemanticGraphMaterializer } from "./document-semantic-enrichment-processor"; import type { IndexProjectionRepository } from "./index-projection-repository"; import type { IncrementalReindexer } from "./index-reindexer"; import { isPlainObject } from "./json-utils"; +import type { KnowledgePathRepository } from "./knowledge-path-repository"; import { lockKnowledgeSpaceForDeletionAdmission } from "./knowledge-space-deletion-admission"; import type { KnowledgeSpaceProfileMigrationProfileReference, @@ -259,6 +267,8 @@ export interface RepositoryKnowledgeSpaceProfileMigrationCandidateBuilderOptions readonly assets: Pick; readonly maxDocuments: number; readonly maxMembers: number; + readonly maxPathReadPageSize?: number | undefined; + readonly maxPathsPerDocument?: number | undefined; readonly maxProjectionBatchSize: number; readonly members: Pick; readonly now?: (() => string) | undefined; @@ -269,6 +279,9 @@ export interface RepositoryKnowledgeSpaceProfileMigrationCandidateBuilderOptions PublishedPageIndexBuildRepository, "hasCompleteBuild" | "materializeBuilding" >; + readonly paths?: + | Pick + | undefined; readonly profiles: Pick; readonly projections: Required>; readonly publications: Pick< @@ -276,6 +289,7 @@ export interface RepositoryKnowledgeSpaceProfileMigrationCandidateBuilderOptions "createCandidate" | "getByFingerprint" | "getPublished" | "validate" >; readonly reindexer: Pick; + readonly semanticGraph?: JointSemanticGraphMaterializer | undefined; readonly snapshots: KnowledgeSpaceProfileMigrationCandidateSnapshotRepository; } @@ -303,6 +317,8 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ assets, maxDocuments, maxMembers, + maxPathReadPageSize = 100, + maxPathsPerDocument = 20_000, maxProjectionBatchSize, members, now = () => new Date().toISOString(), @@ -310,14 +326,18 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ outlineSummaryEnhancer, outlines, pageIndexBuild, + paths, profiles, projections, publications, reindexer, + semanticGraph, snapshots, }: RepositoryKnowledgeSpaceProfileMigrationCandidateBuilderOptions): KnowledgeSpaceProfileMigrationCandidateBuilder { positiveInteger(maxDocuments, "maxDocuments"); positiveInteger(maxMembers, "maxMembers"); + positiveInteger(maxPathReadPageSize, "maxPathReadPageSize"); + positiveInteger(maxPathsPerDocument, "maxPathsPerDocument"); positiveInteger(maxProjectionBatchSize, "maxProjectionBatchSize"); const loadBase = async ( @@ -457,11 +477,69 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ return buildResult(candidate, { successorMembersCloned: true }, requireValidating); } if (input.rebuildScope === "full-page-index-summary-outline") { + if (!paths) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_UNAVAILABLE", + "Reasoning migration requires outline-derived KnowledgeFS path persistence", + ); + } + if (!input.baseEmbeddingProfile) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_UNAVAILABLE", + "Reasoning migration requires the frozen active embedding profile", + ); + } + const baseProjectionMembers = base.members.filter( + (member) => member.componentType === "index-projection", + ); + const baseProjections = await loadProjections( + projections, + baseProjectionMembers.map((member) => member.componentKey), + input.knowledgeSpaceId, + maxProjectionBatchSize, + ); + const preservedProjectionIds = new Set( + baseProjections + .filter((projection) => !isOrdinarySearchProjection(projection)) + .map((projection) => projection.id), + ); + const baseDerivedPathIds = await resolveBaseOutlineDerivedPathIds({ + documents: base.documents, + maxPaths: maxPathsPerDocument, + pageSize: maxPathReadPageSize, + paths, + tenantId: input.tenantId, + }); + const expectedCandidatePathGenerations = new Set( + base.documents.map((document) => + migrationGenerationId(input.runId, "page-index", document.documentAssetId), + ), + ); assertSameMemberSnapshot( - base.members.filter((member) => member.componentType !== "document-outline"), - candidateMembers.filter((member) => member.componentType !== "document-outline"), + base.members.filter( + (member) => + member.componentType !== "document-outline" && + !(semanticGraph && isGraphMember(member)) && + !( + member.componentType === "knowledge-path" && + baseDerivedPathIds.has(member.componentKey) + ) && + (member.componentType !== "index-projection" || + preservedProjectionIds.has(member.componentKey)), + ), + candidateMembers.filter( + (member) => + member.componentType !== "document-outline" && + !(semanticGraph && isGraphMember(member)) && + !( + member.componentType === "knowledge-path" && + expectedCandidatePathGenerations.has(member.generationId) + ) && + (member.componentType !== "index-projection" || + preservedProjectionIds.has(member.componentKey)), + ), "PROFILE_MIGRATION_PAGE_INDEX_REBUILD_INCOMPLETE", - "Reasoning migration changed or dropped a non-outline publication member", + "Reasoning migration changed or dropped a preserved publication member", ); const candidateOutlineMembers = candidateMembers.filter( (member) => member.componentType === "document-outline", @@ -480,9 +558,31 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ "candidate", ); const profile = KnowledgeSpaceRetrievalProfileSchema.parse(retrieval.snapshot); + const embedding = await requireProfile( + profiles, + input, + "embedding", + input.baseEmbeddingProfile, + "active", + ); + const embeddingProfile = KnowledgeSpaceEmbeddingProfileSchema.parse(embedding.snapshot); const outlinesByDocument = groupByDocument( candidateMembers.filter((member) => member.componentType === "document-outline"), ); + const candidateProjectionMembers = candidateMembers.filter( + (member) => member.componentType === "index-projection", + ); + const candidateProjections = await loadProjections( + projections, + candidateProjectionMembers.map((member) => member.componentKey), + input.knowledgeSpaceId, + maxProjectionBatchSize, + ); + const candidateProjectionById = new Map( + candidateProjections.map((projection) => [projection.id, projection]), + ); + const candidateProjectionsByDocument = groupByDocument(candidateProjectionMembers); + const candidateGraphByDocument = groupByDocument(candidateMembers.filter(isGraphMember)); for (const document of base.documents) { const expectedGeneration = migrationGenerationId( input.runId, @@ -510,6 +610,84 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ `Document ${document.documentAssetId} PageIndex Summary/Outline rebuild is incomplete`, ); } + if ( + (candidateGraphByDocument.get(document.documentAssetId) ?? []).some( + (member) => member.generationId !== expectedGeneration, + ) + ) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_INCOMPLETE", + `Document ${document.documentAssetId} Graph lineage is incomplete`, + ); + } + const expectedPaths = buildOutlineDerivedPaths({ + asset: document.asset, + outline, + publicationGenerationId: expectedGeneration, + tenantId: input.tenantId, + }); + await assertOutlineDerivedPathClosure({ + expected: expectedPaths, + maxPaths: maxPathsPerDocument, + members: candidateMembers.filter( + (member) => + member.componentType === "knowledge-path" && + expectedPaths.some((path) => path.id === member.componentKey), + ), + pageSize: maxPathReadPageSize, + paths, + }); + const ordinary = (candidateProjectionsByDocument.get(document.documentAssetId) ?? []) + .filter((member) => !preservedProjectionIds.has(member.componentKey)) + .map((member) => { + const projection = candidateProjectionById.get(member.componentKey); + if ( + !projection || + !isOrdinarySearchProjection(projection) || + projection.publicationGenerationId !== expectedGeneration || + member.generationId !== expectedGeneration || + projectionDocumentAssetId(projection) !== document.documentAssetId || + projection.status !== "ready" + ) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_INCOMPLETE", + `Document ${document.documentAssetId} semantic projection lineage is incomplete`, + ); + } + return projection; + }); + const fts = ordinary.filter((projection) => projection.type === "fts"); + const dense = ordinary.filter((projection) => projection.type === "dense-vector"); + if ( + fts.length < 1 || + dense.length !== fts.length || + dense.some((projection) => projection.model !== embeddingProfile.vectorSpaceId) + ) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_INCOMPLETE", + `Document ${document.documentAssetId} semantic search projection closure is incomplete`, + ); + } + } + const baseDocumentIds = new Set(base.documents.map((document) => document.documentAssetId)); + if ( + candidateMembers.some( + (member) => + isGraphMember(member) && + (!member.documentAssetId || !baseDocumentIds.has(member.documentAssetId)), + ) || + candidateProjectionMembers.some( + (member) => + !preservedProjectionIds.has(member.componentKey) && + (!member.documentAssetId || + !baseDocumentIds.has(member.documentAssetId) || + !isOrdinarySearchProjection(candidateProjectionById.get(member.componentKey))), + ) + ) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_INCOMPLETE", + "Reasoning migration candidate contains an extra or unowned search projection", + ); } return buildResult(candidate, { pageIndexSummaryOutlineRebuilt: true }, requireValidating); } @@ -526,8 +704,14 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ (member) => member.componentType === "index-projection", ); assertSameMemberSnapshot( - base.members.filter((member) => member.componentType !== "index-projection"), - candidateMembers.filter((member) => member.componentType !== "index-projection"), + base.members.filter( + (member) => + member.componentType !== "index-projection" && !(semanticGraph && isGraphMember(member)), + ), + candidateMembers.filter( + (member) => + member.componentType !== "index-projection" && !(semanticGraph && isGraphMember(member)), + ), "PROFILE_MIGRATION_VECTOR_REBUILD_INCOMPLETE", "Embedding migration changed or dropped a non-index publication member", ); @@ -562,6 +746,7 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ ); const projectionsById = new Map(loaded.map((projection) => [projection.id, projection])); const membersByDocument = groupByDocument(projectionMembers); + const graphByDocument = groupByDocument(candidateMembers.filter(isGraphMember)); for (const document of base.documents) { const expectedGeneration = migrationGenerationId( input.runId, @@ -587,6 +772,16 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ } return projection; }); + if ( + (graphByDocument.get(document.documentAssetId) ?? []).some( + (member) => member.generationId !== expectedGeneration, + ) + ) { + throw candidateError( + "PROFILE_MIGRATION_VECTOR_REBUILD_INCOMPLETE", + `Document ${document.documentAssetId} Graph lineage is incomplete`, + ); + } const baseOwned = baseProjectionMembers .filter((member) => member.documentAssetId === document.documentAssetId) .flatMap((member) => { @@ -620,6 +815,11 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ } const baseDocumentIds = new Set(base.documents.map((document) => document.documentAssetId)); if ( + candidateMembers.some( + (member) => + isGraphMember(member) && + (!member.documentAssetId || !baseDocumentIds.has(member.documentAssetId)), + ) || projectionMembers.some( (member) => !preservedProjectionIds.has(member.componentKey) && @@ -691,6 +891,18 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ if (input.rebuildScope === "clone-publication") { nextMembers = base.members.map(memberInput); } else if (input.rebuildScope === "full-page-index-summary-outline") { + if (!paths) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_UNAVAILABLE", + "Reasoning migration requires outline-derived KnowledgeFS path persistence", + ); + } + if (!input.baseEmbeddingProfile) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_UNAVAILABLE", + "Reasoning migration requires the frozen active embedding profile", + ); + } const retrieval = await requireProfile( profiles, input, @@ -699,52 +911,12 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ "candidate", ); const retrievalProfile = KnowledgeSpaceRetrievalProfileSchema.parse(retrieval.snapshot); - const rebuilt: KnowledgeSpaceProfileMigrationCandidateMemberInput[] = []; - for (const document of base.documents) { - await input.execution?.heartbeat(); - const generationId = migrationGenerationId( - input.runId, - "page-index", - document.documentAssetId, - ); - const deterministicOutline = outlineBuilder.build({ - knowledgeSpaceId: input.knowledgeSpaceId, - parseArtifact: document.artifact, - publicationGenerationId: generationId, - }); - const enhanced = await outlineSummaryEnhancer.enhance({ - outline: deterministicOutline, - parseArtifact: document.artifact, - retrievalProfile, - tenantId: input.tenantId, - }); - const outline = await outlines.upsert(enhanced); - await pageIndexBuild.materializeBuilding({ - builtAt: outline.updatedAt ?? outline.createdAt, - outline, - tenantId: input.tenantId, - }); - rebuilt.push({ - componentKey: outline.id, - componentType: "document-outline", - documentAssetId: document.documentAssetId, - generationId, - }); - await input.execution?.heartbeat(); - } - nextMembers = [ - ...base.members - .filter((member) => member.componentType !== "document-outline") - .map(memberInput), - ...rebuilt, - ]; - } else { const embedding = await requireProfile( profiles, input, "embedding", - input.candidateProfile, - "candidate", + input.baseEmbeddingProfile, + "active", ); const embeddingProfile = KnowledgeSpaceEmbeddingProfileSchema.parse(embedding.snapshot); const baseProjectionMembers = base.members.filter( @@ -761,7 +933,185 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ .filter((projection) => !isOrdinarySearchProjection(projection)) .map((projection) => projection.id), ); + const baseDerivedPathIds = await resolveBaseOutlineDerivedPathIds({ + documents: base.documents, + maxPaths: maxPathsPerDocument, + pageSize: maxPathReadPageSize, + paths, + tenantId: input.tenantId, + }); const rebuilt: KnowledgeSpaceProfileMigrationCandidateMemberInput[] = []; + for (const document of base.documents) { + await input.execution?.heartbeat(); + const generationId = migrationGenerationId( + input.runId, + "page-index", + document.documentAssetId, + ); + const reindexResult = await reindexer.reindex({ + denseModel: embeddingProfile.vectorSpaceId, + embeddingProfile, + enableGraph: true, + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifact: document.artifact, + permissionScope: stringArray(document.asset.metadata.permissionScope), + projectionStatus: "ready", + projectionVersion: document.asset.version, + publicationGenerationId: generationId, + retrievalProfile, + skipVisual: true, + tenantId: input.tenantId, + }); + if ( + reindexResult.status !== "rebuilt" || + !reindexResult.outlineArtifact || + !reindexResult.projectionIds || + reindexResult.projectionIds.length === 0 || + reindexResult.projectionIds.length !== reindexResult.projectionsCreated || + (reindexResult.nodeIds?.length ?? 0) !== reindexResult.nodesCreated + ) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_REBUILD_INCOMPLETE", + `Document ${document.documentAssetId} did not produce a complete semantic generation receipt`, + ); + } + const deterministicOutline = outlineBuilder.build({ + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifact: reindexResult.outlineArtifact, + publicationGenerationId: generationId, + }); + const enhanced = await outlineSummaryEnhancer.enhance({ + outline: deterministicOutline, + parseArtifact: reindexResult.outlineArtifact, + retrievalProfile, + tenantId: input.tenantId, + }); + const outline = await outlines.upsert(enhanced); + await pageIndexBuild.materializeBuilding({ + builtAt: outline.updatedAt ?? outline.createdAt, + outline, + tenantId: input.tenantId, + }); + const rebuiltPaths = buildOutlineDerivedPaths({ + asset: document.asset, + outline, + publicationGenerationId: generationId, + tenantId: input.tenantId, + }); + await persistOutlineDerivedPaths({ + batchSize: maxProjectionBatchSize, + expected: rebuiltPaths, + paths, + }); + rebuilt.push( + { + componentKey: outline.id, + componentType: "document-outline", + documentAssetId: document.documentAssetId, + generationId, + }, + ...rebuiltPaths.map((path) => ({ + componentKey: path.id, + componentType: "knowledge-path" as const, + documentAssetId: document.documentAssetId, + generationId, + })), + ); + rebuilt.push( + ...reindexResult.projectionIds.map((componentKey) => ({ + componentKey, + componentType: "index-projection" as const, + documentAssetId: document.documentAssetId, + generationId, + })), + ); + if (semanticGraph) { + const graph = await semanticGraph.materialize({ + createdAt: candidate.createdAt, + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifactId: document.artifact.id, + publicationGenerationId: generationId, + retrievalProfile, + }); + rebuilt.push( + ...graph.graphEntityIds.map((componentKey) => ({ + componentKey, + componentType: "graph-entity" as const, + documentAssetId: document.documentAssetId, + generationId, + })), + ...graph.graphRelationIds.map((componentKey) => ({ + componentKey, + componentType: "graph-relation" as const, + documentAssetId: document.documentAssetId, + generationId, + })), + ); + } + await input.execution?.heartbeat(); + } + nextMembers = [ + ...base.members + .filter( + (member) => + member.componentType !== "document-outline" && + !(semanticGraph && isGraphMember(member)) && + !( + member.componentType === "knowledge-path" && + baseDerivedPathIds.has(member.componentKey) + ) && + (member.componentType !== "index-projection" || + preservedProjectionIds.has(member.componentKey)), + ) + .map(memberInput), + ...rebuilt, + ]; + } else { + const embedding = await requireProfile( + profiles, + input, + "embedding", + input.candidateProfile, + "candidate", + ); + const embeddingProfile = KnowledgeSpaceEmbeddingProfileSchema.parse(embedding.snapshot); + const retrieval = await requireProfile( + profiles, + input, + "retrieval", + input.baseRetrievalProfile, + "active", + ); + const retrievalProfile = KnowledgeSpaceRetrievalProfileSchema.parse(retrieval.snapshot); + const baseProjectionMembers = base.members.filter( + (member) => member.componentType === "index-projection", + ); + const baseProjections = await loadProjections( + projections, + baseProjectionMembers.map((member) => member.componentKey), + input.knowledgeSpaceId, + maxProjectionBatchSize, + ); + const baseProjectionById = new Map( + baseProjections.map((projection) => [projection.id, projection]), + ); + const ordinaryNodeGenerationsByDocument = new Map>(); + for (const member of baseProjectionMembers) { + if (!member.documentAssetId) continue; + const projection = baseProjectionById.get(member.componentKey); + if (!projection || !isOrdinarySearchProjection(projection)) continue; + const generations = + ordinaryNodeGenerationsByDocument.get(member.documentAssetId) ?? new Set(); + generations.add(member.generationId); + ordinaryNodeGenerationsByDocument.set(member.documentAssetId, generations); + } + const preservedProjectionIds = new Set( + baseProjections + .filter((projection) => !isOrdinarySearchProjection(projection)) + .map((projection) => projection.id), + ); + const rebuilt: KnowledgeSpaceProfileMigrationCandidateMemberInput[] = []; + const rebuiltGraph: KnowledgeSpaceProfileMigrationCandidateMemberInput[] = []; for (const document of base.documents) { await input.execution?.heartbeat(); const generationId = migrationGenerationId( @@ -769,6 +1119,15 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ "vector-space", document.documentAssetId, ); + const sourceNodeGenerations = + ordinaryNodeGenerationsByDocument.get(document.documentAssetId) ?? new Set(); + if (sourceNodeGenerations.size !== 1) { + throw candidateError( + "PROFILE_MIGRATION_VECTOR_REBUILD_INCOMPLETE", + `Document ${document.documentAssetId} must have exactly one reusable ordinary node generation`, + ); + } + const reuseNodeGenerationId = [...sourceNodeGenerations][0] as string; const result = await reindexer.reindex({ denseModel: embeddingProfile.vectorSpaceId, embeddingProfile, @@ -778,6 +1137,9 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ projectionStatus: "ready", projectionVersion: document.asset.version, publicationGenerationId: generationId, + retrievalProfile, + reuseNodeGenerationId, + skipVisual: true, tenantId: input.tenantId, }); if ( @@ -799,17 +1161,42 @@ export function createRepositoryKnowledgeSpaceProfileMigrationCandidateBuilder({ generationId, })), ); + if (semanticGraph) { + const graph = await semanticGraph.materialize({ + createdAt: candidate.createdAt, + knowledgeSpaceId: input.knowledgeSpaceId, + parseArtifactId: document.artifact.id, + publicationGenerationId: generationId, + retrievalProfile, + }); + rebuiltGraph.push( + ...graph.graphEntityIds.map((componentKey) => ({ + componentKey, + componentType: "graph-entity" as const, + documentAssetId: document.documentAssetId, + generationId, + })), + ...graph.graphRelationIds.map((componentKey) => ({ + componentKey, + componentType: "graph-relation" as const, + documentAssetId: document.documentAssetId, + generationId, + })), + ); + } await input.execution?.heartbeat(); } nextMembers = [ ...base.members .filter( (member) => - member.componentType !== "index-projection" || - preservedProjectionIds.has(member.componentKey), + !(semanticGraph && isGraphMember(member)) && + (member.componentType !== "index-projection" || + preservedProjectionIds.has(member.componentKey)), ) .map(memberInput), ...rebuilt, + ...rebuiltGraph, ]; } if (nextMembers.length > maxMembers) { @@ -894,10 +1281,22 @@ export function createRepositoryKnowledgeSpaceProfileMigrationEvaluator({ assertSameMemberSnapshot(baseMembers, candidateMembers); } else if (run.rebuildScope === "full-page-index-summary-outline") { assertSameMemberSnapshot( - baseMembers.filter((member) => member.componentType !== "document-outline"), - candidateMembers.filter((member) => member.componentType !== "document-outline"), + baseMembers.filter( + (member) => + member.componentType !== "document-outline" && + member.componentType !== "index-projection" && + member.componentType !== "knowledge-path" && + !isGraphMember(member), + ), + candidateMembers.filter( + (member) => + member.componentType !== "document-outline" && + member.componentType !== "index-projection" && + member.componentType !== "knowledge-path" && + !isGraphMember(member), + ), "PROFILE_MIGRATION_PAGE_INDEX_REBUILD_INCOMPLETE", - "Reasoning evaluation found a changed or missing non-outline publication member", + "Reasoning evaluation found a changed or missing preserved publication member", ); if ( candidateMembers.filter((member) => member.componentType === "document-outline") @@ -910,8 +1309,12 @@ export function createRepositoryKnowledgeSpaceProfileMigrationEvaluator({ } } else { assertSameMemberSnapshot( - baseMembers.filter((member) => member.componentType !== "index-projection"), - candidateMembers.filter((member) => member.componentType !== "index-projection"), + baseMembers.filter( + (member) => member.componentType !== "index-projection" && !isGraphMember(member), + ), + candidateMembers.filter( + (member) => member.componentType !== "index-projection" && !isGraphMember(member), + ), "PROFILE_MIGRATION_VECTOR_REBUILD_INCOMPLETE", "Embedding evaluation found a changed or missing non-index publication member", ); @@ -928,6 +1331,29 @@ export function createRepositoryKnowledgeSpaceProfileMigrationEvaluator({ ) { return failedEvaluation("candidate document ownership differs from the frozen base"); } + if (run.rebuildScope === "full-page-index-summary-outline") { + for (const documentAssetId of candidateDocuments) { + const expectedGeneration = migrationGenerationId(run.id, "page-index", documentAssetId); + if ( + !candidateMembers.some( + (member) => + member.componentType === "knowledge-path" && + member.documentAssetId === documentAssetId && + member.generationId === expectedGeneration, + ) || + candidateMembers.some( + (member) => + isGraphMember(member) && + member.documentAssetId === documentAssetId && + member.generationId !== expectedGeneration, + ) + ) { + return failedEvaluation( + `document ${documentAssetId} semantic path or Graph generation is incomplete`, + ); + } + } + } if (baseMembers.length === 0 && candidateMembers.length === 0) { return { passed: true, @@ -1054,6 +1480,15 @@ export function createRepositoryKnowledgeSpaceProfileMigrationEvaluator({ ) { return failedEvaluation(`projection ${member.componentKey} lineage is invalid`); } + if ( + reasoningProfile !== undefined && + isOrdinarySearchProjection(projection) && + member.generationId !== migrationGenerationId(run.id, "page-index", documentAssetId) + ) { + return failedEvaluation( + `document ${documentAssetId} contains a stale reasoning search projection`, + ); + } if (projection.type === "fts") { ftsProjections += 1; ftsCount += 1; @@ -1352,6 +1787,198 @@ function isOrdinarySearchProjection(projection: IndexProjection | undefined): bo ); } +function isGraphMember(member: Pick): boolean { + return member.componentType === "graph-entity" || member.componentType === "graph-relation"; +} + +function buildOutlineDerivedPaths({ + asset, + outline, + publicationGenerationId, + tenantId, +}: { + readonly asset: CandidateDocument["asset"]; + readonly outline: DocumentOutline; + readonly publicationGenerationId: string; + readonly tenantId: string; +}): readonly KnowledgePath[] { + let sequence = 0; + const generateId = () => + deterministicChildId(publicationGenerationId, `reasoning-path-seed:${sequence++}`); + const derived = [ + buildDocumentOutlineKnowledgePath({ + asset, + id: generateId(), + publicationGenerationId, + tenantId, + }), + ...buildDocumentSectionKnowledgePaths({ + asset, + generateId, + outline, + publicationGenerationId, + tenantId, + }), + ]; + if ( + new Set(derived.map((path) => path.id)).size !== derived.length || + new Set(derived.map((path) => path.virtualPath)).size !== derived.length + ) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_PATH_REBUILD_INCOMPLETE", + `Document ${asset.id} produced duplicate outline-derived KnowledgeFS paths`, + ); + } + return derived; +} + +async function persistOutlineDerivedPaths({ + batchSize, + expected, + paths, +}: { + readonly batchSize: number; + readonly expected: readonly KnowledgePath[]; + readonly paths: Pick; +}): Promise { + const persisted: KnowledgePath[] = []; + for (const batch of batches(expected, batchSize)) { + persisted.push(...(await paths.upsertMany(batch))); + } + assertExactKnowledgePaths(expected, persisted); +} + +async function assertOutlineDerivedPathClosure({ + expected, + maxPaths, + members, + pageSize, + paths, +}: { + readonly expected: readonly KnowledgePath[]; + readonly maxPaths: number; + readonly members: readonly Pick< + ProjectionSetPublicationMember, + "componentKey" | "componentType" | "documentAssetId" | "generationId" + >[]; + readonly pageSize: number; + readonly paths: Pick; +}): Promise { + assertSameMemberSnapshot( + expected.map((path) => ({ + componentKey: path.id, + componentType: "knowledge-path" as const, + documentAssetId: path.targetId, + generationId: path.publicationGenerationId as string, + })), + members, + "PROFILE_MIGRATION_REASONING_PATH_REBUILD_INCOMPLETE", + `Document ${expected[0]?.targetId ?? "unknown"} outline-derived path membership is incomplete`, + ); + const stored = await listDocumentGenerationPaths({ + anchor: expected[0] as KnowledgePath, + maxPaths, + pageSize, + paths, + }); + const expectedVirtualPaths = new Set(expected.map((path) => path.virtualPath)); + assertExactKnowledgePaths( + expected, + stored.filter((path) => expectedVirtualPaths.has(path.virtualPath)), + ); +} + +async function listDocumentGenerationPaths({ + anchor, + maxPaths, + pageSize, + paths, +}: { + readonly anchor: KnowledgePath; + readonly maxPaths: number; + readonly pageSize: number; + readonly paths: Pick; +}): Promise { + const parentPath = anchor.virtualPath.replace(/\/outline\.json$/u, ""); + const matched: KnowledgePath[] = []; + let cursor: Awaited>["nextCursor"]; + do { + const page = await paths.listPhysicalDescendants({ + ...(cursor ? { cursor } : {}), + knowledgeSpaceId: anchor.knowledgeSpaceId, + limit: Math.min(pageSize, maxPaths - matched.length), + parentPath, + publicationGenerationId: anchor.publicationGenerationId, + viewName: anchor.viewName, + }); + matched.push(...page.items); + cursor = page.nextCursor; + if (cursor && matched.length >= maxPaths) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_PATH_REBUILD_INCOMPLETE", + `Document ${anchor.targetId} path count exceeds ${maxPaths}`, + ); + } + } while (cursor); + return matched; +} + +async function resolveBaseOutlineDerivedPathIds({ + documents, + maxPaths, + pageSize, + paths, + tenantId, +}: { + readonly documents: readonly CandidateDocument[]; + readonly maxPaths: number; + readonly pageSize: number; + readonly paths: Pick; + readonly tenantId: string; +}): Promise> { + const ids = new Set(); + for (const document of documents) { + const expected = buildOutlineDerivedPaths({ + asset: document.asset, + outline: document.baseOutline, + publicationGenerationId: PublicationGenerationIdSchema.parse( + document.baseOutline.publicationGenerationId, + ), + tenantId, + }); + const resolved = await listDocumentGenerationPaths({ + anchor: expected[0] as KnowledgePath, + maxPaths, + pageSize, + paths, + }); + for (const path of resolved) { + const contentKind = path.metadata.contentKind; + if ( + path.targetId === document.documentAssetId && + (contentKind === "document-outline" || contentKind === "document-section") + ) { + ids.add(path.id); + } + } + } + return ids; +} + +function assertExactKnowledgePaths( + expected: readonly KnowledgePath[], + actual: readonly KnowledgePath[], +): void { + const left = expected.map((path) => stableJson(path)).sort(); + const right = actual.map((path) => stableJson(path)).sort(); + if (left.length !== right.length || left.some((value, index) => value !== right[index])) { + throw candidateError( + "PROFILE_MIGRATION_REASONING_PATH_REBUILD_INCOMPLETE", + "Outline-derived KnowledgeFS path receipt is incomplete or incompatible", + ); + } +} + function failedEvaluation(reason: string): KnowledgeSpaceProfileMigrationEvaluationResult { return { passed: false, summary: { reason: reason.slice(0, 512) } }; } diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts new file mode 100644 index 00000000000..3e26083bd1f --- /dev/null +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.test.ts @@ -0,0 +1,2020 @@ +import { + type KnowledgeNode, + KnowledgeNodeSchema, + type KnowledgeSpaceModelSelection, + KnowledgeSpaceRetrievalProfileSchema, + type ParseArtifact, + ParseArtifactSchema, +} from "@knowledge/core"; +import { countGraphemes } from "unicode-segmenter/grapheme"; +import { describe, expect, it } from "vitest"; + +import { + DEFAULT_MAX_SEMANTIC_WINDOWS, + type LlmSemanticCompletionCatalogEntry, + type LlmSemanticWindowManifestEntry, + type SemanticChunkingLlmProvider, + type SemanticChunkingLlmStreamInput, + assertValidLlmSemanticGenerationReplay, + assertValidLlmSemanticWindowManifestReplay, + createLlmSemanticChunker, + hasValidLlmSemanticJointExtraction, + llmSemanticCompletionFingerprint, + preflightLlmSemanticWindows, +} from "./llm-semantic-chunker"; + +const KNOWLEDGE_SPACE_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c41"; +const DOCUMENT_ASSET_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42"; +const PARSE_ARTIFACT_ID = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43"; +const GENERATION_A = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44"; +const GENERATION_B = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c45"; + +interface PromptUnit { + readonly graphemeLength: number; + readonly id: string; + readonly text: string; + readonly type: string; +} + +interface PromptPayload { + readonly lookAheadUnits?: readonly PromptUnit[]; + readonly sectionPath: readonly string[]; + readonly units: readonly PromptUnit[]; + readonly windowId: string; +} + +type Script = (payload: PromptPayload) => unknown; + +class ScriptedProvider implements SemanticChunkingLlmProvider { + readonly calls: SemanticChunkingLlmStreamInput[] = []; + readonly kind: string; + private callIndex = 0; + + constructor( + private readonly scripts: readonly Script[], + private readonly terminal?: + | { + readonly finishReason?: string | undefined; + readonly metadata?: unknown; + } + | undefined, + kind = "test-plugin-daemon", + ) { + this.kind = kind; + } + + async *stream(input: SemanticChunkingLlmStreamInput): AsyncIterable<{ + delta?: string; + finishReason?: string; + metadata?: unknown; + type: "delta" | "done"; + }> { + this.calls.push(input); + const script = this.scripts[this.callIndex] ?? this.scripts.at(-1); + this.callIndex += 1; + if (!script) { + throw new Error("Missing test response script"); + } + const userMessage = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(userMessage?.content ?? "{}") as PromptPayload; + const response = JSON.stringify(script(payload)); + const midpoint = Math.ceil(response.length / 2); + yield { delta: response.slice(0, midpoint), type: "delta" }; + yield { delta: response.slice(midpoint), type: "delta" }; + yield { + finishReason: this.terminal?.finishReason ?? "stop", + metadata: this.terminal?.metadata ?? { model: input.model, provider: "plugin-daemon" }, + type: "done", + }; + } +} + +describe("LLM semantic chunker", () => { + it("uses the frozen reasoning selection and accepts semantic boundaries below the hard cap", async () => { + const provider = new ScriptedProvider([ + ({ units }) => ({ + chunks: [ + { + endUnitId: units[1]?.id, + entities: [ + { + aliases: ["Policy A", "Policy A"], + canonicalName: "Policy A", + confidence: 0.96, + id: "e-policy", + text: "策略 A", + type: "policy", + }, + { confidence: 0.92, id: "e-product", text: "产品 B", type: "product" }, + ], + relations: [ + { + confidence: 0.9, + objectEntityId: "e-product", + subjectEntityId: "e-policy", + type: "depends_on", + }, + ], + startUnitId: units[0]?.id, + }, + { + endUnitId: units[2]?.id, + entities: [], + relations: [], + startUnitId: units[2]?.id, + }, + ], + }), + ]); + const selections: KnowledgeSpaceModelSelection[] = []; + const chunker = createLlmSemanticChunker({ + now: () => "2026-07-19T12:00:00.000Z", + reasoningProviderFactory: (selection) => { + selections.push(selection); + return provider; + }, + }); + const nodes = await chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "paragraph-1", + metadata: {}, + pageNumber: 2, + sectionPath: ["Setup"], + text: " 策略 A 已发布。策略 A 依赖产品 B。独立主题。 ", + type: "paragraph", + }, + ]), + permissionScope: ["team:platform"], + retrievalProfile: profile(), + tenantId: "tenant-1", + }); + + expect(nodes).toHaveLength(2); + expect(nodes.map((node) => node.text)).toEqual([ + "策略 A 已发布。策略 A 依赖产品 B。", + "独立主题。", + ]); + expect(nodes[0]).toMatchObject({ + endOffset: new TextEncoder().encode("策略 A 已发布。策略 A 依赖产品 B。").byteLength, + kind: "chunk", + permissionScope: ["team:platform"], + sourceLocation: { pageNumber: 2, sectionPath: ["Setup"], startOffset: 0 }, + startOffset: 0, + }); + expect(nodes[0]?.metadata).toMatchObject({ + entityExtraction: { + completed: true, + entityCount: 2, + model: "reasoner-model", + promptVersion: "semantic-chunking-v1", + }, + relationExtraction: { completed: true, relationCount: 1 }, + semanticChunking: { + completed: true, + completion: { + actual: { finishReason: "stop", model: "reasoner-model", provider: "plugin-daemon" }, + requested: profile().reasoningModel, + }, + documentChunkCount: 2, + model: "reasoner-model", + provider: "test-plugin-daemon", + schemaVersion: 1, + strategy: "llm-semantic-v1", + }, + }); + expect(nodes[0]?.metadata.extractedRelations).toEqual([ + { + confidence: 0.9, + metadata: { + objectEntityId: "e-product", + source: "llm-semantic-chunking", + subjectEntityId: "e-policy", + }, + object: "产品 B", + subject: "Policy A", + type: "depends_on", + }, + ]); + expect( + (nodes[0]?.metadata.extractedEntities as Array<{ metadata?: { aliases?: string[] } }>)[0] + ?.metadata?.aliases, + ).toEqual(["Policy A"]); + expect( + nodes.every( + (node) => + (node.metadata.semanticChunking as { documentChunkCount?: number }).documentChunkCount === + 2, + ), + ).toBe(true); + expect(selections).toEqual([profile().reasoningModel]); + expect(provider.calls).toHaveLength(1); + expect(provider.calls[0]).toMatchObject({ + model: "reasoner-model", + temperature: 0, + tenantId: "tenant-1", + }); + expect(provider.calls[0]?.messages[0]?.content).toContain( + "prefer natural topic boundaries over filling chunks", + ); + }); + + it("hard-splits an overlong sentence by Unicode grapheme without overlap", async () => { + const provider = new ScriptedProvider([echoEachUnit]); + const chunker = createLlmSemanticChunker({ + maxChunkChars: 3, + maxWindowChars: 12, + reasoningProviderFactory: () => provider, + }); + const text = "A👨‍👩‍👧‍👦BCDE"; + const nodes = await chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "long-sentence", + metadata: {}, + sectionPath: ["Unicode"], + text, + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }); + + expect(nodes.map((node) => node.text)).toEqual(["A👨‍👩‍👧‍👦B", "CDE"]); + expect(nodes.every((node) => countGraphemes(node.text) <= 3)).toBe(true); + expect(nodes.map((node) => node.text).join("")).toBe(text); + expect(nodes[0]?.endOffset).toBe(nodes[1]?.startOffset); + expect(nodes[1]?.endOffset).toBe(new TextEncoder().encode(text).byteLength); + }); + + it("records legacy overlap as unapplied provenance and keeps semantic chunks contiguous", async () => { + const chunker = createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoEachUnit]), + }); + const text = "Alpha. Beta."; + const nodes = await chunker.chunk({ + config: { maxChunkChars: 20, overlapChars: 5 }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "legacy-overlap", + metadata: {}, + sectionPath: ["Overlap"], + text, + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }); + + expect(nodes).toHaveLength(2); + expect(nodes.map((node) => node.text).join("")).toBe(text); + expect(nodes[0]?.endOffset).toBe(nodes[1]?.startOffset); + expect(nodes[0]?.metadata.semanticChunking).toMatchObject({ + overlapApplied: false, + overlapPolicy: "non-overlapping-semantic-output", + requestedOverlapChars: 5, + }); + }); + + it("never crosses sections and isolates table/image windows without degrading later context", async () => { + const provider = new ScriptedProvider([ + echoWholeWindow, + echoWholeWindow, + echoWholeWindow, + echoWholeWindow, + echoWholeWindow, + ]); + const chunker = createLlmSemanticChunker({ reasoningProviderFactory: () => provider }); + const nodes = await chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "section-a", + metadata: {}, + sectionPath: ["A"], + text: "Section A.", + type: "paragraph", + }, + { + id: "before-table", + metadata: {}, + sectionPath: ["B"], + text: "Before table.", + type: "paragraph", + }, + { + id: "table-1", + metadata: { table: { rows: 1 }, title: "Metrics" }, + sectionPath: ["B"], + text: "Metric | Value", + type: "table", + }, + { + id: "image-1", + metadata: { assetRef: { objectKey: "assets/image.png" }, caption: "Architecture" }, + sectionPath: ["B"], + text: "Architecture diagram", + type: "image", + }, + { + id: "after-image-1", + metadata: {}, + sectionPath: ["B"], + text: "After image one.", + type: "paragraph", + }, + { + id: "after-image-2", + metadata: {}, + sectionPath: ["B"], + text: "After image two.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }); + + expect(provider.calls).toHaveLength(5); + const promptWindows = provider.calls.map((call) => + JSON.parse(call.messages.find((message) => message.role === "user")?.content ?? "{}"), + ) as PromptPayload[]; + expect(promptWindows.map((window) => window.units.map((unit) => unit.type))).toEqual([ + ["paragraph"], + ["paragraph"], + ["table"], + ["image"], + ["paragraph", "paragraph"], + ]); + expect(nodes.map((node) => node.kind)).toEqual(["chunk", "chunk", "table", "image", "chunk"]); + expect(nodes[2]?.metadata).toMatchObject({ table: { rows: 1 }, title: "Metrics" }); + expect(nodes[3]?.metadata).toMatchObject({ + assetRef: { objectKey: "assets/image.png" }, + caption: "Architecture", + }); + expect(nodes[4]?.text).toBe("After image one.\nAfter image two."); + }); + + it("lets the reasoning model replace unproven Unstructured title boundaries", async () => { + const provider = new ScriptedProvider([ + ({ units }) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units.at(-1)?.id), + sectionPath: ["电子发票", "购买方与金额"], + sectionSummary: "包含发票号码、购方身份和价税合计。", + }, + ], + }), + ]); + const chunker = createLlmSemanticChunker({ reasoningProviderFactory: () => provider }); + const parseArtifact = ParseArtifactSchema.parse({ + ...artifact([ + { + id: "invoice-title", + metadata: {}, + pageNumber: 1, + sectionPath: ["电子发票（普通发票）"], + text: "电子发票（普通发票）", + type: "title", + }, + { + id: "invoice-number", + metadata: {}, + pageNumber: 1, + sectionPath: ["电子发票（普通发票）"], + text: "发票号码：26322000000000000000", + type: "paragraph", + }, + { + id: "false-company-title", + metadata: {}, + pageNumber: 1, + sectionPath: ["名称：示例人工智能有限公司"], + text: "名称：示例人工智能有限公司", + type: "title", + }, + { + id: "buyer-tax-id", + metadata: {}, + pageNumber: 1, + sectionPath: ["名称：示例人工智能有限公司"], + text: "统一社会信用代码：91320506EXAMPLE01", + type: "paragraph", + }, + { + id: "false-tax-id-title", + metadata: {}, + pageNumber: 1, + sectionPath: ["91320506EXAMPLE02"], + text: "91320506EXAMPLE02", + type: "title", + }, + { + id: "totals", + metadata: {}, + pageNumber: 1, + sectionPath: ["91320506EXAMPLE02"], + text: "餐饮服务，合计：566.00", + type: "paragraph", + }, + ]), + parser: "unstructured", + }); + + const nodes = await chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + + expect(provider.calls).toHaveLength(1); + expect(nodes).toHaveLength(1); + expect(nodes[0]?.sourceLocation.sectionPath).toEqual(["电子发票", "购买方与金额"]); + expect(nodes[0]?.text).toBe( + [ + "电子发票（普通发票）", + "发票号码：26322000000000000000", + "名称：示例人工智能有限公司", + "统一社会信用代码：91320506EXAMPLE01", + "91320506EXAMPLE02", + "餐饮服务，合计：566.00", + ].join("\n"), + ); + expect(nodes[0]?.metadata.semanticChunking).toMatchObject({ + layoutRecomposition: { + modelDecidedHeadingBoundaries: 3, + trustedHeadingBoundaries: 0, + }, + section: { + path: ["电子发票", "购买方与金额"], + summary: "包含发票号码、购方身份和价税合计。", + }, + }); + }); + + it("rebases deterministic node IDs onto the immutable publication generation", async () => { + const chunker = createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow]), + }); + const input = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "paragraph", + metadata: {}, + sectionPath: ["A"], + text: "Stable text.", + type: "paragraph" as const, + }, + ]), + retrievalProfile: profile(), + }; + const [first] = await chunker.chunk({ ...input, publicationGenerationId: GENERATION_A }); + const [replay] = await chunker.chunk({ ...input, publicationGenerationId: GENERATION_A }); + const [nextGeneration] = await chunker.chunk({ + ...input, + publicationGenerationId: GENERATION_B, + }); + + expect(replay?.id).toBe(first?.id); + expect(nextGeneration?.id).not.toBe(first?.id); + expect(first?.publicationGenerationId).toBe(GENERATION_A); + expect(nextGeneration?.publicationGenerationId).toBe(GENERATION_B); + }); + + it.each([ + { + label: "gapped coverage", + response: ({ units }: PromptPayload) => ({ + chunks: [chunkRange(units[1]?.id, units[1]?.id)], + }), + error: "contiguously without gaps or overlap", + }, + { + label: "unknown unit", + response: ({ units }: PromptPayload) => ({ + chunks: [chunkRange(units[0]?.id, "u-missing")], + }), + error: "unknown unit ID", + }, + { + label: "relation to an entity from outside the chunk", + response: ({ units }: PromptPayload) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units.at(-1)?.id), + entities: [{ confidence: 1, id: "e-alpha", text: "Alpha", type: "term" }], + relations: [ + { + confidence: 1, + objectEntityId: "e-missing", + subjectEntityId: "e-alpha", + type: "references", + }, + ], + }, + ], + }), + error: "relation endpoint ids must reference entities in the same chunk", + }, + { + label: "hallucinated entity text", + response: ({ units }: PromptPayload) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units.at(-1)?.id), + entities: [ + { confidence: 1, id: "e-hallucination", text: "Hallucination", type: "term" }, + ], + }, + ], + }), + error: "entity text must be an exact chunk substring", + }, + { + label: "duplicate response-local entity ids", + response: ({ units }: PromptPayload) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units.at(-1)?.id), + entities: [ + { confidence: 1, id: "e-same", text: "Alpha", type: "term" }, + { confidence: 1, id: "e-same", text: "Beta", type: "term" }, + ], + }, + ], + }), + error: "entity ids must be unique", + }, + ])("rejects $label", async ({ response, error }) => { + const chunker = createLlmSemanticChunker({ + maxChunkChars: 20, + reasoningProviderFactory: () => new ScriptedProvider([response]), + }); + + await expect( + chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "paragraph", + metadata: {}, + sectionPath: ["Validation"], + text: "Alpha. Beta.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow(error); + }); + + it("rejects a model range over the Unicode-grapheme hard limit", async () => { + const provider = new ScriptedProvider([ + ({ units }) => ({ + chunks: [chunkRange(units[0]?.id, units.at(-1)?.id)], + }), + ]); + const chunker = createLlmSemanticChunker({ + maxChunkChars: 7, + maxWindowChars: 20, + reasoningProviderFactory: () => provider, + }); + + await expect( + chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "paragraph", + metadata: {}, + sectionPath: ["Validation"], + text: "One. Two.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow("exceeded maxChunkChars=7"); + }); + + it("fails closed on invalid structured output or a missing terminal stream event", async () => { + const invalidSchema = new ScriptedProvider([ + ({ units }) => ({ + chunks: [{ ...chunkRange(units[0]?.id, units[0]?.id), text: "rewritten" }], + }), + ]); + const input = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "paragraph", + metadata: {}, + sectionPath: [], + text: "Original.", + type: "paragraph" as const, + }, + ]), + retrievalProfile: profile(), + }; + await expect( + createLlmSemanticChunker({ reasoningProviderFactory: () => invalidSchema }).chunk(input), + ).rejects.toThrow("invalid response schema"); + + const noTerminalProvider: SemanticChunkingLlmProvider = { + async *stream() { + yield { delta: '{"chunks":[]}', type: "delta" }; + }, + }; + await expect( + createLlmSemanticChunker({ reasoningProviderFactory: () => noTerminalProvider }).chunk(input), + ).rejects.toThrow("without a terminal event"); + }); + + it("rejects a terminal actual model that differs from the frozen reasoning selection", async () => { + const provider = new ScriptedProvider([echoWholeWindow], { + finishReason: "stop", + metadata: { model: "silently-routed-model", provider: "plugin-daemon" }, + }); + await expect( + createLlmSemanticChunker({ reasoningProviderFactory: () => provider }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "model-identity", + metadata: {}, + sectionPath: [], + text: "Frozen model content.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow("expected frozen model=reasoner-model"); + }); + + it("recomputes canonical generation identity and fails closed on replay corruption", async () => { + const parseArtifact = artifact([ + { + id: "replay", + metadata: {}, + pageNumber: 4, + sectionPath: ["Replay"], + text: "Replay-safe content.", + type: "paragraph", + }, + ]); + const permissionScope = ["tenant:one", "team:search"]; + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow]), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + retrievalProfile: profile(), + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes, + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + }), + ).not.toThrow(); + + const node = nodes[0] as (typeof nodes)[number]; + const marker = node.metadata.semanticChunking as Record; + const corruptedFingerprint = KnowledgeNodeSchema.parse({ + ...node, + metadata: { + ...node.metadata, + semanticChunking: { ...marker, inputFingerprint: `sha256:${"0".repeat(64)}` }, + }, + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes: [corruptedFingerprint], + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + }), + ).toThrow("window fingerprint"); + + const corruptedAcl = KnowledgeNodeSchema.parse({ ...node, permissionScope: ["tenant:other"] }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes: [corruptedAcl], + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + }), + ).toThrow("ACL"); + + const corruptedIdentity = KnowledgeNodeSchema.parse({ ...node, id: GENERATION_B }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes: [corruptedIdentity], + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + }), + ).toThrow("identity"); + + const corruptedLanguage = KnowledgeNodeSchema.parse({ + ...node, + metadata: { ...node.metadata, language: "fr" }, + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes: [corruptedLanguage], + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + }), + ).toThrow("language"); + + const impossibleDocumentCount = KnowledgeNodeSchema.parse({ + ...node, + metadata: { + ...node.metadata, + semanticChunking: { + ...marker, + documentChunkCount: Number.MAX_SAFE_INTEGER, + }, + }, + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + config: { maxNodes: 1 }, + modelSelection: profile().reasoningModel, + nodes: [impossibleDocumentCount], + parseArtifact, + permissionScope, + publicationGenerationId: GENERATION_A, + }), + ).toThrow("documentChunkCount exceeds maxNodes=1"); + }); + + it("requires frozen terminal identity when replay provenance says plugin-daemon", async () => { + const parseArtifact = artifact([ + { + id: "plugin-replay", + metadata: {}, + sectionPath: [], + text: "Plugin replay identity.", + type: "paragraph", + }, + ]); + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => + new ScriptedProvider([echoWholeWindow], undefined, "plugin-daemon"), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + const node = nodes[0] as (typeof nodes)[number]; + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes, + parseArtifact, + }), + ).not.toThrow(); + + for (const actual of [ + { model: "silently-routed-model", provider: "plugin-daemon" }, + { model: "reasoner-model", provider: "proxy" }, + { provider: "plugin-daemon" }, + { model: "reasoner-model" }, + ]) { + const semantic = node.metadata.semanticChunking as Record; + const completion = semantic.completion as Record; + const corrupted = KnowledgeNodeSchema.parse({ + ...node, + metadata: { + ...node.metadata, + semanticChunking: { + ...semantic, + completion: { ...completion, actual }, + }, + }, + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes: [corrupted], + parseArtifact, + }), + ).toThrow("incompatible semantic provenance"); + } + + const semantic = node.metadata.semanticChunking as Record; + const completion = semantic.completion as Record; + const nonPluginWithoutActualRoute = KnowledgeNodeSchema.parse({ + ...node, + metadata: { + ...node.metadata, + semanticChunking: { + ...semantic, + completion: { ...completion, actual: {} }, + provider: "custom-provider", + }, + }, + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + modelSelection: profile().reasoningModel, + nodes: [nonPluginWithoutActualRoute], + parseArtifact, + }), + ).not.toThrow(); + }); + + it("canonically replays a complete compact manifest for all- and middle-excluded windows", async () => { + const parseArtifact = artifact([ + { + id: "manifest-first", + metadata: {}, + sectionPath: ["First"], + text: "Alpha policy.", + type: "paragraph", + }, + { + id: "manifest-middle", + metadata: {}, + sectionPath: ["Middle"], + text: "Beta contract.", + type: "paragraph", + }, + { + id: "manifest-last", + metadata: {}, + sectionPath: ["Last"], + text: "Gamma proof.", + type: "paragraph", + }, + ]); + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => + new ScriptedProvider([echoWholeWindow], undefined, "plugin-daemon"), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + const receipt = compactWindowManifest(nodes); + + expect(receipt.windowManifest).toHaveLength(3); + expect(() => + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: receipt.completionCatalog, + documentChunkCount: nodes.length, + modelSelection: profile().reasoningModel, + parseArtifact, + windowManifest: receipt.windowManifest, + }), + ).not.toThrow(); + // Stored nodes may be empty or omit the complete middle window; canonical validation relies on + // the complete receipt manifest and therefore has no dependency on either visible node set. + expect(nodes.filter((_, index) => index !== 1)).toHaveLength(2); + }); + + it("fails closed on compact manifest window, completion, coverage, and cap corruption", async () => { + const parseArtifact = artifact([ + { + id: "compact-1", + metadata: {}, + sectionPath: ["A"], + text: "1234567.", + type: "paragraph", + }, + { + id: "compact-2", + metadata: {}, + sectionPath: ["A"], + text: "abc.", + type: "paragraph", + }, + { + id: "compact-3", + metadata: {}, + sectionPath: ["A"], + text: "def.", + type: "paragraph", + }, + ]); + const nodes = await createLlmSemanticChunker({ + maxChunkChars: 10, + maxWindowChars: 15, + reasoningProviderFactory: () => + new ScriptedProvider( + [ + ({ lookAheadUnits = [], units }) => ({ + chunks: [ + chunkRange(units[0]?.id, units[0]?.id), + chunkRange(units[1]?.id, lookAheadUnits[0]?.id), + ], + }), + ], + undefined, + "plugin-daemon", + ), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + const receipt = compactWindowManifest(nodes); + const validate = (overrides: { + completionCatalog?: readonly LlmSemanticCompletionCatalogEntry[]; + documentChunkCount?: number; + windowManifest?: readonly LlmSemanticWindowManifestEntry[]; + }) => + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: overrides.completionCatalog ?? receipt.completionCatalog, + config: { maxChunkChars: 10, maxWindowChars: 15 }, + documentChunkCount: overrides.documentChunkCount ?? nodes.length, + modelSelection: profile().reasoningModel, + parseArtifact, + windowManifest: overrides.windowManifest ?? receipt.windowManifest, + }); + const firstWindow = receipt.windowManifest[0] as LlmSemanticWindowManifestEntry; + const replaceWindow = ( + patch: Partial, + ): LlmSemanticWindowManifestEntry[] => [{ ...firstWindow, ...patch }]; + + expect(() => + validate({ + windowManifest: replaceWindow({ + coreUnitRange: [firstWindow.coreUnitRange[0], "u-000002-000000"], + }), + }), + ).toThrow("canonical core/look-ahead"); + expect(() => + validate({ + windowManifest: replaceWindow({ + coreUnitRange: { + endUnitId: firstWindow.coreUnitRange[1], + startUnitId: firstWindow.coreUnitRange[0], + } as unknown as LlmSemanticWindowManifestEntry["coreUnitRange"], + }), + }), + ).toThrow("canonical core/look-ahead"); + expect(() => + validate({ windowManifest: replaceWindow({ lookAheadUnitRange: undefined }) }), + ).toThrow("canonical core/look-ahead"); + expect(() => + validate({ + windowManifest: replaceWindow({ + committedUnitRange: [firstWindow.committedUnitRange[0], "u-000001-000000"], + }), + }), + ).toThrow("gap, overlap, or context-only range"); + expect(() => + validate({ + windowManifest: replaceWindow({ inputFingerprint: `sha256:${"0".repeat(64)}` }), + }), + ).toThrow("canonical core/look-ahead"); + expect(() => validate({ windowManifest: replaceWindow({ firstChunkIndex: 1 }) })).toThrow( + "invalid or unbounded chunk list", + ); + expect(() => + validate({ windowManifest: replaceWindow({ responseFingerprint: "not-a-hash" }) }), + ).toThrow("invalid or unbounded chunk list"); + + const completion = receipt.completionCatalog[0] as LlmSemanticCompletionCatalogEntry; + expect(() => + validate({ + completionCatalog: [{ ...completion, fingerprint: `sha256:${"0".repeat(64)}` }], + }), + ).toThrow("invalid or duplicate identity"); + const wrongCompletion = { + ...completion, + actualProvider: "proxy", + fingerprint: "", + }; + wrongCompletion.fingerprint = llmSemanticCompletionFingerprint(wrongCompletion); + expect(() => validate({ completionCatalog: [wrongCompletion] })).toThrow("frozen model"); + + expect(() => + validate({ + windowManifest: replaceWindow({ + chunkRanges: [ + firstWindow.chunkRanges[0] as readonly [string, string], + ["u-000002-000000", "u-000002-000000"], + ], + }), + }), + ).toThrow("gap, overlap, or context-only range"); + expect(() => + validate({ + windowManifest: replaceWindow({ + chunkRanges: [ + ["u-000000-000000", "u-000001-000000"], + firstWindow.chunkRanges[1] as readonly [string, string], + ], + }), + }), + ).toThrow("exceeds maxChunkChars=10"); + expect(() => validate({ documentChunkCount: Number.MAX_SAFE_INTEGER })).toThrow( + "documentChunkCount exceeds maxNodes=20000", + ); + }); + + it("validates empty and minimal compact manifests without trusting optional completion fields", async () => { + expect(() => + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: [], + documentChunkCount: 0, + modelSelection: profile().reasoningModel, + parseArtifact: artifact([]), + windowManifest: [], + }), + ).not.toThrow(); + expect(() => + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: [], + documentChunkCount: 1, + modelSelection: profile().reasoningModel, + parseArtifact: artifact([]), + windowManifest: [], + }), + ).toThrow("empty canonical input has receipt windows or chunks"); + + const parseArtifact = artifact([ + { + id: "minimal-manifest", + metadata: {}, + sectionPath: [], + text: "Minimal manifest.", + type: "paragraph", + }, + ]); + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow], undefined, "custom"), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + const receipt = compactWindowManifest(nodes); + const minimalCompletion = { fingerprint: llmSemanticCompletionFingerprint({}) }; + const validate = (overrides: { + completionCatalog?: readonly LlmSemanticCompletionCatalogEntry[]; + windowManifest?: readonly LlmSemanticWindowManifestEntry[]; + }) => + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: overrides.completionCatalog ?? [minimalCompletion], + documentChunkCount: 1, + modelSelection: profile().reasoningModel, + parseArtifact, + windowManifest: overrides.windowManifest ?? receipt.windowManifest, + }); + + expect(() => validate({})).not.toThrow(); + expect(() => + validate({ completionCatalog: [null as unknown as LlmSemanticCompletionCatalogEntry] }), + ).toThrow("must be an object"); + expect(() => + validate({ + windowManifest: [ + { + ...receipt.windowManifest[0], + windowId: "window-invalid", + } as LlmSemanticWindowManifestEntry, + ], + }), + ).toThrow("invalid or non-sequential windowId"); + expect(() => + validate({ + windowManifest: [ + { + ...receipt.windowManifest[0], + chunkRanges: [["invalid", "invalid"]], + } as LlmSemanticWindowManifestEntry, + ], + }), + ).toThrow("chunk 0 has invalid identity"); + expect(() => + assertValidLlmSemanticWindowManifestReplay({ + completionCatalog: [], + documentChunkCount: 1, + modelSelection: profile().reasoningModel, + parseArtifact, + windowManifest: [], + }), + ).toThrow("non-empty canonical input has an incomplete receipt manifest"); + }); + + it("fails closed when stored joint extraction metadata is tampered", async () => { + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => + new ScriptedProvider([ + ({ units }) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units.at(-1)?.id), + entities: [ + { + canonicalName: "Alpha", + confidence: 0.9, + id: "alpha", + text: "Alpha", + type: "term", + }, + { confidence: 0.8, id: "beta", text: "Beta", type: "term" }, + ], + relations: [ + { + confidence: 0.7, + objectEntityId: "beta", + subjectEntityId: "alpha", + type: "references", + }, + ], + }, + ], + }), + ]), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "joint-extraction", + metadata: {}, + sectionPath: [], + text: "Alpha references Beta.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }); + const node = nodes[0] as KnowledgeNode; + const semantic = node.metadata.semanticChunking as Record; + const entities = node.metadata.extractedEntities as readonly Record[]; + const relations = node.metadata.extractedRelations as readonly Record[]; + const entityMetadata = entities[0]?.metadata as Record; + const relationMetadata = relations[0]?.metadata as Record; + const withMetadata = (metadata: Record) => + ({ ...node, metadata: { ...node.metadata, ...metadata } }) as KnowledgeNode; + + expect(hasValidLlmSemanticJointExtraction(node)).toBe(true); + for (const corrupted of [ + withMetadata({ extractedEntities: "invalid" }), + withMetadata({ semanticChunking: { ...semantic, completed: false } }), + withMetadata({ semanticChunking: { ...semantic, windowId: "invalid" } }), + withMetadata({ semanticChunking: { ...semantic, completion: {} } }), + withMetadata({ extractedEntities: [null], extractedRelations: [] }), + withMetadata({ + extractedEntities: [{ ...entities[0], text: "Not present" }, entities[1]], + }), + withMetadata({ + extractedEntities: [ + { ...entities[0], metadata: { ...entityMetadata, canonicalName: " " } }, + entities[1], + ], + }), + withMetadata({ + extractedEntities: [ + entities[0], + { + ...entities[1], + metadata: { ...(entities[1]?.metadata as object), responseEntityId: "alpha" }, + }, + ], + }), + withMetadata({ extractedRelations: [null] }), + withMetadata({ + extractedRelations: [ + { ...relations[0], metadata: { ...relationMetadata, source: "tampered" } }, + ], + }), + withMetadata({ extractedRelations: [{ ...relations[0], subject: "Beta" }] }), + ]) { + expect(hasValidLlmSemanticJointExtraction(corrupted)).toBe(false); + } + }); + + it("bounds terminal completion metadata and accepts each exact boundary", async () => { + const parseArtifact = artifact([ + { + id: "terminal-bounds", + metadata: {}, + sectionPath: [], + text: "Bounded terminal metadata.", + type: "paragraph", + }, + ]); + const selectedProfile = KnowledgeSpaceRetrievalProfileSchema.parse({ + ...profile(), + reasoningModel: { + ...profile().reasoningModel, + model: "m".repeat(255), + }, + }); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => + new ScriptedProvider([echoWholeWindow], { + finishReason: "f".repeat(64), + metadata: { model: "m".repeat(255), provider: "p".repeat(255) }, + }), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: selectedProfile, + }), + ).resolves.toHaveLength(1); + + for (const terminal of [ + { finishReason: "f".repeat(65), metadata: { model: "reasoner-model" } }, + { metadata: { model: "m".repeat(256) } }, + { metadata: { model: "reasoner-model", provider: "p".repeat(256) } }, + ]) { + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow], terminal), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }), + ).rejects.toThrow("at most"); + } + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => + new ScriptedProvider([echoWholeWindow], undefined, "p".repeat(256)), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }), + ).rejects.toThrow("at most 255"); + }); + + it("rejects an excessive deterministic window count before provider construction", async () => { + const parseArtifact = artifact( + Array.from({ length: DEFAULT_MAX_SEMANTIC_WINDOWS + 1 }, (_, index) => ({ + id: `preflight-${index}`, + metadata: {}, + sectionPath: [`Section ${index}`], + text: "x", + type: "paragraph" as const, + })), + ); + expect(() => preflightLlmSemanticWindows({ parseArtifact })).toThrow( + `maxSemanticWindows=${DEFAULT_MAX_SEMANTIC_WINDOWS}`, + ); + let providerConstructions = 0; + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => { + providerConstructions += 1; + return new ScriptedProvider([echoWholeWindow]); + }, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }), + ).rejects.toThrow(`maxSemanticWindows=${DEFAULT_MAX_SEMANTIC_WINDOWS}`); + expect(providerConstructions).toBe(0); + }); + + it("validates construction and request bounds before invoking the model", async () => { + const factory = () => new ScriptedProvider([echoWholeWindow]); + expect(() => + createLlmSemanticChunker({ maxChunkChars: 0, reasoningProviderFactory: factory }), + ).toThrow("maxChunkChars must be at least 1"); + expect(() => + createLlmSemanticChunker({ + maxChunkChars: 20, + maxWindowChars: 10, + reasoningProviderFactory: factory, + }), + ).toThrow("maxWindowChars must be at least maxChunkChars"); + expect(() => + createLlmSemanticChunker({ promptVersion: " ", reasoningProviderFactory: factory }), + ).toThrow("promptVersion is required"); + expect(() => + createLlmSemanticChunker({ reasoningProviderFactory: factory, temperature: -1 }), + ).toThrow("temperature must be non-negative"); + for (const [name, options] of [ + ["maxEntitiesPerChunk", { maxEntitiesPerChunk: 0 }], + ["maxNodes", { maxNodes: 0 }], + ["maxOutputTokens", { maxOutputTokens: 0 }], + ["maxRelationsPerChunk", { maxRelationsPerChunk: 0 }], + ["maxResponseChars", { maxResponseChars: 0 }], + ] as const) { + expect(() => + createLlmSemanticChunker({ ...options, reasoningProviderFactory: factory }), + ).toThrow(`${name} must be at least 1`); + } + + let factoryCalls = 0; + const chunker = createLlmSemanticChunker({ + reasoningProviderFactory: () => { + factoryCalls += 1; + return new ScriptedProvider([echoWholeWindow]); + }, + }); + await expect( + chunker.chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "empty", + metadata: {}, + sectionPath: [], + text: " ", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).resolves.toEqual([]); + expect(factoryCalls).toBe(0); + + await expect( + chunker.chunk({ + config: { maxChunkChars: 20, maxWindowChars: 10 }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "non-empty", + metadata: {}, + sectionPath: [], + text: "Content.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow("maxWindowChars must be at least maxChunkChars"); + + const largeCapProvider = new ScriptedProvider([echoWholeWindow]); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => largeCapProvider, + }).chunk({ + config: { maxChunkChars: 8_192 }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "large-cap", + metadata: {}, + sectionPath: [], + text: "Content.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).resolves.toHaveLength(1); + expect(largeCapProvider.calls[0]?.messages[0]?.content).toContain("at most 8192 Unicode"); + + await expect( + chunker.chunk({ + config: { maxChunkChars: 8_192, maxWindowChars: 8_191 }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "explicit-small-window", + metadata: {}, + sectionPath: [], + text: "Content.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow("maxWindowChars must be at least maxChunkChars"); + + for (const overlapChars of [-1, 1.5, 20]) { + await expect( + chunker.chunk({ + config: { maxChunkChars: 20, overlapChars }, + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "invalid-overlap", + metadata: {}, + sectionPath: [], + text: "Content.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow(overlapChars === 20 ? "less than maxChunkChars" : "non-negative integer"); + } + }); + + it("fails closed on replay shape, empty-input, and exclusion corruption", async () => { + const parseArtifact = artifact([ + { + id: "replay-validation", + metadata: {}, + sectionPath: [], + text: "Replay validation.", + type: "paragraph", + }, + ]); + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow]), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + retrievalProfile: profile(), + }); + const node = nodes[0] as KnowledgeNode; + const marker = node.metadata.semanticChunking as Record; + const replay = (overrides: { + excludedNodeOrdinals?: readonly number[]; + nodes?: readonly KnowledgeNode[]; + parseArtifact?: ParseArtifact; + promptVersion?: string; + }) => + assertValidLlmSemanticGenerationReplay({ + ...(overrides.excludedNodeOrdinals !== undefined + ? { excludedNodeOrdinals: overrides.excludedNodeOrdinals } + : {}), + modelSelection: profile().reasoningModel, + nodes: overrides.nodes ?? nodes, + parseArtifact: overrides.parseArtifact ?? parseArtifact, + ...(overrides.promptVersion !== undefined + ? { promptVersion: overrides.promptVersion } + : {}), + }); + + expect(() => replay({ promptVersion: " " })).toThrow("promptVersion is required"); + expect(() => replay({ nodes: [], parseArtifact: artifact([]) })).not.toThrow(); + expect(() => replay({ parseArtifact: artifact([]) })).toThrow( + "stored nodes exist for an empty parse artifact", + ); + expect(() => replay({ nodes: [] })).toThrow("no stored nodes cover"); + expect(() => + replay({ + nodes: [ + { + ...node, + metadata: { + ...node.metadata, + semanticChunking: { ...marker, documentChunkCount: "one" }, + }, + }, + ], + }), + ).toThrow("documentChunkCount is missing or invalid"); + for (const ordinal of [-1, 1, Number.NaN]) { + expect(() => replay({ excludedNodeOrdinals: [ordinal] })).toThrow( + "outside documentChunkCount", + ); + } + }); + + it("rejects malformed terminal events and data emitted after completion", async () => { + const input = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "terminal-validation", + metadata: {}, + sectionPath: [], + text: "Terminal validation.", + type: "paragraph" as const, + }, + ]), + retrievalProfile: profile(), + }; + for (const terminal of [ + { finishReason: " ", metadata: {} }, + { finishReason: 1 as unknown as string, metadata: {} }, + { metadata: { model: 1 } }, + { metadata: { model: " " } }, + { metadata: { provider: 1 } }, + { metadata: { provider: " " } }, + ]) { + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow], terminal), + }).chunk(input), + ).rejects.toThrow("must be a non-empty string"); + } + + const afterDone: SemanticChunkingLlmProvider = { + async *stream(streamInput) { + const userMessage = streamInput.messages.find((message) => message.role === "user"); + const payload = JSON.parse(userMessage?.content ?? "{}") as PromptPayload; + yield { type: "done" }; + yield { delta: JSON.stringify(echoWholeWindow(payload)), type: "delta" }; + }, + }; + await expect( + createLlmSemanticChunker({ reasoningProviderFactory: () => afterDone }).chunk(input), + ).rejects.toThrow("emitted data after its terminal event"); + }); + + it("bounds distinct provider completion identities across a document", async () => { + let completionIndex = 0; + const provider: SemanticChunkingLlmProvider = { + async *stream(input) { + const userMessage = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(userMessage?.content ?? "{}") as PromptPayload; + yield { delta: JSON.stringify(echoWholeWindow(payload)), type: "delta" }; + yield { finishReason: `stop-${completionIndex++}`, type: "done" }; + }, + }; + await expect( + createLlmSemanticChunker({ reasoningProviderFactory: () => provider }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact( + Array.from({ length: 65 }, (_, index) => ({ + id: `completion-${index}`, + metadata: {}, + sectionPath: [`Section ${index}`], + text: `Completion ${index}.`, + type: "paragraph" as const, + })), + ), + retrievalProfile: profile(), + }), + ).rejects.toThrow("maxCompletionCatalogEntries=64"); + }); + + it("uses no page number when a semantic chunk spans multiple source pages", async () => { + const nodes = await createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow]), + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "page-one", + metadata: {}, + pageNumber: 1, + sectionPath: ["Shared"], + text: "First page.", + type: "paragraph", + }, + { + id: "page-two", + metadata: {}, + pageNumber: 2, + sectionPath: ["Shared"], + text: "Second page.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }); + + expect(nodes).toHaveLength(1); + expect(nodes[0]?.sourceLocation.pageNumber).toBeUndefined(); + }); + + it("bounds windows independently from chunks and enforces the document node cap", async () => { + const provider = new ScriptedProvider([echoWholeWindow]); + const windowBounded = createLlmSemanticChunker({ + maxChunkChars: 10, + maxWindowChars: 10, + reasoningProviderFactory: () => provider, + }); + const input = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "windowed", + metadata: {}, + sectionPath: ["A"], + text: "First. Second. Third.", + type: "paragraph" as const, + }, + ]), + retrievalProfile: profile(), + }; + const nodes = await windowBounded.chunk(input); + expect(provider.calls.length).toBeGreaterThan(1); + expect(nodes.map((node) => node.text).join("")).toBe("First. Second. Third."); + + const capped = createLlmSemanticChunker({ + maxChunkChars: 20, + maxNodes: 1, + reasoningProviderFactory: () => new ScriptedProvider([echoEachUnit]), + }); + await expect(capped.chunk(input)).rejects.toThrow("output exceeds maxNodes=1"); + }); + + it("lets the final core chunk commit across deterministic look-ahead without overlap", async () => { + const observedPrompts: PromptPayload[] = []; + const provider = new ScriptedProvider([ + (payload) => { + observedPrompts.push(payload); + return { + chunks: [ + chunkRange(payload.units[0]?.id, payload.units[0]?.id), + chunkRange(payload.units[1]?.id, payload.lookAheadUnits?.[0]?.id), + ], + }; + }, + ]); + const parseArtifact = artifact([ + { + id: "look-ahead-1", + metadata: {}, + sectionPath: ["A"], + text: "1234567.", + type: "paragraph", + }, + { + id: "look-ahead-2", + metadata: {}, + sectionPath: ["A"], + text: "abc.", + type: "paragraph", + }, + { + id: "look-ahead-3", + metadata: {}, + sectionPath: ["A"], + text: "def.", + type: "paragraph", + }, + ]); + const nodes = await createLlmSemanticChunker({ + maxChunkChars: 10, + maxWindowChars: 15, + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact, + publicationGenerationId: GENERATION_A, + retrievalProfile: profile(), + }); + + expect(provider.calls).toHaveLength(1); + expect(observedPrompts[0]?.units.map((unit) => unit.text)).toEqual(["1234567.", "abc."]); + expect(observedPrompts[0]?.lookAheadUnits?.map((unit) => unit.text)).toEqual(["def."]); + expect(nodes.map((node) => node.text)).toEqual(["1234567.", "abc.\ndef."]); + expect(nodes.map((node) => countGraphemes(node.text))).toEqual([8, 9]); + expect((nodes[0]?.endOffset ?? 0) + 1).toBe(nodes[1]?.startOffset); + expect(nodes[1]?.metadata.semanticChunking).toMatchObject({ + windowCommittedUnitRange: { + endUnitId: "u-000002-000000", + startUnitId: "u-000000-000000", + }, + windowCoreUnitRange: { + endUnitId: "u-000001-000000", + startUnitId: "u-000000-000000", + }, + windowLookAheadUnitRange: { + endUnitId: "u-000002-000000", + startUnitId: "u-000002-000000", + }, + }); + expect(() => + assertValidLlmSemanticGenerationReplay({ + config: { maxChunkChars: 10, maxWindowChars: 15 }, + modelSelection: profile().reasoningModel, + nodes, + parseArtifact, + publicationGenerationId: GENERATION_A, + }), + ).not.toThrow(); + }); + + it("rejects output chunks that start wholly inside look-ahead context", async () => { + const provider = new ScriptedProvider([ + ({ lookAheadUnits = [], units }) => ({ + chunks: [ + chunkRange(units[0]?.id, units[0]?.id), + chunkRange(units[1]?.id, units[1]?.id), + chunkRange(lookAheadUnits[0]?.id, lookAheadUnits[0]?.id), + ], + }), + ]); + await expect( + createLlmSemanticChunker({ + maxChunkChars: 10, + maxWindowChars: 15, + reasoningProviderFactory: () => provider, + }).chunk({ + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "look-ahead-only-1", + metadata: {}, + sectionPath: ["A"], + text: "1234567.", + type: "paragraph", + }, + { + id: "look-ahead-only-2", + metadata: {}, + sectionPath: ["A"], + text: "abc.", + type: "paragraph", + }, + { + id: "look-ahead-only-3", + metadata: {}, + sectionPath: ["A"], + text: "def.", + type: "paragraph", + }, + ]), + retrievalProfile: profile(), + }), + ).rejects.toThrow("final chunk must start in the core window"); + }); + + it("fails closed on response size, empty JSON, malformed JSON, and incomplete coverage", async () => { + const input = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "validation", + metadata: {}, + sectionPath: [], + text: "Alpha. Beta.", + type: "paragraph" as const, + }, + ]), + retrievalProfile: profile(), + }; + await expect( + createLlmSemanticChunker({ + maxResponseChars: 5, + reasoningProviderFactory: () => new ScriptedProvider([echoWholeWindow]), + }).chunk(input), + ).rejects.toThrow("response exceeds maxResponseChars=5"); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => rawProvider("", true), + }).chunk(input), + ).rejects.toThrow("empty response"); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => rawProvider("not-json", true), + }).chunk(input), + ).rejects.toThrow("non-JSON output"); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => rawProvider("prefix {broken} suffix", true), + }).chunk(input), + ).rejects.toThrow("invalid JSON"); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => new ScriptedProvider([() => ({ chunks: [] })]), + }).chunk(input), + ).rejects.toThrow("did not cover any input units"); + await expect( + createLlmSemanticChunker({ + reasoningProviderFactory: () => + new ScriptedProvider([ + ({ units }) => ({ chunks: [chunkRange(units[0]?.id, units[0]?.id)] }), + ]), + }).chunk(input), + ).rejects.toThrow("contiguously without gaps or overlap"); + }); + + it("accepts JSON wrapped in provider prose but strictly caps joint extraction arrays", async () => { + const input = { + knowledgeSpaceId: KNOWLEDGE_SPACE_ID, + parseArtifact: artifact([ + { + id: "validation", + metadata: {}, + sectionPath: [], + text: "Alpha references Beta.", + type: "paragraph" as const, + }, + ]), + retrievalProfile: profile(), + }; + const wrapped = wrappingProvider(({ units }) => ({ + chunks: [chunkRange(units[0]?.id, units[0]?.id)], + })); + await expect( + createLlmSemanticChunker({ reasoningProviderFactory: () => wrapped }).chunk(input), + ).resolves.toHaveLength(1); + + const tooManyEntities = new ScriptedProvider([ + ({ units }) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units[0]?.id), + entities: [ + { confidence: 1, id: "e-alpha", text: "Alpha", type: "term" }, + { confidence: 1, id: "e-beta", text: "Beta", type: "term" }, + ], + }, + ], + }), + ]); + await expect( + createLlmSemanticChunker({ + maxEntitiesPerChunk: 1, + reasoningProviderFactory: () => tooManyEntities, + }).chunk(input), + ).rejects.toThrow("exceeded maxEntitiesPerChunk=1"); + + const tooManyRelations = new ScriptedProvider([ + ({ units }) => ({ + chunks: [ + { + ...chunkRange(units[0]?.id, units[0]?.id), + entities: [ + { confidence: 1, id: "e-alpha", text: "Alpha", type: "term" }, + { confidence: 1, id: "e-beta", text: "Beta", type: "term" }, + ], + relations: [ + { + confidence: 1, + objectEntityId: "e-beta", + subjectEntityId: "e-alpha", + type: "references", + }, + { + confidence: 1, + objectEntityId: "e-alpha", + subjectEntityId: "e-beta", + type: "mentions", + }, + ], + }, + ], + }), + ]); + await expect( + createLlmSemanticChunker({ + maxRelationsPerChunk: 1, + reasoningProviderFactory: () => tooManyRelations, + }).chunk(input), + ).rejects.toThrow("exceeded maxRelationsPerChunk=1"); + }); +}); + +function artifact(elements: ParseArtifact["elements"]) { + return ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "structured", + createdAt: "2026-07-19T00:00:00.000Z", + documentAssetId: DOCUMENT_ASSET_ID, + elements, + id: PARSE_ARTIFACT_ID, + metadata: {}, + parser: "native-structured", + version: 1, + }); +} + +function profile() { + return KnowledgeSpaceRetrievalProfileSchema.parse({ + defaultMode: "deep", + reasoningModel: { + model: "reasoner-model", + pluginId: "reasoning-plugin", + provider: "reasoning-provider", + }, + rerank: { + enabled: true, + model: { + model: "rerank-model", + pluginId: "rerank-plugin", + provider: "rerank-provider", + }, + }, + revision: 4, + scoreThreshold: { enabled: false, stage: "mode-final" }, + topK: 10, + }); +} + +function compactWindowManifest(nodes: readonly KnowledgeNode[]): { + readonly completionCatalog: readonly LlmSemanticCompletionCatalogEntry[]; + readonly windowManifest: readonly LlmSemanticWindowManifestEntry[]; +} { + const completionCatalog: LlmSemanticCompletionCatalogEntry[] = []; + const completionIndexes = new Map(); + const windows = new Map(); + + for (const node of nodes) { + const semantic = node.metadata.semanticChunking as Record; + const completion = semantic.completion as { + actual: { finishReason?: string; model?: string; provider?: string }; + }; + const identityWithoutFingerprint = { + ...(completion.actual.model ? { actualModel: completion.actual.model } : {}), + ...(completion.actual.provider ? { actualProvider: completion.actual.provider } : {}), + ...(completion.actual.finishReason ? { finishReason: completion.actual.finishReason } : {}), + ...(typeof semantic.provider === "string" ? { transportProvider: semantic.provider } : {}), + }; + const fingerprint = llmSemanticCompletionFingerprint(identityWithoutFingerprint); + let completionIndex = completionIndexes.get(fingerprint); + if (completionIndex === undefined) { + completionIndex = completionCatalog.length; + completionIndexes.set(fingerprint, completionIndex); + completionCatalog.push({ ...identityWithoutFingerprint, fingerprint }); + } + const windowId = semantic.windowId as string; + const unitRange = semantic.unitRange as { endUnitId: string; startUnitId: string }; + const committedUnitRange = semantic.windowCommittedUnitRange as { + endUnitId: string; + startUnitId: string; + }; + const coreUnitRange = semantic.windowCoreUnitRange as { + endUnitId: string; + startUnitId: string; + }; + const lookAheadUnitRange = semantic.windowLookAheadUnitRange as + | { endUnitId: string; startUnitId: string } + | undefined; + const existing = windows.get(windowId); + if (existing) { + windows.set(windowId, { + ...existing, + chunkRanges: [ + ...existing.chunkRanges, + [unitRange.startUnitId, unitRange.endUnitId] as const, + ], + }); + continue; + } + windows.set(windowId, { + chunkRanges: [[unitRange.startUnitId, unitRange.endUnitId]], + committedUnitRange: [committedUnitRange.startUnitId, committedUnitRange.endUnitId], + completionIndex, + coreUnitRange: [coreUnitRange.startUnitId, coreUnitRange.endUnitId], + firstChunkIndex: node.metadata.chunkIndex as number, + inputFingerprint: semantic.inputFingerprint as string, + ...(lookAheadUnitRange + ? { + lookAheadUnitRange: [ + lookAheadUnitRange.startUnitId, + lookAheadUnitRange.endUnitId, + ] as const, + } + : {}), + responseFingerprint: semantic.inputFingerprint as string, + windowId, + }); + } + + return { completionCatalog, windowManifest: [...windows.values()] }; +} + +function echoEachUnit({ units }: PromptPayload) { + return { + chunks: units.map((unit) => chunkRange(unit.id, unit.id)), + }; +} + +function echoWholeWindow({ units }: PromptPayload) { + return { + chunks: [chunkRange(units[0]?.id, units.at(-1)?.id)], + }; +} + +function chunkRange(startUnitId: string | undefined, endUnitId: string | undefined) { + return { + endUnitId, + entities: [], + relations: [], + startUnitId, + }; +} + +function rawProvider(text: string, terminal: boolean): SemanticChunkingLlmProvider { + return { + async *stream() { + if (text) { + yield { delta: text, type: "delta" }; + } + if (terminal) { + yield { type: "done" }; + } + }, + }; +} + +function wrappingProvider(script: Script): SemanticChunkingLlmProvider { + return { + async *stream(input) { + const userMessage = input.messages.find((message) => message.role === "user"); + const payload = JSON.parse(userMessage?.content ?? "{}") as PromptPayload; + yield { delta: `Result follows:\n${JSON.stringify(script(payload))}\nEnd.`, type: "delta" }; + yield { type: "done" }; + }, + }; +} diff --git a/knowledge-fs/packages/api/src/llm-semantic-chunker.ts b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts new file mode 100644 index 00000000000..d77db4deb84 --- /dev/null +++ b/knowledge-fs/packages/api/src/llm-semantic-chunker.ts @@ -0,0 +1,2326 @@ +import { createHash } from "node:crypto"; + +import { + DateTimeSchema, + type KnowledgeNode, + KnowledgeNodeSchema, + type KnowledgeSpaceModelSelection, + KnowledgeSpaceModelSelectionSchema, + type KnowledgeSpaceRetrievalProfile, + type ParseArtifact, +} from "@knowledge/core"; +import { + countGraphemes as countUnicodeGraphemes, + graphemeSegments, +} from "unicode-segmenter/grapheme"; +import { z } from "zod"; + +import { deterministicChildId } from "./api-shared-utils"; +import { + type DocumentLayoutRecompositionStats, + recomposeDocumentLayoutForSemanticSegmentation, +} from "./document-layout-recomposer"; +import { + DOCUMENT_ELEMENT_SEPARATOR, + DOCUMENT_ELEMENT_TEXT_NORMALIZATION, + DOCUMENT_OFFSET_ENCODING, + materializeDocumentElementByteSpan, +} from "./document-offsets"; +import { cloneJsonObject, isPlainObject } from "./json-utils"; +import { + MAX_LLM_SEMANTIC_COMPLETION_IDENTITIES as MAX_COMPLETION_CATALOG_ENTRIES, + MAX_LLM_SEMANTIC_FINISH_REASON_CODE_POINTS as MAX_COMPLETION_FINISH_REASON_CHARS, + MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS as MAX_COMPLETION_MODEL_CHARS, + MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS as MAX_COMPLETION_PROVIDER_CHARS, + MAX_LLM_SEMANTIC_WINDOWS as MAX_RECEIPT_SEMANTIC_WINDOWS, + MAX_LLM_SEMANTIC_UNIT_ID_CODE_POINTS as MAX_SEMANTIC_UNIT_ID_CHARS, + MAX_LLM_SEMANTIC_WINDOW_ID_CODE_POINTS as MAX_SEMANTIC_WINDOW_ID_CHARS, + llmSemanticCompletionFingerprint, +} from "./semantic-generation-receipt"; + +export { llmSemanticCompletionFingerprint } from "./semantic-generation-receipt"; + +const DEFAULT_MAX_CHUNK_CHARS = 1_200; +const DEFAULT_MAX_WINDOW_CHARS = 4_800; +const DEFAULT_MAX_NODES = 20_000; +const DEFAULT_MAX_ENTITIES_PER_CHUNK = 100; +const DEFAULT_MAX_RELATIONS_PER_CHUNK = 100; +const DEFAULT_MAX_OUTPUT_TOKENS = 6_000; +const DEFAULT_MAX_RESPONSE_CHARS = 1_000_000; +const DEFAULT_PROMPT_VERSION = "semantic-chunking-v1"; +const SEMANTIC_CHUNKING_STRATEGY = "llm-semantic-v1"; +const SEMANTIC_CHUNKING_SCHEMA_VERSION = 1; +/** + * Pre-provider admission under the 4 MiB compact receipt budget. With 20,000 range tuples, 4,096 + * windows, 64 deduplicated completions, bounded IDs/terminal fields, and worst-case UTF-8 expansion, + * the receipt's deterministic portion remains below 4 MiB. The receipt layer additionally admits + * the exact request envelope and enforces exact serialized bytes. + */ +export const DEFAULT_MAX_SEMANTIC_WINDOWS = MAX_RECEIPT_SEMANTIC_WINDOWS; +export const MAX_LLM_SEMANTIC_WINDOWS = DEFAULT_MAX_SEMANTIC_WINDOWS; +const encoder = new TextEncoder(); + +export interface SemanticChunkingLlmMessage { + readonly content: string; + readonly role: "assistant" | "system" | "user"; +} + +export interface SemanticChunkingLlmStreamInput { + readonly maxOutputTokens?: number | undefined; + readonly messages: readonly SemanticChunkingLlmMessage[]; + readonly model: string; + readonly temperature?: number | undefined; + readonly tenantId?: string | undefined; +} + +export interface SemanticChunkingLlmStreamEvent { + readonly delta?: string | undefined; + readonly finishReason?: string | undefined; + readonly metadata?: unknown; + readonly type: "delta" | "done"; +} + +/** Structural subset of the generation provider, kept independent from `@knowledge/generation`. */ +export interface SemanticChunkingLlmProvider { + readonly kind?: string | undefined; + stream(input: SemanticChunkingLlmStreamInput): AsyncIterable; +} + +export interface SemanticChunkerInput { + readonly config?: + | { + readonly maxChunkChars?: number | undefined; + readonly maxNodes?: number | undefined; + readonly maxWindowChars?: number | undefined; + /** Legacy document setting retained as provenance; semantic output never overlaps. */ + readonly overlapChars?: number | undefined; + } + | undefined; + readonly knowledgeSpaceId: string; + readonly parseArtifact: ParseArtifact; + readonly permissionScope?: readonly string[] | undefined; + readonly publicationGenerationId?: string | undefined; + /** Frozen profile revision for the candidate publication being compiled. */ + readonly retrievalProfile: KnowledgeSpaceRetrievalProfile; + readonly tenantId?: string | undefined; +} + +export interface SemanticChunker { + readonly replayDefaults?: + | { + readonly maxChunkChars: number; + readonly maxWindowChars: number; + readonly promptVersion: string; + } + | undefined; + chunk(input: SemanticChunkerInput): Promise; +} + +export interface LlmSemanticChunkerOptions { + readonly maxChunkChars?: number | undefined; + readonly maxEntitiesPerChunk?: number | undefined; + readonly maxNodes?: number | undefined; + readonly maxOutputTokens?: number | undefined; + readonly maxRelationsPerChunk?: number | undefined; + readonly maxResponseChars?: number | undefined; + readonly maxWindowChars?: number | undefined; + readonly now?: (() => string) | undefined; + readonly promptVersion?: string | undefined; + readonly reasoningProviderFactory: ( + selection: KnowledgeSpaceModelSelection, + ) => SemanticChunkingLlmProvider; + readonly temperature?: number | undefined; +} + +export interface LlmSemanticGenerationReplayAssertionInput { + readonly config?: + | { + readonly maxChunkChars?: number | undefined; + readonly maxNodes?: number | undefined; + readonly maxWindowChars?: number | undefined; + readonly overlapChars?: number | undefined; + } + | undefined; + readonly excludedNodeOrdinals?: readonly number[] | ReadonlySet | undefined; + readonly language?: string | undefined; + readonly modelSelection: KnowledgeSpaceModelSelection; + readonly nodes: readonly KnowledgeNode[]; + readonly parseArtifact: ParseArtifact; + readonly permissionScope?: readonly string[] | undefined; + readonly promptVersion?: string | undefined; + readonly publicationGenerationId?: string | undefined; +} + +export interface LlmSemanticCompletionCatalogEntry { + readonly actualModel?: string | undefined; + readonly actualProvider?: string | undefined; + readonly fingerprint: string; + readonly finishReason?: string | undefined; + readonly transportProvider?: string | undefined; +} + +export type LlmSemanticUnitRangeTuple = readonly [startUnitId: string, endUnitId: string]; + +export interface LlmSemanticWindowManifestEntry { + readonly chunkRanges: readonly LlmSemanticUnitRangeTuple[]; + readonly committedUnitRange: LlmSemanticUnitRangeTuple; + readonly completionIndex: number; + readonly coreUnitRange: LlmSemanticUnitRangeTuple; + readonly firstChunkIndex: number; + readonly inputFingerprint: string; + readonly lookAheadUnitRange?: LlmSemanticUnitRangeTuple | undefined; + /** Generation-time commitment to the full semantic payload; not canonical from parser input. */ + readonly responseFingerprint: string; + readonly windowId: string; +} + +export interface LlmSemanticWindowManifestReplayAssertionInput { + readonly completionCatalog: readonly LlmSemanticCompletionCatalogEntry[]; + readonly config?: + | { + readonly maxChunkChars?: number | undefined; + readonly maxNodes?: number | undefined; + readonly maxWindowChars?: number | undefined; + readonly overlapChars?: number | undefined; + } + | undefined; + readonly documentChunkCount: number; + readonly modelSelection: KnowledgeSpaceModelSelection; + readonly parseArtifact: ParseArtifact; + readonly promptVersion?: string | undefined; + readonly windowManifest: readonly LlmSemanticWindowManifestEntry[]; +} + +export interface LlmSemanticWindowPreflightInput { + readonly config?: SemanticChunkerInput["config"] | undefined; + readonly parseArtifact: ParseArtifact; +} + +export interface LlmSemanticWindowPreflightResult { + readonly maximumWindowCount: number; + readonly unitCount: number; +} + +interface EffectiveChunkConfig { + readonly maxChunkChars: number; + readonly maxNodes: number; + readonly maxWindowChars: number; + readonly requestedOverlapChars: number; +} + +interface MaterializedElement { + readonly elementId: string; + readonly elementIndex: number; + readonly elementMetadata: Record; + readonly elementType: ParseArtifact["elements"][number]["type"]; + readonly endCodeUnit: number; + readonly endOffset: number; + readonly pageNumber?: number | undefined; + readonly sectionPath: readonly string[]; + readonly startCodeUnit: number; + readonly startOffset: number; + readonly text: string; +} + +interface AtomicUnit { + readonly elementId: string; + readonly elementMetadata: Record; + readonly elementType: ParseArtifact["elements"][number]["type"]; + readonly endCodeUnit: number; + readonly endOffset: number; + readonly graphemeLength: number; + readonly id: string; + readonly isolationKey?: string | undefined; + readonly pageNumber?: number | undefined; + readonly sectionPath: readonly string[]; + readonly startCodeUnit: number; + readonly startOffset: number; + readonly text: string; +} + +interface SemanticWindow { + readonly id: string; + readonly inputFingerprint: string; + readonly lookAheadUnits: readonly AtomicUnit[]; + readonly sectionPath: readonly string[]; + /** Deterministic core units which must be covered by this request. */ + readonly units: readonly AtomicUnit[]; +} + +interface MaterializedChunk { + readonly completion: ProviderCompletionProvenance; + readonly endUnitId: string; + readonly entities: readonly LlmSemanticEntity[]; + readonly kind: KnowledgeNode["kind"]; + readonly relations: readonly MaterializedSemanticRelation[]; + readonly sectionPath: readonly string[]; + readonly sectionSummary?: string | undefined; + readonly startUnitId: string; + readonly units: readonly AtomicUnit[]; + readonly window: SemanticWindow; + readonly windowCommitEndUnitId: string; +} + +type WindowMaterializedChunk = Omit; +type UncommittedWindowChunk = Omit; + +interface MaterializedSemanticRelation { + readonly confidence: number; + readonly object: string; + readonly objectEntityId: string; + readonly subject: string; + readonly subjectEntityId: string; + readonly type: z.infer; +} + +interface ProviderCompletionProvenance { + readonly actualModel?: string | undefined; + readonly actualProvider?: string | undefined; + readonly finishReason?: string | undefined; +} + +interface CollectedProviderCompletion extends ProviderCompletionProvenance { + readonly text: string; +} + +export function preflightLlmSemanticWindows({ + config, + parseArtifact, +}: LlmSemanticWindowPreflightInput): LlmSemanticWindowPreflightResult { + const effectiveConfig = resolveConfig(config, { + maxChunkChars: DEFAULT_MAX_CHUNK_CHARS, + maxNodes: DEFAULT_MAX_NODES, + maxWindowChars: DEFAULT_MAX_WINDOW_CHARS, + requestedOverlapChars: 0, + }); + const { canonicalText, elements } = materializeElements(parseArtifact); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + return preflightMaterializedSemanticWindows({ + canonicalText, + effectiveConfig, + units, + }); +} + +/** + * Creates a profile-aware semantic chunker. `maxChunkChars` is a hard Unicode-grapheme cap, not a + * target fill size: the LLM may choose any smaller complete semantic range. + */ +export function createLlmSemanticChunker({ + maxChunkChars = DEFAULT_MAX_CHUNK_CHARS, + maxEntitiesPerChunk = DEFAULT_MAX_ENTITIES_PER_CHUNK, + maxNodes = DEFAULT_MAX_NODES, + maxOutputTokens = DEFAULT_MAX_OUTPUT_TOKENS, + maxRelationsPerChunk = DEFAULT_MAX_RELATIONS_PER_CHUNK, + maxResponseChars = DEFAULT_MAX_RESPONSE_CHARS, + maxWindowChars = DEFAULT_MAX_WINDOW_CHARS, + now = () => new Date().toISOString(), + promptVersion = DEFAULT_PROMPT_VERSION, + reasoningProviderFactory, + temperature = 0, +}: LlmSemanticChunkerOptions): SemanticChunker { + validatePositiveInteger("maxChunkChars", maxChunkChars); + validatePositiveInteger("maxEntitiesPerChunk", maxEntitiesPerChunk); + validatePositiveInteger("maxNodes", maxNodes); + validatePositiveInteger("maxOutputTokens", maxOutputTokens); + validatePositiveInteger("maxRelationsPerChunk", maxRelationsPerChunk); + validatePositiveInteger("maxResponseChars", maxResponseChars); + validatePositiveInteger("maxWindowChars", maxWindowChars); + if (maxWindowChars < maxChunkChars) { + throw new Error("LLM semantic chunking maxWindowChars must be at least maxChunkChars"); + } + if (!promptVersion.trim()) { + throw new Error("LLM semantic chunking promptVersion is required"); + } + if (!Number.isFinite(temperature) || temperature < 0) { + throw new Error("LLM semantic chunking temperature must be non-negative"); + } + + return { + replayDefaults: { maxChunkChars, maxWindowChars, promptVersion }, + chunk: async (input) => { + const effectiveConfig = resolveConfig(input.config, { + maxChunkChars, + maxNodes, + maxWindowChars, + requestedOverlapChars: 0, + }); + const { canonicalText, elements, layoutRecomposition } = materializeElements( + input.parseArtifact, + ); + if (elements.length === 0) { + return []; + } + + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + preflightMaterializedSemanticWindows({ canonicalText, effectiveConfig, units }); + const reasoningSelection = input.retrievalProfile.reasoningModel; + const provider = reasoningProviderFactory(reasoningSelection); + assertBoundedOptionalCompletionField( + provider.kind, + "transportProvider", + MAX_COMPLETION_PROVIDER_CHARS, + "terminal", + ); + const chunks: MaterializedChunk[] = []; + const completionFingerprints = new Set(); + const globalUnitIndex = new Map(units.map((unit, index) => [unit.id, index])); + let nextUnitIndex = 0; + let windowIndex = 0; + + while (nextUnitIndex < units.length) { + const window = materializeSemanticWindow({ + canonicalText, + maxChunkChars: effectiveConfig.maxChunkChars, + maxWindowChars: effectiveConfig.maxWindowChars, + startUnitIndex: nextUnitIndex, + units, + windowIndex, + }); + const completion = await collectProviderCompletion({ + maxOutputTokens, + maxResponseChars, + messages: semanticChunkingMessages({ + maxChunkChars: effectiveConfig.maxChunkChars, + maxEntitiesPerChunk, + maxRelationsPerChunk, + window, + }), + model: reasoningSelection.model, + provider, + temperature, + tenantId: input.tenantId, + }); + const completionFingerprint = llmSemanticCompletionFingerprint({ + ...(completion.actualModel ? { actualModel: completion.actualModel } : {}), + ...(completion.actualProvider ? { actualProvider: completion.actualProvider } : {}), + ...(completion.finishReason ? { finishReason: completion.finishReason } : {}), + ...(provider.kind ? { transportProvider: provider.kind } : {}), + }); + completionFingerprints.add(completionFingerprint); + if (completionFingerprints.size > MAX_COMPLETION_CATALOG_ENTRIES) { + throw new Error( + `LLM semantic chunking completion identities exceed maxCompletionCatalogEntries=${MAX_COMPLETION_CATALOG_ENTRIES}`, + ); + } + const output = parseSemanticChunkingOutput(completion.text); + const windowChunks = validateAndMaterializeWindowOutput({ + maxChunkChars: effectiveConfig.maxChunkChars, + maxEntitiesPerChunk, + maxRelationsPerChunk, + output, + window, + }).map((chunk) => ({ + ...chunk, + completion: { + ...(completion.actualModel ? { actualModel: completion.actualModel } : {}), + ...(completion.actualProvider ? { actualProvider: completion.actualProvider } : {}), + ...(completion.finishReason ? { finishReason: completion.finishReason } : {}), + }, + })); + chunks.push(...windowChunks); + if (chunks.length > effectiveConfig.maxNodes) { + throw new Error( + `LLM semantic chunking output exceeds maxNodes=${effectiveConfig.maxNodes}`, + ); + } + const committedEndUnitId = windowChunks.at(-1)?.windowCommitEndUnitId; + const committedEndIndex = committedEndUnitId + ? globalUnitIndex.get(committedEndUnitId) + : undefined; + if (committedEndIndex === undefined || committedEndIndex < nextUnitIndex) { + throw new Error("LLM semantic chunking response did not advance the document cursor"); + } + nextUnitIndex = committedEndIndex + 1; + windowIndex += 1; + } + + const extractedAt = now(); + return chunks.map((chunk, chunkIndex) => + materializeKnowledgeNode({ + canonicalText, + chunk, + chunkIndex, + documentChunkCount: chunks.length, + extractedAt, + input, + layoutRecomposition, + maxChunkChars: effectiveConfig.maxChunkChars, + modelSelection: reasoningSelection, + promptVersion, + providerKind: provider.kind, + requestedOverlapChars: effectiveConfig.requestedOverlapChars, + }), + ); + }, + }; +} + +/** + * Fail-closed validation for a stored LLM semantic generation. This recomputes the canonical + * parser text, atomic units, windows, and window fingerprints rather than trusting replayed JSON. + */ +export function assertValidLlmSemanticGenerationReplay({ + config, + excludedNodeOrdinals = [], + language, + modelSelection: requestedModelSelection, + nodes, + parseArtifact, + permissionScope = [], + promptVersion = DEFAULT_PROMPT_VERSION, + publicationGenerationId, +}: LlmSemanticGenerationReplayAssertionInput): void { + const modelSelection = KnowledgeSpaceModelSelectionSchema.parse(requestedModelSelection); + const effectiveConfig = resolveConfig(config, { + maxChunkChars: DEFAULT_MAX_CHUNK_CHARS, + maxNodes: DEFAULT_MAX_NODES, + maxWindowChars: DEFAULT_MAX_WINDOW_CHARS, + requestedOverlapChars: 0, + }); + if (!promptVersion.trim()) { + throw new Error("LLM semantic replay promptVersion is required"); + } + const { canonicalText, elements, layoutRecomposition } = materializeElements(parseArtifact); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + if (units.length === 0) { + assertSemanticReplay(nodes.length === 0, "stored nodes exist for an empty parse artifact"); + return; + } + assertSemanticReplay(nodes.length > 0, "no stored nodes cover the non-empty parse artifact"); + + const firstMarker = nodes[0]?.metadata.semanticChunking; + const documentChunkCount = + isPlainObject(firstMarker) && Number.isSafeInteger(firstMarker.documentChunkCount) + ? (firstMarker.documentChunkCount as number) + : -1; + assertSemanticReplay(documentChunkCount >= 1, "documentChunkCount is missing or invalid"); + assertSemanticReplay( + documentChunkCount <= effectiveConfig.maxNodes, + `documentChunkCount exceeds maxNodes=${effectiveConfig.maxNodes}`, + ); + const excluded = + excludedNodeOrdinals instanceof Set + ? new Set(excludedNodeOrdinals) + : new Set(excludedNodeOrdinals); + for (const ordinal of excluded) { + assertSemanticReplay( + Number.isSafeInteger(ordinal) && ordinal >= 0 && ordinal < documentChunkCount, + `excluded chunk ordinal ${String(ordinal)} is outside documentChunkCount`, + ); + } + const expectedIndexes = Array.from({ length: documentChunkCount }, (_, index) => index).filter( + (index) => !excluded.has(index), + ); + assertSemanticReplay( + nodes.length === expectedIndexes.length, + "stored node count does not match the non-excluded document chunks", + ); + + const globalUnitIndex = new Map(units.map((unit, index) => [unit.id, index])); + const validatedRanges: Array<{ + readonly chunkIndex: number; + readonly end: number; + readonly start: number; + readonly windowId: string; + }> = []; + const completionByWindow = new Map(); + const replayWindows = new Map< + string, + { + readonly commitEnd: number; + readonly coreEnd: number; + readonly coreStart: number; + readonly firstChunkIndex: number; + readonly lastChunkIndex: number; + readonly ordinal: number; + } + >(); + + for (const [nodeOrdinal, node] of nodes.entries()) { + const chunkIndex = expectedIndexes[nodeOrdinal] as number; + const semantic = node.metadata.semanticChunking; + assertSemanticReplay( + node.artifactHash === parseArtifact.artifactHash && + node.documentAssetId === parseArtifact.documentAssetId && + node.parseArtifactId === parseArtifact.id && + (publicationGenerationId === undefined || + node.publicationGenerationId === publicationGenerationId), + `chunk ${chunkIndex} does not belong to the requested immutable parse generation`, + ); + assertSemanticReplay( + isPlainObject(semantic) && + semantic.completed === true && + semantic.strategy === SEMANTIC_CHUNKING_STRATEGY && + semantic.schemaVersion === SEMANTIC_CHUNKING_SCHEMA_VERSION, + `chunk ${chunkIndex} has an invalid semantic marker`, + ); + // The assertion above establishes the runtime shape for subsequent guarded property reads. + const marker = semantic as Record; + assertSemanticReplay( + node.metadata.chunkIndex === chunkIndex && + marker.documentChunkCount === documentChunkCount && + marker.maxChunkChars === effectiveConfig.maxChunkChars && + marker.requestedOverlapChars === effectiveConfig.requestedOverlapChars && + marker.overlapApplied === false && + marker.overlapPolicy === "non-overlapping-semantic-output" && + marker.model === modelSelection.model && + marker.promptVersion === promptVersion && + isExactModelSelection(marker.modelSelection, modelSelection) && + isMatchingLayoutRecomposition(marker.layoutRecomposition, layoutRecomposition) && + isValidCompletionIdentity( + marker.completion, + modelSelection, + nonEmptyString(marker.provider), + ), + `chunk ${chunkIndex} has incompatible semantic provenance`, + ); + + const windowId = nonEmptyString(marker.windowId); + const windowOrdinal = windowId ? semanticWindowOrdinal(windowId) : undefined; + assertSemanticReplay( + windowId !== undefined && windowOrdinal !== undefined, + `chunk ${chunkIndex} references an invalid window`, + ); + const coreRange = semanticMarkerUnitRange(marker.windowCoreUnitRange); + const committedRange = semanticMarkerUnitRange(marker.windowCommittedUnitRange); + const coreStart = coreRange ? globalUnitIndex.get(coreRange.startUnitId) : undefined; + const coreEnd = coreRange ? globalUnitIndex.get(coreRange.endUnitId) : undefined; + const commitStart = committedRange + ? globalUnitIndex.get(committedRange.startUnitId) + : undefined; + const commitEnd = committedRange ? globalUnitIndex.get(committedRange.endUnitId) : undefined; + assertSemanticReplay( + coreStart !== undefined && + coreEnd !== undefined && + commitStart === coreStart && + commitEnd !== undefined && + commitEnd >= coreEnd, + `chunk ${chunkIndex} has invalid core or committed window ranges`, + ); + const resolvedWindow = materializeSemanticWindow({ + canonicalText, + maxChunkChars: effectiveConfig.maxChunkChars, + maxWindowChars: effectiveConfig.maxWindowChars, + startUnitIndex: coreStart as number, + units, + windowIndex: windowOrdinal as number, + }); + const expectedCore = semanticWindowCoreRange(resolvedWindow); + const expectedLookAhead = semanticWindowLookAheadRange(resolvedWindow); + const markerLookAhead = semanticMarkerUnitRange(marker.windowLookAheadUnitRange); + assertSemanticReplay( + resolvedWindow.id === windowId && + marker.inputFingerprint === resolvedWindow.inputFingerprint && + sameSemanticUnitRange(coreRange, expectedCore) && + sameSemanticUnitRange(markerLookAhead, expectedLookAhead) && + (commitEnd as number) <= + (globalUnitIndex.get( + (resolvedWindow.lookAheadUnits.at(-1) ?? resolvedWindow.units.at(-1))?.id ?? "", + ) ?? -1), + `chunk ${chunkIndex} window fingerprint or layout does not match canonical parser input`, + ); + const unitRange = marker.unitRange; + const startUnitId = isPlainObject(unitRange) + ? nonEmptyString(unitRange.startUnitId) + : undefined; + const endUnitId = isPlainObject(unitRange) ? nonEmptyString(unitRange.endUnitId) : undefined; + const start = startUnitId ? globalUnitIndex.get(startUnitId) : undefined; + const end = endUnitId ? globalUnitIndex.get(endUnitId) : undefined; + assertSemanticReplay( + start !== undefined && + end !== undefined && + start <= end && + start >= (coreStart as number) && + end <= (commitEnd as number) && + start <= (coreEnd as number) && + (end <= (coreEnd as number) || end === commitEnd), + `chunk ${chunkIndex} has an invalid unit range`, + ); + const first = units[start as number] as AtomicUnit; + const last = units[end as number] as AtomicUnit; + const rangeUnits = units.slice(start as number, (end as number) + 1); + const expectedText = canonicalText.slice(first.startCodeUnit, last.endCodeUnit); + const expectedPageNumber = commonPageNumber(rangeUnits); + const expectedKind = commonSpecialKind(rangeUnits) ?? "chunk"; + const expectedNodeId = deterministicChildId( + publicationGenerationId ?? parseArtifact.id, + `${SEMANTIC_CHUNKING_STRATEGY}:${parseArtifact.id}:${parseArtifact.artifactHash}:${first.startOffset}:${last.endOffset}`, + ); + assertSemanticReplay( + node.id === expectedNodeId && + node.text === expectedText && + countUnicodeGraphemes(node.text) <= effectiveConfig.maxChunkChars && + node.startOffset === first.startOffset && + node.endOffset === last.endOffset && + node.sourceLocation.startOffset === first.startOffset && + node.sourceLocation.endOffset === last.endOffset && + node.sourceLocation.pageNumber === expectedPageNumber && + hasValidSemanticSectionReplay( + marker.section, + first.sectionPath, + node.sourceLocation.sectionPath, + ) && + node.kind === expectedKind && + sameStrings(node.permissionScope, permissionScope) && + (language === undefined + ? node.metadata.language === undefined + : node.metadata.language === language), + `chunk ${chunkIndex} identity, text, UTF-8 offsets, location, ACL, language, or kind does not match canonical input`, + ); + assertSemanticReplay( + node.metadata.offsetEncoding === DOCUMENT_OFFSET_ENCODING && + node.metadata.textNormalization === DOCUMENT_ELEMENT_TEXT_NORMALIZATION && + node.metadata.elementSeparator === DOCUMENT_ELEMENT_SEPARATOR && + sameStrings( + arrayOfStrings(node.metadata.elementIds), + uniqueStrings(rangeUnits.map((unit) => unit.elementId)), + ) && + sameStrings( + arrayOfStrings(node.metadata.elementTypes), + uniqueStrings(rangeUnits.map((unit) => unit.elementType)), + ) && + hasValidLlmSemanticJointExtraction(node), + `chunk ${chunkIndex} canonical or joint-extraction metadata is invalid`, + ); + const completionFingerprint = JSON.stringify(marker.completion); + const existingCompletion = completionByWindow.get(resolvedWindow.id); + assertSemanticReplay( + existingCompletion === undefined || existingCompletion === completionFingerprint, + `chunk ${chunkIndex} completion identity differs inside the same window`, + ); + completionByWindow.set(resolvedWindow.id, completionFingerprint); + const existingReplayWindow = replayWindows.get(resolvedWindow.id); + if (existingReplayWindow) { + assertSemanticReplay( + existingReplayWindow.coreStart === coreStart && + existingReplayWindow.coreEnd === coreEnd && + existingReplayWindow.commitEnd === commitEnd, + `chunk ${chunkIndex} window ranges differ inside the same window`, + ); + replayWindows.set(resolvedWindow.id, { + ...existingReplayWindow, + lastChunkIndex: chunkIndex, + }); + } else { + replayWindows.set(resolvedWindow.id, { + commitEnd: commitEnd as number, + coreEnd: coreEnd as number, + coreStart: coreStart as number, + firstChunkIndex: chunkIndex, + lastChunkIndex: chunkIndex, + ordinal: windowOrdinal as number, + }); + } + validatedRanges.push({ + chunkIndex, + end: end as number, + start: start as number, + windowId: resolvedWindow.id, + }); + } + + const orderedWindows = [...replayWindows.values()].sort( + (left, right) => left.firstChunkIndex - right.firstChunkIndex, + ); + for (const [windowIndex, replayWindow] of orderedWindows.entries()) { + const previous = orderedWindows[windowIndex - 1]; + if (replayWindow.firstChunkIndex === 0) { + assertSemanticReplay( + replayWindow.ordinal === 0 && replayWindow.coreStart === 0, + "the first semantic window does not start at unit 0", + ); + } + if (previous && replayWindow.firstChunkIndex === previous.lastChunkIndex + 1) { + assertSemanticReplay( + replayWindow.ordinal === previous.ordinal + 1 && + replayWindow.coreStart === previous.commitEnd + 1, + "adjacent semantic windows do not advance from the preceding committed boundary", + ); + } else if (previous) { + assertSemanticReplay( + replayWindow.ordinal > previous.ordinal && + replayWindow.coreStart > previous.commitEnd && + replayWindow.ordinal - previous.ordinal <= + replayWindow.firstChunkIndex - previous.lastChunkIndex, + "semantic windows separated by excluded chunks have an invalid layout", + ); + } + const visibleWindowRanges = validatedRanges.filter( + (range) => range.windowId === `window-${replayWindow.ordinal.toString().padStart(6, "0")}`, + ); + const firstVisible = visibleWindowRanges[0]; + const lastVisible = visibleWindowRanges.at(-1); + if (firstVisible && !excluded.has(firstVisible.chunkIndex - 1)) { + assertSemanticReplay( + firstVisible.start === replayWindow.coreStart, + `window ${replayWindow.ordinal} does not start at its deterministic core boundary`, + ); + } + if (lastVisible && !excluded.has(lastVisible.chunkIndex + 1)) { + assertSemanticReplay( + lastVisible.end === replayWindow.commitEnd, + `window ${replayWindow.ordinal} does not end at its committed boundary`, + ); + } + } + + for (const [index, range] of validatedRanges.entries()) { + const previous = validatedRanges[index - 1]; + if (range.chunkIndex === 0) { + assertSemanticReplay(range.start === 0, "the first document chunk does not start at unit 0"); + } + if (range.chunkIndex === documentChunkCount - 1) { + assertSemanticReplay( + range.end === units.length - 1, + "the last document chunk does not end at the final unit", + ); + } + if (previous && range.chunkIndex === previous.chunkIndex + 1) { + assertSemanticReplay( + range.start === previous.end + 1, + `chunks ${previous.chunkIndex} and ${range.chunkIndex} have a gap or overlap`, + ); + } else if (previous) { + assertSemanticReplay( + range.start > previous.end, + `non-excluded chunk ${range.chunkIndex} overlaps a preceding range`, + ); + } + } +} + +/** + * Replays a compact, complete semantic-window receipt against canonical parser input. Unlike node + * replay, this remains complete when every chunk, or every chunk from an intermediate window, was + * editorially excluded from the persisted node set. + */ +export function assertValidLlmSemanticWindowManifestReplay({ + completionCatalog, + config, + documentChunkCount, + modelSelection: requestedModelSelection, + parseArtifact, + promptVersion = DEFAULT_PROMPT_VERSION, + windowManifest, +}: LlmSemanticWindowManifestReplayAssertionInput): void { + const modelSelection = KnowledgeSpaceModelSelectionSchema.parse(requestedModelSelection); + const effectiveConfig = resolveConfig(config, { + maxChunkChars: DEFAULT_MAX_CHUNK_CHARS, + maxNodes: DEFAULT_MAX_NODES, + maxWindowChars: DEFAULT_MAX_WINDOW_CHARS, + requestedOverlapChars: 0, + }); + assertSemanticManifestReplay(Boolean(promptVersion.trim()), "promptVersion is required"); + assertSemanticManifestReplay( + Number.isSafeInteger(documentChunkCount) && + documentChunkCount >= 0 && + documentChunkCount <= effectiveConfig.maxNodes, + `documentChunkCount exceeds maxNodes=${effectiveConfig.maxNodes} or is invalid`, + ); + assertSemanticManifestReplay(Array.isArray(windowManifest), "windowManifest must be an array"); + assertSemanticManifestReplay( + windowManifest.length <= effectiveConfig.maxNodes && + windowManifest.length <= DEFAULT_MAX_SEMANTIC_WINDOWS && + windowManifest.length <= documentChunkCount, + `windowManifest exceeds maxNodes=${effectiveConfig.maxNodes}, maxSemanticWindows=${DEFAULT_MAX_SEMANTIC_WINDOWS}, or documentChunkCount`, + ); + assertSemanticManifestReplay( + Array.isArray(completionCatalog) && + completionCatalog.length <= windowManifest.length && + completionCatalog.length <= MAX_COMPLETION_CATALOG_ENTRIES, + "completionCatalog is not bounded by the window manifest", + ); + + const completionFingerprints = new Set(); + for (const [completionIndex, completion] of completionCatalog.entries()) { + assertSemanticManifestReplay( + plainObjectValue(completion), + `completionCatalog entry ${completionIndex} must be an object`, + ); + assertBoundedOptionalCompletionField( + completion.actualModel, + "actualModel", + MAX_COMPLETION_MODEL_CHARS, + "manifest", + ); + assertBoundedOptionalCompletionField( + completion.actualProvider, + "actualProvider", + MAX_COMPLETION_PROVIDER_CHARS, + "manifest", + ); + assertBoundedOptionalCompletionField( + completion.finishReason, + "finishReason", + MAX_COMPLETION_FINISH_REASON_CHARS, + "manifest", + ); + assertBoundedOptionalCompletionField( + completion.transportProvider, + "transportProvider", + MAX_COMPLETION_PROVIDER_CHARS, + "manifest", + ); + const expectedFingerprint = llmSemanticCompletionFingerprint(completion); + assertSemanticManifestReplay( + completion.fingerprint === expectedFingerprint && + !completionFingerprints.has(completion.fingerprint), + `completionCatalog entry ${completionIndex} has an invalid or duplicate identity`, + ); + completionFingerprints.add(completion.fingerprint); + assertSemanticManifestReplay( + isValidCompletionIdentity( + { + actual: { + ...(completion.actualModel ? { model: completion.actualModel } : {}), + ...(completion.actualProvider ? { provider: completion.actualProvider } : {}), + ...(completion.finishReason ? { finishReason: completion.finishReason } : {}), + }, + requested: modelSelection, + }, + modelSelection, + completion.transportProvider, + ), + `completionCatalog entry ${completionIndex} is incompatible with the frozen model`, + ); + } + + const { canonicalText, elements } = materializeElements(parseArtifact); + const units = materializeAtomicUnits(elements, effectiveConfig.maxChunkChars); + preflightMaterializedSemanticWindows({ canonicalText, effectiveConfig, units }); + if (units.length === 0) { + assertSemanticManifestReplay( + documentChunkCount === 0 && windowManifest.length === 0 && completionCatalog.length === 0, + "empty canonical input has receipt windows or chunks", + ); + return; + } + assertSemanticManifestReplay( + documentChunkCount > 0 && windowManifest.length > 0 && completionCatalog.length > 0, + "non-empty canonical input has an incomplete receipt manifest", + ); + + const globalUnitIndex = new Map(units.map((unit, index) => [unit.id, index])); + const usedCompletions = new Set(); + let nextUnitIndex = 0; + let nextChunkIndex = 0; + + for (const [windowOrdinal, manifestWindow] of windowManifest.entries()) { + assertSemanticManifestReplay( + plainObjectValue(manifestWindow), + `window ${windowOrdinal} must be an object`, + ); + assertSemanticManifestReplay( + isSemanticWindowId(manifestWindow.windowId) && + manifestWindow.windowId === semanticWindowId(windowOrdinal), + `window ${windowOrdinal} has an invalid or non-sequential windowId`, + ); + assertSemanticManifestReplay( + /^sha256:[a-f0-9]{64}$/u.test(manifestWindow.inputFingerprint), + `window ${windowOrdinal} has an invalid inputFingerprint`, + ); + assertSemanticManifestReplay( + Number.isSafeInteger(manifestWindow.completionIndex) && + manifestWindow.completionIndex >= 0 && + manifestWindow.completionIndex < completionCatalog.length, + `window ${windowOrdinal} has an invalid completionIndex`, + ); + usedCompletions.add(manifestWindow.completionIndex); + assertSemanticManifestReplay( + manifestWindow.firstChunkIndex === nextChunkIndex && + // The non-deterministic LLM payload may be absent after exclusions. Its generation-time + // commitment is bound by the outer receipt; canonical replay can only validate its shape. + /^sha256:[a-f0-9]{64}$/u.test(manifestWindow.responseFingerprint) && + Array.isArray(manifestWindow.chunkRanges) && + manifestWindow.chunkRanges.length > 0 && + manifestWindow.chunkRanges.length <= documentChunkCount - nextChunkIndex, + `window ${windowOrdinal} has an invalid or unbounded chunk list`, + ); + + const expectedWindow = materializeSemanticWindow({ + canonicalText, + maxChunkChars: effectiveConfig.maxChunkChars, + maxWindowChars: effectiveConfig.maxWindowChars, + startUnitIndex: nextUnitIndex, + units, + windowIndex: windowOrdinal, + }); + const expectedCoreRange = semanticWindowCoreRange(expectedWindow); + const expectedLookAheadRange = semanticWindowLookAheadRange(expectedWindow); + assertSemanticManifestReplay( + manifestWindow.inputFingerprint === expectedWindow.inputFingerprint && + isSemanticUnitRangeTuple(manifestWindow.coreUnitRange) && + sameSemanticUnitRangeTuple(manifestWindow.coreUnitRange, expectedCoreRange) && + (manifestWindow.lookAheadUnitRange === undefined || + isSemanticUnitRangeTuple(manifestWindow.lookAheadUnitRange)) && + sameSemanticUnitRangeTuple(manifestWindow.lookAheadUnitRange, expectedLookAheadRange) && + isSemanticUnitRangeTuple(manifestWindow.committedUnitRange) && + manifestWindow.committedUnitRange[0] === expectedCoreRange.startUnitId, + `window ${windowOrdinal} does not match its canonical core/look-ahead input`, + ); + const coreEnd = globalUnitIndex.get(expectedCoreRange.endUnitId) as number; + const eligibleEnd = globalUnitIndex.get( + expectedLookAheadRange?.endUnitId ?? expectedCoreRange.endUnitId, + ) as number; + const commitEnd = globalUnitIndex.get(manifestWindow.committedUnitRange[1]); + assertSemanticManifestReplay( + commitEnd !== undefined && commitEnd >= coreEnd && commitEnd <= eligibleEnd, + `window ${windowOrdinal} has an invalid committed boundary`, + ); + + let expectedRangeStart = nextUnitIndex; + for (const [windowChunkIndex, chunkRange] of manifestWindow.chunkRanges.entries()) { + assertSemanticManifestReplay( + Array.isArray(chunkRange) && + chunkRange.length === 2 && + isSemanticUnitId(chunkRange[0]) && + isSemanticUnitId(chunkRange[1]), + `window ${windowOrdinal} chunk ${windowChunkIndex} has invalid identity`, + ); + const startUnitId = chunkRange[0]; + const endUnitId = chunkRange[1]; + const start = globalUnitIndex.get(startUnitId); + const end = globalUnitIndex.get(endUnitId); + assertSemanticManifestReplay( + start === expectedRangeStart && + end !== undefined && + end >= start && + end <= commitEnd && + start <= coreEnd && + (end <= coreEnd || end === commitEnd), + `window ${windowOrdinal} chunk ${windowChunkIndex} has a gap, overlap, or context-only range`, + ); + const first = units[start] as AtomicUnit; + const last = units[end] as AtomicUnit; + assertSemanticManifestReplay( + countUnicodeGraphemes(canonicalText.slice(first.startCodeUnit, last.endCodeUnit)) <= + effectiveConfig.maxChunkChars, + `window ${windowOrdinal} chunk ${windowChunkIndex} exceeds maxChunkChars=${effectiveConfig.maxChunkChars}`, + ); + expectedRangeStart = end + 1; + nextChunkIndex += 1; + } + assertSemanticManifestReplay( + expectedRangeStart === commitEnd + 1, + `window ${windowOrdinal} chunks do not end at the committed boundary`, + ); + nextUnitIndex = commitEnd + 1; + } + + assertSemanticManifestReplay( + nextChunkIndex === documentChunkCount, + "window chunks do not match documentChunkCount", + ); + assertSemanticManifestReplay( + nextUnitIndex === units.length, + "window manifest does not cover every canonical unit", + ); + assertSemanticManifestReplay( + usedCompletions.size === completionCatalog.length, + "completionCatalog contains unused entries", + ); +} + +/** Strict marker/payload validation shared by replay and semantic post-processing. */ +export function hasValidLlmSemanticJointExtraction(node: KnowledgeNode): boolean { + const semantic = node.metadata.semanticChunking; + const entitiesValue = node.metadata.extractedEntities; + const relationsValue = node.metadata.extractedRelations; + if ( + !isPlainObject(semantic) || + semantic.completed !== true || + semantic.strategy !== SEMANTIC_CHUNKING_STRATEGY || + semantic.schemaVersion !== SEMANTIC_CHUNKING_SCHEMA_VERSION || + !Array.isArray(entitiesValue) || + !Array.isArray(relationsValue) + ) { + return false; + } + const selection = KnowledgeSpaceModelSelectionSchema.safeParse(semantic.modelSelection); + const model = nonEmptyString(semantic.model); + const markerPromptVersion = nonEmptyString(semantic.promptVersion); + const chunkIndex = node.metadata.chunkIndex; + const documentChunkCount = semantic.documentChunkCount; + const maxChunkChars = semantic.maxChunkChars; + const requestedOverlapChars = semantic.requestedOverlapChars; + const coreRange = semanticMarkerUnitRange(semantic.windowCoreUnitRange); + const committedRange = semanticMarkerUnitRange(semantic.windowCommittedUnitRange); + if ( + !selection.success || + !model || + model !== selection.data.model || + !markerPromptVersion || + !isValidCompletionIdentity( + semantic.completion, + selection.data, + nonEmptyString(semantic.provider), + ) || + !nonEmptyString(semantic.windowId) || + semanticWindowOrdinal(nonEmptyString(semantic.windowId) ?? "") === undefined || + !isPlainObject(semantic.unitRange) || + !nonEmptyString(semantic.unitRange.startUnitId) || + !nonEmptyString(semantic.unitRange.endUnitId) || + !coreRange || + !committedRange || + coreRange.startUnitId !== committedRange.startUnitId || + (Object.hasOwn(semantic, "windowLookAheadUnitRange") && + !semanticMarkerUnitRange(semantic.windowLookAheadUnitRange)) || + typeof semantic.inputFingerprint !== "string" || + !/^sha256:[a-f0-9]{64}$/.test(semantic.inputFingerprint) || + !Number.isSafeInteger(chunkIndex) || + (chunkIndex as number) < 0 || + !Number.isSafeInteger(documentChunkCount) || + (documentChunkCount as number) < 1 || + (chunkIndex as number) >= (documentChunkCount as number) || + !Number.isSafeInteger(maxChunkChars) || + (maxChunkChars as number) < 1 || + countUnicodeGraphemes(node.text) > (maxChunkChars as number) || + !Number.isSafeInteger(requestedOverlapChars) || + (requestedOverlapChars as number) < 0 || + (requestedOverlapChars as number) >= (maxChunkChars as number) || + semantic.overlapApplied !== false || + semantic.overlapPolicy !== "non-overlapping-semantic-output" || + !isMatchingJointExtractionMetadata({ + count: entitiesValue.length, + countKey: "entityCount", + metadata: node.metadata.entityExtraction, + model, + promptVersion: markerPromptVersion, + }) || + !isMatchingJointExtractionMetadata({ + count: relationsValue.length, + countKey: "relationCount", + metadata: node.metadata.relationExtraction, + model, + promptVersion: markerPromptVersion, + }) + ) { + return false; + } + + const entitiesByResponseId = new Map(); + for (const entity of entitiesValue) { + if ( + !isPlainObject(entity) || + typeof entity.text !== "string" || + !entity.text.trim() || + !node.text.includes(entity.text.trim()) || + !EntityTypeSchema.safeParse(entity.type).success || + typeof entity.confidence !== "number" || + !Number.isFinite(entity.confidence) || + entity.confidence < 0 || + entity.confidence > 1 || + !isPlainObject(entity.metadata) + ) { + return false; + } + const responseEntityId = nonEmptyString(entity.metadata.responseEntityId); + const hasCanonicalName = Object.hasOwn(entity.metadata, "canonicalName"); + const canonicalName = hasCanonicalName + ? nonEmptyString(entity.metadata.canonicalName) + : undefined; + if ( + !responseEntityId || + entitiesByResponseId.has(responseEntityId) || + (hasCanonicalName && !canonicalName) || + entity.metadata.source !== "llm-semantic-chunking" + ) { + return false; + } + entitiesByResponseId.set(responseEntityId, canonicalName ?? entity.text.trim()); + } + + for (const relation of relationsValue) { + if ( + !isPlainObject(relation) || + typeof relation.subject !== "string" || + typeof relation.object !== "string" || + !RelationTypeSchema.safeParse(relation.type).success || + typeof relation.confidence !== "number" || + !Number.isFinite(relation.confidence) || + relation.confidence < 0 || + relation.confidence > 1 || + !isPlainObject(relation.metadata) + ) { + return false; + } + const subjectEntityId = nonEmptyString(relation.metadata.subjectEntityId); + const objectEntityId = nonEmptyString(relation.metadata.objectEntityId); + if ( + !subjectEntityId || + !objectEntityId || + relation.metadata.source !== "llm-semantic-chunking" || + entitiesByResponseId.get(subjectEntityId) !== relation.subject.trim() || + entitiesByResponseId.get(objectEntityId) !== relation.object.trim() + ) { + return false; + } + } + return true; +} + +function isValidCompletionIdentity( + value: unknown, + selection: KnowledgeSpaceModelSelection, + transportProvider?: string | undefined, +): boolean { + if (!isPlainObject(value) || !isPlainObject(value.actual)) { + return false; + } + if (!isExactModelSelection(value.requested, selection)) { + return false; + } + if ( + transportProvider !== undefined && + boundedCompletionString(transportProvider, MAX_COMPLETION_PROVIDER_CHARS) === undefined + ) { + return false; + } + const actual = value.actual; + if (transportProvider === "plugin-daemon") { + return ( + boundedCompletionString(actual.model, MAX_COMPLETION_MODEL_CHARS) === selection.model && + boundedCompletionString(actual.provider, MAX_COMPLETION_PROVIDER_CHARS) === "plugin-daemon" && + (!Object.hasOwn(actual, "finishReason") || + Boolean(boundedCompletionString(actual.finishReason, MAX_COMPLETION_FINISH_REASON_CHARS))) + ); + } + if (Object.hasOwn(actual, "model")) { + const actualModel = boundedCompletionString(actual.model, MAX_COMPLETION_MODEL_CHARS); + if (!actualModel || actualModel !== selection.model) { + return false; + } + } + if ( + Object.hasOwn(actual, "provider") && + !boundedCompletionString(actual.provider, MAX_COMPLETION_PROVIDER_CHARS) + ) { + return false; + } + if ( + Object.hasOwn(actual, "finishReason") && + !boundedCompletionString(actual.finishReason, MAX_COMPLETION_FINISH_REASON_CHARS) + ) { + return false; + } + return true; +} + +function boundedCompletionString(value: unknown, maxChars: number): string | undefined { + const normalized = nonEmptyString(value); + return normalized && unicodeCodePointLength(normalized) <= maxChars ? normalized : undefined; +} + +function isExactModelSelection(value: unknown, expected: KnowledgeSpaceModelSelection): boolean { + const parsed = KnowledgeSpaceModelSelectionSchema.safeParse(value); + return ( + parsed.success && + parsed.data.model === expected.model && + parsed.data.pluginId === expected.pluginId && + parsed.data.provider === expected.provider + ); +} + +function isMatchingLayoutRecomposition( + value: unknown, + expected: DocumentLayoutRecompositionStats & { readonly fingerprint: string }, +): boolean { + return ( + isPlainObject(value) && + value.fingerprint === expected.fingerprint && + value.elementsRecomposed === expected.elementsRecomposed && + value.modelDecidedHeadingBoundaries === expected.modelDecidedHeadingBoundaries && + value.trustedHeadingBoundaries === expected.trustedHeadingBoundaries + ); +} + +function isMatchingJointExtractionMetadata({ + count, + countKey, + metadata, + model, + promptVersion, +}: { + readonly count: number; + readonly countKey: "entityCount" | "relationCount"; + readonly metadata: unknown; + readonly model: string; + readonly promptVersion: string; +}): boolean { + return ( + isPlainObject(metadata) && + metadata.completed === true && + metadata.model === model && + metadata.promptVersion === promptVersion && + metadata.source === SEMANTIC_CHUNKING_STRATEGY && + typeof metadata.extractedAt === "string" && + DateTimeSchema.safeParse(metadata.extractedAt).success && + metadata[countKey] === count + ); +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +export interface SemanticUnitRange { + readonly endUnitId: string; + readonly startUnitId: string; +} + +function assertBoundedOptionalCompletionField( + value: string | undefined, + field: string, + maxChars: number, + source: "manifest" | "terminal", +): void { + if ( + value !== undefined && + (typeof value !== "string" || + !value.trim() || + value !== value.trim() || + unicodeCodePointLength(value) > maxChars) + ) { + const message = `LLM semantic chunking ${source} ${field} must be a trimmed non-empty string of at most ${maxChars} characters`; + if (source === "manifest") { + throw new Error(`LLM semantic window manifest replay validation failed: ${message}`); + } + throw new Error(message); + } +} + +function unicodeCodePointLength(value: string): number { + let count = 0; + for (const _character of value) count += 1; + return count; +} + +function isSemanticWindowId(value: unknown): value is string { + return ( + typeof value === "string" && + unicodeCodePointLength(value) <= MAX_SEMANTIC_WINDOW_ID_CHARS && + /^window-\d{6,}$/u.test(value) + ); +} + +function isSemanticUnitId(value: unknown): value is string { + return ( + typeof value === "string" && + unicodeCodePointLength(value) <= MAX_SEMANTIC_UNIT_ID_CHARS && + /^u-\d{6,}-\d{6,}$/u.test(value) + ); +} + +function isSemanticUnitRange(value: unknown): value is SemanticUnitRange { + return ( + isPlainObject(value) && isSemanticUnitId(value.startUnitId) && isSemanticUnitId(value.endUnitId) + ); +} + +function isSemanticUnitRangeTuple(value: unknown): value is LlmSemanticUnitRangeTuple { + return ( + Array.isArray(value) && + value.length === 2 && + isSemanticUnitId(value[0]) && + isSemanticUnitId(value[1]) + ); +} + +function semanticWindowId(ordinal: number): string { + return `window-${ordinal.toString().padStart(6, "0")}`; +} + +function semanticMarkerUnitRange(value: unknown): SemanticUnitRange | undefined { + if (!isPlainObject(value)) return undefined; + const startUnitId = nonEmptyString(value.startUnitId); + const endUnitId = nonEmptyString(value.endUnitId); + return startUnitId && endUnitId ? { endUnitId, startUnitId } : undefined; +} + +function semanticWindowCoreRange(window: SemanticWindow): SemanticUnitRange { + return { + endUnitId: (window.units.at(-1) as AtomicUnit).id, + startUnitId: (window.units[0] as AtomicUnit).id, + }; +} + +function semanticWindowLookAheadRange(window: SemanticWindow): SemanticUnitRange | undefined { + const first = window.lookAheadUnits[0]; + const last = window.lookAheadUnits.at(-1); + return first && last ? { endUnitId: last.id, startUnitId: first.id } : undefined; +} + +function sameSemanticUnitRange( + left: SemanticUnitRange | undefined, + right: SemanticUnitRange | undefined, +): boolean { + return ( + left === right || + (left !== undefined && + right !== undefined && + left.startUnitId === right.startUnitId && + left.endUnitId === right.endUnitId) + ); +} + +function sameSemanticUnitRangeTuple( + left: LlmSemanticUnitRangeTuple | undefined, + right: SemanticUnitRange | undefined, +): boolean { + return ( + (left === undefined && right === undefined) || + (left !== undefined && + right !== undefined && + left[0] === right.startUnitId && + left[1] === right.endUnitId) + ); +} + +function semanticWindowOrdinal(windowId: string): number | undefined { + const match = /^window-(\d{6,})$/u.exec(windowId); + if (!match?.[1]) return undefined; + const value = Number(match[1]); + return Number.isSafeInteger(value) ? value : undefined; +} + +function assertSemanticManifestReplay(condition: unknown, reason: string): asserts condition { + if (!condition) { + throw new Error(`LLM semantic window manifest replay validation failed: ${reason}`); + } +} + +function plainObjectValue(value: unknown): boolean { + return isPlainObject(value); +} + +function arrayOfStrings(value: unknown): string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string") ? value : []; +} + +function assertSemanticReplay(condition: unknown, reason: string): asserts condition { + if (!condition) { + throw new Error(`LLM semantic replay validation failed: ${reason}`); + } +} + +function resolveConfig( + requested: SemanticChunkerInput["config"], + defaults: EffectiveChunkConfig, +): EffectiveChunkConfig { + const maxChunkChars = requested?.maxChunkChars ?? defaults.maxChunkChars; + const resolved = { + maxChunkChars, + maxNodes: requested?.maxNodes ?? defaults.maxNodes, + maxWindowChars: requested?.maxWindowChars ?? Math.max(defaults.maxWindowChars, maxChunkChars), + requestedOverlapChars: requested?.overlapChars ?? defaults.requestedOverlapChars, + }; + validatePositiveInteger("maxChunkChars", resolved.maxChunkChars); + validatePositiveInteger("maxNodes", resolved.maxNodes); + validatePositiveInteger("maxWindowChars", resolved.maxWindowChars); + validateNonnegativeInteger("overlapChars", resolved.requestedOverlapChars); + if (resolved.maxWindowChars < resolved.maxChunkChars) { + throw new Error("LLM semantic chunking maxWindowChars must be at least maxChunkChars"); + } + if (resolved.requestedOverlapChars >= resolved.maxChunkChars) { + throw new Error("LLM semantic chunking overlapChars must be less than maxChunkChars"); + } + return resolved; +} + +function materializeElements(parseArtifact: ParseArtifact): { + readonly canonicalText: string; + readonly elements: readonly MaterializedElement[]; + readonly layoutRecomposition: DocumentLayoutRecompositionStats & { + readonly fingerprint: string; + }; +} { + const recomposed = recomposeDocumentLayoutForSemanticSegmentation(parseArtifact); + const elements: MaterializedElement[] = []; + let canonicalText = ""; + let nextOffset = 0; + + for (const [elementIndex, element] of recomposed.artifact.elements.entries()) { + const span = materializeDocumentElementByteSpan(element.text, nextOffset); + if (!span) { + continue; + } + const separator = elements.length === 0 ? "" : DOCUMENT_ELEMENT_SEPARATOR; + const startCodeUnit = canonicalText.length + separator.length; + canonicalText += `${separator}${span.text}`; + nextOffset = span.nextOffset; + elements.push({ + elementId: element.id, + elementIndex, + elementMetadata: cloneJsonObject(element.metadata), + elementType: element.type, + endCodeUnit: canonicalText.length, + endOffset: span.endOffset, + ...(element.pageNumber === undefined ? {} : { pageNumber: element.pageNumber }), + sectionPath: [...element.sectionPath], + startCodeUnit, + startOffset: span.startOffset, + text: span.text, + }); + } + + return { + canonicalText, + elements, + layoutRecomposition: { fingerprint: recomposed.fingerprint, ...recomposed.stats }, + }; +} + +function materializeAtomicUnits( + elements: readonly MaterializedElement[], + maxChunkChars: number, +): AtomicUnit[] { + const units: AtomicUnit[] = []; + + for (const element of elements) { + const sentenceRanges = semanticRanges(element); + let atomicIndex = 0; + for (const range of sentenceRanges) { + const sentence = element.text.slice(range.start, range.end); + for (const hardRange of graphemeRanges(sentence, maxChunkChars)) { + const localStart = range.start + hardRange.start; + const localEnd = range.start + hardRange.end; + const text = element.text.slice(localStart, localEnd); + const startOffset = element.startOffset + utf8ByteLength(element.text.slice(0, localStart)); + const endOffset = element.startOffset + utf8ByteLength(element.text.slice(0, localEnd)); + units.push({ + elementId: element.elementId, + elementMetadata: cloneJsonObject(element.elementMetadata), + elementType: element.elementType, + endCodeUnit: element.startCodeUnit + localEnd, + endOffset, + graphemeLength: countUnicodeGraphemes(text), + id: `u-${element.elementIndex.toString().padStart(6, "0")}-${atomicIndex + .toString() + .padStart(6, "0")}`, + ...(element.elementType === "image" || element.elementType === "table" + ? { isolationKey: `${element.elementType}:${element.elementId}` } + : {}), + ...(element.pageNumber === undefined ? {} : { pageNumber: element.pageNumber }), + sectionPath: [...element.sectionPath], + startCodeUnit: element.startCodeUnit + localStart, + startOffset, + text, + }); + atomicIndex += 1; + } + } + } + + return units; +} + +function semanticRanges(element: MaterializedElement): Array<{ end: number; start: number }> { + if (element.elementType !== "paragraph" && element.elementType !== "list") { + return [{ end: element.text.length, start: 0 }]; + } + + const ranges = Array.from( + new Intl.Segmenter("und", { granularity: "sentence" }).segment(element.text), + (segment) => ({ + end: segment.index + segment.segment.length, + start: segment.index, + }), + ); + + return ranges; +} + +function graphemeRanges( + text: string, + maxChunkChars: number, +): Array<{ end: number; start: number }> { + const ranges: Array<{ end: number; start: number }> = []; + let count = 0; + let start = 0; + let end = 0; + + for (const segment of graphemeSegments(text)) { + if (count === maxChunkChars) { + ranges.push({ end, start }); + start = segment.index; + count = 0; + } + end = segment.index + segment.segment.length; + count += 1; + } + if (end > start) { + ranges.push({ end, start }); + } + return ranges; +} + +function preflightMaterializedSemanticWindows({ + canonicalText, + effectiveConfig, + units, +}: { + readonly canonicalText: string; + readonly effectiveConfig: EffectiveChunkConfig; + readonly units: readonly AtomicUnit[]; +}): LlmSemanticWindowPreflightResult { + let maximumWindowCount = 0; + let nextUnitIndex = 0; + while (nextUnitIndex < units.length) { + if (maximumWindowCount >= DEFAULT_MAX_SEMANTIC_WINDOWS) { + throw new Error( + `LLM semantic chunking deterministic window count exceeds maxSemanticWindows=${DEFAULT_MAX_SEMANTIC_WINDOWS}`, + ); + } + const window = materializeSemanticWindow({ + canonicalText, + maxChunkChars: effectiveConfig.maxChunkChars, + maxWindowChars: effectiveConfig.maxWindowChars, + startUnitIndex: nextUnitIndex, + units, + windowIndex: maximumWindowCount, + }); + nextUnitIndex += window.units.length; + maximumWindowCount += 1; + } + return { maximumWindowCount, unitCount: units.length }; +} + +function materializeSemanticWindow({ + canonicalText, + maxChunkChars, + maxWindowChars, + startUnitIndex, + units, + windowIndex, +}: { + readonly canonicalText: string; + readonly maxChunkChars: number; + readonly maxWindowChars: number; + readonly startUnitIndex: number; + readonly units: readonly AtomicUnit[]; + readonly windowIndex: number; +}): SemanticWindow { + const first = units[startUnitIndex]; + if (!first) { + throw new Error("LLM semantic chunking cannot materialize an empty semantic window"); + } + + const coreUnits: AtomicUnit[] = []; + let cursor = startUnitIndex; + while (cursor < units.length) { + const candidate = units[cursor] as AtomicUnit; + if (!isWindowCompatible(first, candidate)) break; + const prospectiveLength = countUnicodeGraphemes( + canonicalText.slice(first.startCodeUnit, candidate.endCodeUnit), + ); + if (coreUnits.length > 0 && prospectiveLength > maxWindowChars) break; + coreUnits.push(candidate); + cursor += 1; + } + + // Look-ahead is prompt context, not an independently coverable range. It deliberately overlaps + // the next request when the model keeps the nominal core boundary. The final core-starting chunk + // may consume part/all of it, which makes the actual committed boundary semantic rather than a + // fixed maxWindowChars cut. A maxChunkChars budget is sufficient because no valid chunk can + // extend farther than that hard Unicode-grapheme cap. + const lookAheadUnits: AtomicUnit[] = []; + const firstLookAhead = units[cursor]; + while (firstLookAhead && cursor < units.length) { + const candidate = units[cursor] as AtomicUnit; + if (!isWindowCompatible(first, candidate)) break; + const prospectiveLength = countUnicodeGraphemes( + canonicalText.slice(firstLookAhead.startCodeUnit, candidate.endCodeUnit), + ); + if (lookAheadUnits.length > 0 && prospectiveLength > maxChunkChars) break; + if (prospectiveLength > maxChunkChars) break; + lookAheadUnits.push(candidate); + cursor += 1; + } + + const id = `window-${windowIndex.toString().padStart(6, "0")}`; + const fingerprintSource = { + lookAheadUnits: lookAheadUnits.map(semanticPromptUnit), + sectionPath: [...first.sectionPath], + units: coreUnits.map(semanticPromptUnit), + windowId: id, + }; + return { + id, + inputFingerprint: `sha256:${createHash("sha256") + .update(JSON.stringify(fingerprintSource)) + .digest("hex")}`, + lookAheadUnits, + sectionPath: [...first.sectionPath], + units: coreUnits, + }; +} + +function semanticPromptUnit(unit: AtomicUnit): { + readonly graphemeLength: number; + readonly id: string; + readonly text: string; + readonly type: string; +} { + return { + graphemeLength: unit.graphemeLength, + id: unit.id, + text: unit.text, + type: unit.elementType, + }; +} + +function isWindowCompatible(first: AtomicUnit, candidate: AtomicUnit): boolean { + return ( + sameStrings(first.sectionPath, candidate.sectionPath) && + first.isolationKey === candidate.isolationKey && + (first.isolationKey === undefined || first.elementId === candidate.elementId) + ); +} + +function semanticChunkingMessages({ + maxChunkChars, + maxEntitiesPerChunk, + maxRelationsPerChunk, + window, +}: { + readonly maxChunkChars: number; + readonly maxEntitiesPerChunk: number; + readonly maxRelationsPerChunk: number; + readonly window: SemanticWindow; +}): readonly SemanticChunkingLlmMessage[] { + return [ + { + content: [ + "You choose semantically complete chunk boundaries and extract graph facts in one pass.", + "Return strict JSON only. Never return, rewrite, summarize, correct, or duplicate source text.", + "The units field is the core: cover every core unit exactly once, in order, by contiguous inclusive ranges.", + "lookAheadUnits is context-only. Only the final chunk may extend into it, and that final chunk must start in the core.", + "Never emit a chunk that starts wholly in lookAheadUnits. Units not consumed from look-ahead will be reconsidered in the next request.", + "Ranges may be smaller than the maximum; prefer natural topic boundaries over filling chunks.", + `Every range must contain at most ${maxChunkChars} Unicode graphemes including separators.`, + `Return at most ${maxEntitiesPerChunk} entities and ${maxRelationsPerChunk} relations per chunk.`, + "Allowed entity types: date, metric, organization, person, policy, product, term.", + "Allowed relation types: mentions, defines, references, depends_on, supersedes, contradicts.", + "Entity text must be an exact source substring. Give every entity a response-local unique id.", + "Relations must reference entity ids from that same chunk through subjectEntityId/objectEntityId; never use names as relation endpoints.", + "Assign every chunk a concise semantic sectionPath and sectionSummary. Preserve the supplied sectionPath as a prefix when it is non-empty; add only meaningful child levels.", + "Output shape:", + '{"chunks":[{"startUnitId":"u-...","endUnitId":"u-...","sectionPath":["Policy","Eligibility"],"sectionSummary":"Who is eligible and under what conditions.","entities":[{"id":"e-1","text":"Acme","type":"organization","confidence":0.95,"canonicalName":"Acme Corp","aliases":["Acme"]},{"id":"e-2","text":"Policy A","type":"policy","confidence":0.9}],"relations":[{"subjectEntityId":"e-1","type":"references","objectEntityId":"e-2","confidence":0.9}]}]}', + ].join("\n"), + role: "system", + }, + { + content: JSON.stringify({ + lookAheadUnits: window.lookAheadUnits.map(semanticPromptUnit), + sectionPath: window.sectionPath, + units: window.units.map(semanticPromptUnit), + windowId: window.id, + }), + role: "user", + }, + ]; +} + +async function collectProviderCompletion({ + maxOutputTokens, + maxResponseChars, + messages, + model, + provider, + temperature, + tenantId, +}: { + readonly maxOutputTokens: number; + readonly maxResponseChars: number; + readonly messages: readonly SemanticChunkingLlmMessage[]; + readonly model: string; + readonly provider: SemanticChunkingLlmProvider; + readonly temperature: number; + readonly tenantId?: string | undefined; +}): Promise { + let text = ""; + let terminal: + | { + readonly finishReason?: string | undefined; + readonly metadata?: unknown; + } + | undefined; + for await (const event of provider.stream({ + maxOutputTokens, + messages, + model, + temperature, + ...(tenantId ? { tenantId } : {}), + })) { + if (terminal) { + throw new Error("LLM semantic chunking provider emitted data after its terminal event"); + } + if (event.type === "delta" && event.delta) { + text += event.delta; + if (text.length > maxResponseChars) { + throw new Error( + `LLM semantic chunking response exceeds maxResponseChars=${maxResponseChars}`, + ); + } + } + if (event.type === "done") { + terminal = event; + } + } + if (!terminal) { + throw new Error("LLM semantic chunking provider ended without a terminal event"); + } + if (!text.trim()) { + throw new Error("LLM semantic chunking provider returned an empty response"); + } + const finishReason = terminalStringField(terminal.finishReason, "finishReason"); + const actualModel = terminalMetadataStringField(terminal.metadata, "model"); + const actualProvider = terminalMetadataStringField(terminal.metadata, "provider"); + if (actualModel && actualModel !== model) { + throw new Error( + `LLM semantic chunking provider completed with model=${actualModel}, expected frozen model=${model}`, + ); + } + if ( + provider.kind === "plugin-daemon" && + (actualModel !== model || actualProvider !== "plugin-daemon") + ) { + throw new Error( + "LLM semantic chunking plugin-daemon completion must report the frozen model and plugin-daemon provider", + ); + } + return { + ...(actualModel ? { actualModel } : {}), + ...(actualProvider ? { actualProvider } : {}), + ...(finishReason ? { finishReason } : {}), + text, + }; +} + +function terminalMetadataStringField( + metadata: unknown, + field: "model" | "provider", +): string | undefined { + if (!isPlainObject(metadata) || !Object.hasOwn(metadata, field)) { + return undefined; + } + const value = metadata[field]; + if (typeof value !== "string" || !value.trim()) { + throw new Error(`LLM semantic chunking terminal metadata ${field} must be a non-empty string`); + } + const normalized = value.trim(); + assertBoundedOptionalCompletionField( + normalized, + field, + field === "model" ? MAX_COMPLETION_MODEL_CHARS : MAX_COMPLETION_PROVIDER_CHARS, + "terminal", + ); + return normalized; +} + +function terminalStringField(value: unknown, field: "finishReason"): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== "string" || !value.trim()) { + throw new Error(`LLM semantic chunking terminal ${field} must be a non-empty string`); + } + const normalized = value.trim(); + assertBoundedOptionalCompletionField( + normalized, + field, + MAX_COMPLETION_FINISH_REASON_CHARS, + "terminal", + ); + return normalized; +} + +function parseSemanticChunkingOutput(text: string): LlmSemanticChunkingOutput { + const trimmed = text.trim(); + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start < 0 || end <= start) { + throw new Error("LLM semantic chunking provider returned non-JSON output"); + } + try { + parsed = JSON.parse(trimmed.slice(start, end + 1)); + } catch (error) { + throw new Error("LLM semantic chunking provider returned invalid JSON", { cause: error }); + } + } + try { + return LlmSemanticChunkingOutputSchema.parse(parsed); + } catch (error) { + throw new Error("LLM semantic chunking provider returned an invalid response schema", { + cause: error, + }); + } +} + +function validateAndMaterializeWindowOutput({ + maxChunkChars, + maxEntitiesPerChunk, + maxRelationsPerChunk, + output, + window, +}: { + readonly maxChunkChars: number; + readonly maxEntitiesPerChunk: number; + readonly maxRelationsPerChunk: number; + readonly output: LlmSemanticChunkingOutput; + readonly window: SemanticWindow; +}): WindowMaterializedChunk[] { + if (output.chunks.length === 0) { + throw new Error("LLM semantic chunking response did not cover any input units"); + } + const eligibleUnits = [...window.units, ...window.lookAheadUnits]; + const unitIndex = new Map(eligibleUnits.map((unit, index) => [unit.id, index])); + const chunks: UncommittedWindowChunk[] = []; + let expectedStart = 0; + + for (const candidate of output.chunks) { + const start = unitIndex.get(candidate.startUnitId); + const end = unitIndex.get(candidate.endUnitId); + if (start === undefined || end === undefined) { + throw new Error("LLM semantic chunking response referenced an unknown unit ID"); + } + if (start !== expectedStart || end < start) { + throw new Error( + "LLM semantic chunking response must cover units contiguously without gaps or overlap", + ); + } + const chunkUnits = eligibleUnits.slice(start, end + 1); + const first = chunkUnits[0] as AtomicUnit; + const last = chunkUnits.at(-1) as AtomicUnit; + const chunkText = chunkUnits + .map((unit) => unit.text) + .reduce((combined, text, index) => { + if (index === 0) return text; + const previous = chunkUnits[index - 1] as AtomicUnit; + const separator = + previous.endCodeUnit === (chunkUnits[index] as AtomicUnit).startCodeUnit + ? "" + : DOCUMENT_ELEMENT_SEPARATOR; + return `${combined}${separator}${text}`; + }, ""); + if (countUnicodeGraphemes(chunkText) > maxChunkChars) { + throw new Error(`LLM semantic chunking response exceeded maxChunkChars=${maxChunkChars}`); + } + // Unit IDs are scoped to a prevalidated single-section window, so a returned range cannot + // cross sectionPath even when a model tries to reference an ID from another window. + if (candidate.entities.length > maxEntitiesPerChunk) { + throw new Error( + `LLM semantic chunking response exceeded maxEntitiesPerChunk=${maxEntitiesPerChunk}`, + ); + } + if (candidate.relations.length > maxRelationsPerChunk) { + throw new Error( + `LLM semantic chunking response exceeded maxRelationsPerChunk=${maxRelationsPerChunk}`, + ); + } + + const entities = candidate.entities.map((entity) => validateEntity(entity, chunkText)); + const entitiesById = new Map(); + for (const entity of entities) { + if (entitiesById.has(entity.id)) { + throw new Error("LLM semantic chunking entity ids must be unique within the same chunk"); + } + entitiesById.set(entity.id, entity); + } + const relations = candidate.relations.map((relation) => { + const subject = entitiesById.get(relation.subjectEntityId); + const object = entitiesById.get(relation.objectEntityId); + if (!subject || !object) { + throw new Error( + "LLM semantic chunking relation endpoint ids must reference entities in the same chunk", + ); + } + return { + confidence: relation.confidence, + object: object.canonicalName ?? object.text, + objectEntityId: object.id, + subject: subject.canonicalName ?? subject.text, + subjectEntityId: subject.id, + type: relation.type, + }; + }); + const sectionPath = resolveSemanticSectionPath(candidate.sectionPath, window.sectionPath); + const kind = commonSpecialKind(chunkUnits) ?? "chunk"; + chunks.push({ + endUnitId: last.id, + entities, + kind, + relations, + sectionPath, + ...(candidate.sectionSummary ? { sectionSummary: candidate.sectionSummary } : {}), + startUnitId: first.id, + units: chunkUnits, + window, + }); + expectedStart = end + 1; + } + + const finalChunk = chunks.at(-1); + const finalStart = finalChunk ? unitIndex.get(finalChunk.startUnitId) : undefined; + const finalEnd = finalChunk ? unitIndex.get(finalChunk.endUnitId) : undefined; + const coreEnd = window.units.length - 1; + if (finalStart === undefined || finalEnd === undefined || finalEnd < coreEnd) { + throw new Error( + "LLM semantic chunking response must cover units contiguously without gaps or overlap", + ); + } + if (finalStart > coreEnd) { + throw new Error("LLM semantic chunking final chunk must start in the core window"); + } + const commitEndUnitId = eligibleUnits[finalEnd]?.id; + if (!commitEndUnitId) { + throw new Error("LLM semantic chunking response has an invalid committed boundary"); + } + return chunks.map((chunk) => ({ ...chunk, windowCommitEndUnitId: commitEndUnitId })); +} + +function validateEntity(entity: LlmSemanticEntity, chunkText: string): LlmSemanticEntity { + const text = entity.text.trim(); + if (!chunkText.includes(text)) { + throw new Error("LLM semantic chunking entity text must be an exact chunk substring"); + } + return { + ...(entity.aliases + ? { aliases: [...new Set(entity.aliases.map((alias) => alias.trim()))] } + : {}), + ...(entity.canonicalName ? { canonicalName: entity.canonicalName.trim() } : {}), + confidence: entity.confidence, + id: entity.id, + text, + type: entity.type, + }; +} + +function resolveSemanticSectionPath( + proposed: readonly string[] | undefined, + trustedPrefix: readonly string[], +): string[] { + const path = proposed ? proposed.map((segment) => segment.trim()) : [...trustedPrefix]; + if ( + trustedPrefix.length > 0 && + !sameStrings(path.slice(0, trustedPrefix.length), trustedPrefix) + ) { + throw new Error("LLM semantic chunking sectionPath must preserve the trusted parser prefix"); + } + return path; +} + +function hasValidSemanticSectionReplay( + value: unknown, + trustedPrefix: readonly string[], + sourceSectionPath: readonly string[], +): boolean { + if (!isPlainObject(value)) { + return sameStrings(sourceSectionPath, trustedPrefix); + } + const path = arrayOfStrings(value.path); + const summary = value.summary; + return ( + (path.length >= 1 || trustedPrefix.length === 0) && + path.length <= 8 && + path.every((segment) => segment.length <= 160) && + sameStrings(path, sourceSectionPath) && + (trustedPrefix.length === 0 || + sameStrings(path.slice(0, trustedPrefix.length), trustedPrefix)) && + (summary === undefined || + (typeof summary === "string" && summary.trim().length > 0 && summary.length <= 2_000)) + ); +} + +function materializeKnowledgeNode({ + canonicalText, + chunk, + chunkIndex, + documentChunkCount, + extractedAt, + input, + layoutRecomposition, + maxChunkChars, + modelSelection, + promptVersion, + providerKind, + requestedOverlapChars, +}: { + readonly canonicalText: string; + readonly chunk: MaterializedChunk; + readonly chunkIndex: number; + readonly documentChunkCount: number; + readonly extractedAt: string; + readonly input: SemanticChunkerInput; + readonly layoutRecomposition: DocumentLayoutRecompositionStats & { + readonly fingerprint: string; + }; + readonly maxChunkChars: number; + readonly modelSelection: KnowledgeSpaceModelSelection; + readonly promptVersion: string; + readonly providerKind?: string | undefined; + readonly requestedOverlapChars: number; +}): KnowledgeNode { + const first = chunk.units[0] as AtomicUnit; + const last = chunk.units.at(-1) as AtomicUnit; + const text = canonicalText.slice(first.startCodeUnit, last.endCodeUnit); + const entities = chunk.entities.map((entity) => ({ + confidence: entity.confidence, + metadata: { + ...(entity.aliases && entity.aliases.length > 0 ? { aliases: entity.aliases } : {}), + ...(entity.canonicalName ? { canonicalName: entity.canonicalName } : {}), + responseEntityId: entity.id, + source: "llm-semantic-chunking", + }, + text: entity.text, + type: entity.type, + })); + const relations = chunk.relations.map((relation) => ({ + confidence: relation.confidence, + metadata: { + objectEntityId: relation.objectEntityId, + source: "llm-semantic-chunking", + subjectEntityId: relation.subjectEntityId, + }, + object: relation.object, + subject: relation.subject, + type: relation.type, + })); + const metadata: Record = { + chunkIndex, + elementIds: uniqueStrings(chunk.units.map((unit) => unit.elementId)), + elementSeparator: DOCUMENT_ELEMENT_SEPARATOR, + elementTypes: uniqueStrings(chunk.units.map((unit) => unit.elementType)), + entityExtraction: { + completed: true, + entityCount: entities.length, + extractedAt, + model: modelSelection.model, + promptVersion, + source: SEMANTIC_CHUNKING_STRATEGY, + }, + extractedEntities: entities, + extractedRelations: relations, + offsetEncoding: DOCUMENT_OFFSET_ENCODING, + relationExtraction: { + completed: true, + extractedAt, + model: modelSelection.model, + promptVersion, + relationCount: relations.length, + source: SEMANTIC_CHUNKING_STRATEGY, + }, + semanticChunking: { + completed: true, + completion: { + actual: { + ...(chunk.completion.actualModel ? { model: chunk.completion.actualModel } : {}), + ...(chunk.completion.actualProvider ? { provider: chunk.completion.actualProvider } : {}), + ...(chunk.completion.finishReason ? { finishReason: chunk.completion.finishReason } : {}), + }, + requested: cloneJsonObject(modelSelection), + }, + documentChunkCount, + inputFingerprint: chunk.window.inputFingerprint, + layoutRecomposition: { + elementsRecomposed: layoutRecomposition.elementsRecomposed, + fingerprint: layoutRecomposition.fingerprint, + modelDecidedHeadingBoundaries: layoutRecomposition.modelDecidedHeadingBoundaries, + trustedHeadingBoundaries: layoutRecomposition.trustedHeadingBoundaries, + }, + maxChunkChars, + model: modelSelection.model, + modelSelection: cloneJsonObject(modelSelection), + overlapApplied: false, + overlapPolicy: "non-overlapping-semantic-output", + promptVersion, + ...(providerKind ? { provider: providerKind } : {}), + requestedOverlapChars, + schemaVersion: SEMANTIC_CHUNKING_SCHEMA_VERSION, + section: { + path: [...chunk.sectionPath], + ...(chunk.sectionSummary ? { summary: chunk.sectionSummary } : {}), + }, + strategy: SEMANTIC_CHUNKING_STRATEGY, + unitRange: { + endUnitId: chunk.endUnitId, + startUnitId: chunk.startUnitId, + }, + windowCommittedUnitRange: { + endUnitId: chunk.windowCommitEndUnitId, + startUnitId: (chunk.window.units[0] as AtomicUnit).id, + }, + windowCoreUnitRange: { + endUnitId: (chunk.window.units.at(-1) as AtomicUnit).id, + startUnitId: (chunk.window.units[0] as AtomicUnit).id, + }, + windowId: chunk.window.id, + ...(chunk.window.lookAheadUnits.length > 0 + ? { + windowLookAheadUnitRange: { + endUnitId: (chunk.window.lookAheadUnits.at(-1) as AtomicUnit).id, + startUnitId: (chunk.window.lookAheadUnits[0] as AtomicUnit).id, + }, + } + : {}), + }, + textNormalization: DOCUMENT_ELEMENT_TEXT_NORMALIZATION, + }; + if (uniqueStrings(chunk.units.map((unit) => unit.elementId)).length === 1) { + mergeSingleElementMetadata(metadata, first.elementMetadata); + } + const pageNumber = commonPageNumber(chunk.units); + + return KnowledgeNodeSchema.parse({ + artifactHash: input.parseArtifact.artifactHash, + documentAssetId: input.parseArtifact.documentAssetId, + endOffset: last.endOffset, + id: deterministicChildId( + input.publicationGenerationId ?? input.parseArtifact.id, + `${SEMANTIC_CHUNKING_STRATEGY}:${input.parseArtifact.id}:${input.parseArtifact.artifactHash}:${first.startOffset}:${last.endOffset}`, + ), + kind: chunk.kind, + knowledgeSpaceId: input.knowledgeSpaceId, + metadata, + parseArtifactId: input.parseArtifact.id, + permissionScope: [...(input.permissionScope ?? [])], + ...(input.publicationGenerationId + ? { publicationGenerationId: input.publicationGenerationId } + : {}), + sourceLocation: { + endOffset: last.endOffset, + ...(pageNumber === undefined ? {} : { pageNumber }), + sectionPath: [...chunk.sectionPath], + startOffset: first.startOffset, + }, + startOffset: first.startOffset, + text, + }); +} + +function commonSpecialKind(units: readonly AtomicUnit[]): "image" | "table" | undefined { + const first = units[0]; + if ( + first && + (first.elementType === "image" || first.elementType === "table") && + units.every( + (unit) => unit.elementId === first.elementId && unit.elementType === first.elementType, + ) + ) { + return first.elementType; + } + return undefined; +} + +function commonPageNumber(units: readonly AtomicUnit[]): number | undefined { + const pageNumber = units[0]?.pageNumber; + return units.every((unit) => unit.pageNumber === pageNumber) ? pageNumber : undefined; +} + +function mergeSingleElementMetadata( + target: Record, + source: Readonly>, +): void { + for (const key of [ + "assetRef", + "boundingBox", + "caption", + "ocrText", + "table", + "textAsHtml", + "title", + ]) { + if (Object.hasOwn(source, key)) { + target[key] = JSON.parse(JSON.stringify(source[key])) as unknown; + } + } +} + +function validatePositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value < 1) { + throw new Error(`LLM semantic chunking ${name} must be at least 1`); + } +} + +function validateNonnegativeInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`LLM semantic chunking ${name} must be a non-negative integer`); + } +} + +function sameStrings(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + +function uniqueStrings(values: readonly string[]): string[] { + return [...new Set(values)]; +} + +function utf8ByteLength(text: string): number { + return encoder.encode(text).byteLength; +} + +const EntityTypeSchema = z.enum([ + "date", + "metric", + "organization", + "person", + "policy", + "product", + "term", +]); + +const RelationTypeSchema = z.enum([ + "contradicts", + "defines", + "depends_on", + "mentions", + "references", + "supersedes", +]); + +const LlmSemanticEntitySchema = z + .object({ + aliases: z.array(z.string().trim().min(1)).max(12).optional(), + canonicalName: z.string().trim().min(1).optional(), + confidence: z.number().finite().min(0).max(1), + id: z.string().trim().min(1).max(128), + text: z.string().trim().min(1), + type: EntityTypeSchema, + }) + .strict(); + +const LlmSemanticRelationSchema = z + .object({ + confidence: z.number().finite().min(0).max(1), + objectEntityId: z.string().trim().min(1).max(128), + subjectEntityId: z.string().trim().min(1).max(128), + type: RelationTypeSchema, + }) + .strict(); + +const LlmSemanticChunkingOutputSchema = z + .object({ + chunks: z.array( + z + .object({ + endUnitId: z.string().min(1), + entities: z.array(LlmSemanticEntitySchema), + relations: z.array(LlmSemanticRelationSchema), + sectionPath: z.array(z.string().trim().min(1).max(160)).min(1).max(8).optional(), + sectionSummary: z.string().trim().min(1).max(2_000).optional(), + startUnitId: z.string().min(1), + }) + .strict(), + ), + }) + .strict(); + +type LlmSemanticChunkingOutput = z.infer; +type LlmSemanticEntity = z.infer; +type LlmSemanticRelation = z.infer; diff --git a/knowledge-fs/packages/api/src/semantic-generation-receipt.ts b/knowledge-fs/packages/api/src/semantic-generation-receipt.ts new file mode 100644 index 00000000000..06f4483d904 --- /dev/null +++ b/knowledge-fs/packages/api/src/semantic-generation-receipt.ts @@ -0,0 +1,169 @@ +import { createHash } from "node:crypto"; + +import type { KnowledgeSpaceModelSelection } from "@knowledge/core"; +import { stableJson } from "@knowledge/core"; + +export const MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_BYTES = 4 * 1024 * 1024; +export const MAX_KNOWLEDGE_NODE_GENERATION_RECEIPT_DATABASE_TEXT_BYTES = 8 * 1024 * 1024; +export const MAX_LLM_SEMANTIC_WINDOWS = 4_096; +export const MAX_LLM_SEMANTIC_COMPLETION_IDENTITIES = 64; +export const MAX_LLM_SEMANTIC_WINDOW_ID_CODE_POINTS = 32; +export const MAX_LLM_SEMANTIC_UNIT_ID_CODE_POINTS = 32; +export const MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS = 255; +export const MAX_LLM_SEMANTIC_FINISH_REASON_CODE_POINTS = 64; + +const SHA256_FINGERPRINT = `sha256:${"f".repeat(64)}`; +const MAX_WINDOW_ID = `window-${"9".repeat( + MAX_LLM_SEMANTIC_WINDOW_ID_CODE_POINTS - "window-".length, +)}`; +const MAX_UNIT_ID = `u-${"9".repeat(14)}-${"9".repeat(15)}`; +const MAX_TERMINAL_IDENTITY = "\u{1f600}".repeat(MAX_LLM_SEMANTIC_TERMINAL_IDENTITY_CODE_POINTS); +const MAX_FINISH_REASON = "\u{1f600}".repeat(MAX_LLM_SEMANTIC_FINISH_REASON_CODE_POINTS); + +export interface KnowledgeNodeSemanticGenerationConfig { + readonly maxChunkChars: number; + readonly maxNodes: number; + readonly maxWindowChars: number; + readonly overlapChars: number; + readonly promptVersion: string; +} + +export interface KnowledgeNodeGenerationCompletionReceipt { + readonly actualModel?: string | undefined; + readonly actualProvider?: string | undefined; + readonly fingerprint: string; + readonly finishReason?: string | undefined; + readonly transportProvider?: string | undefined; +} + +export type KnowledgeNodeGenerationUnitRangeReceipt = readonly [ + startUnitId: string, + endUnitId: string, +]; + +export interface KnowledgeNodeGenerationWindowReceipt { + readonly chunkRanges: readonly KnowledgeNodeGenerationUnitRangeReceipt[]; + readonly committedUnitRange: KnowledgeNodeGenerationUnitRangeReceipt; + readonly completionIndex: number; + readonly coreUnitRange: KnowledgeNodeGenerationUnitRangeReceipt; + readonly firstChunkIndex: number; + readonly inputFingerprint: string; + readonly lookAheadUnitRange?: KnowledgeNodeGenerationUnitRangeReceipt | undefined; + /** Opaque hash of the complete generated semantic response payload for this window. */ + readonly responseFingerprint: string; + readonly windowId: string; +} + +/** + * Durable proof that semantic generation completed even when editorial exclusions persist no node + * rows. Window entries deliberately contain only canonical replay fields; completion identities + * are de-duplicated in a bounded catalog. + */ +export interface KnowledgeNodeGenerationReceipt { + readonly artifactHash: string; + readonly completionCatalog: readonly KnowledgeNodeGenerationCompletionReceipt[]; + readonly documentAssetId: string; + readonly documentChunkCount: number; + readonly excludedNodeOrdinals: readonly number[]; + readonly knowledgeSpaceId: string; + readonly language?: string | undefined; + readonly modelSelection: KnowledgeSpaceModelSelection; + readonly parseArtifactId: string; + readonly permissionScope: readonly string[]; + readonly promptResponseFingerprint: string; + readonly publicationGenerationId: string; + readonly requestFingerprint: string; + readonly responseFingerprint: string; + readonly schemaVersion: 1; + readonly semanticConfig: KnowledgeNodeSemanticGenerationConfig; + readonly storedNodeCount: number; + readonly storedResponseFingerprint: string; + readonly windowManifest: readonly KnowledgeNodeGenerationWindowReceipt[]; +} + +export function llmSemanticCompletionFingerprint( + entry: Omit, +): string { + return `sha256:${createHash("sha256") + .update( + stableJson({ + ...(entry.actualModel ? { actualModel: entry.actualModel } : {}), + ...(entry.actualProvider ? { actualProvider: entry.actualProvider } : {}), + ...(entry.finishReason ? { finishReason: entry.finishReason } : {}), + ...(entry.transportProvider ? { transportProvider: entry.transportProvider } : {}), + }), + ) + .digest("hex")}`; +} + +export function knowledgeNodeGenerationReceiptSerializedBytes(value: unknown): number { + return new TextEncoder().encode(stableJson(value)).byteLength; +} + +/** + * Exact upper bound for an admitted receipt. The caller supplies an envelope with the real ACL, + * language, model selection, prompt version and exclusions plus empty dynamic arrays. Dynamic + * bytes use the repository-enforced identifier/terminal caps, one chunk range per possible node, + * and the maximum bounded completion catalog. + */ +export function maximumKnowledgeNodeGenerationReceiptSerializedBytes({ + emptyReceipt, + maximumChunkCount, + maximumWindowCount, +}: { + readonly emptyReceipt: KnowledgeNodeGenerationReceipt; + readonly maximumChunkCount: number; + readonly maximumWindowCount: number; +}): number { + if (emptyReceipt.completionCatalog.length !== 0 || emptyReceipt.windowManifest.length !== 0) { + throw new Error("Semantic generation receipt admission requires empty dynamic arrays"); + } + if ( + !Number.isSafeInteger(maximumChunkCount) || + maximumChunkCount < 0 || + !Number.isSafeInteger(maximumWindowCount) || + maximumWindowCount < 0 || + maximumWindowCount > MAX_LLM_SEMANTIC_WINDOWS || + maximumWindowCount > maximumChunkCount + ) { + throw new Error("Semantic generation receipt admission bounds are invalid"); + } + + const completionCount = Math.min(maximumWindowCount, MAX_LLM_SEMANTIC_COMPLETION_IDENTITIES); + const maximumCompletion: KnowledgeNodeGenerationCompletionReceipt = { + actualModel: MAX_TERMINAL_IDENTITY, + actualProvider: MAX_TERMINAL_IDENTITY, + fingerprint: SHA256_FINGERPRINT, + finishReason: MAX_FINISH_REASON, + transportProvider: MAX_TERMINAL_IDENTITY, + }; + const completionBytes = knowledgeNodeGenerationReceiptSerializedBytes(maximumCompletion); + const completionCatalogBytes = arraySerializedBytes(completionCount, completionBytes); + + const maximumWindow: KnowledgeNodeGenerationWindowReceipt = { + chunkRanges: [], + committedUnitRange: [MAX_UNIT_ID, MAX_UNIT_ID], + completionIndex: Math.max(0, completionCount - 1), + coreUnitRange: [MAX_UNIT_ID, MAX_UNIT_ID], + firstChunkIndex: Math.max(0, maximumChunkCount - 1), + inputFingerprint: SHA256_FINGERPRINT, + lookAheadUnitRange: [MAX_UNIT_ID, MAX_UNIT_ID], + responseFingerprint: SHA256_FINGERPRINT, + windowId: MAX_WINDOW_ID, + }; + const emptyWindowBytes = knowledgeNodeGenerationReceiptSerializedBytes(maximumWindow); + const rangeBytes = knowledgeNodeGenerationReceiptSerializedBytes([MAX_UNIT_ID, MAX_UNIT_ID]); + // Every admitted window has at least one chunk. Across all window chunk arrays, JSON contributes + // one bracket/comma byte per window plus (rangeBytes + separator) per possible document chunk. + const allChunkRangeArraysBytes = maximumWindowCount + maximumChunkCount * (rangeBytes + 1); + const windowObjectsBytes = maximumWindowCount * (emptyWindowBytes - 2) + allChunkRangeArraysBytes; + const windowManifestBytes = + maximumWindowCount === 0 ? 2 : 2 + windowObjectsBytes + (maximumWindowCount - 1); + const emptyReceiptBytes = knowledgeNodeGenerationReceiptSerializedBytes(emptyReceipt); + + return emptyReceiptBytes - 4 + completionCatalogBytes + windowManifestBytes; +} + +function arraySerializedBytes(itemCount: number, itemBytes: number): number { + return itemCount === 0 ? 2 : 2 + itemCount * itemBytes + (itemCount - 1); +} diff --git a/knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.postgres.sql b/knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.postgres.sql new file mode 100644 index 00000000000..a9d05f301d6 --- /dev/null +++ b/knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.postgres.sql @@ -0,0 +1,44 @@ +-- Knowledge Platform schema migration +-- Migration id: 0043_semantic_generation_receipts +-- Dialect: postgres + +CREATE TABLE IF NOT EXISTS "knowledge_node_generation_receipts" ( + "knowledge_space_id" UUID NOT NULL, + "publication_generation_id" UUID NOT NULL, + "parse_artifact_id" UUID NOT NULL, + "document_asset_id" UUID NOT NULL, + "artifact_hash" VARCHAR(64) NOT NULL, + "document_chunk_count" INTEGER NOT NULL, + "stored_node_count" INTEGER NOT NULL, + "request_fingerprint" VARCHAR(71) NOT NULL, + "response_fingerprint" VARCHAR(71) NOT NULL, + "prompt_response_fingerprint" VARCHAR(71) NOT NULL, + "receipt" JSONB NOT NULL, + PRIMARY KEY ("knowledge_space_id", "publication_generation_id", "parse_artifact_id"), + CONSTRAINT "knowledge_node_generation_receipts_counts_ck" CHECK ( + "document_chunk_count" >= 0 AND "stored_node_count" >= 0 + AND "stored_node_count" <= "document_chunk_count" + ), + CONSTRAINT "knowledge_node_generation_receipts_hashes_ck" CHECK ( + "artifact_hash" ~ '^[a-f0-9]{64}$' + AND "request_fingerprint" ~ '^sha256:[a-f0-9]{64}$' + AND "response_fingerprint" ~ '^sha256:[a-f0-9]{64}$' + AND "prompt_response_fingerprint" ~ '^sha256:[a-f0-9]{64}$' + ), + CONSTRAINT "knowledge_node_generation_receipts_json_ck" + CHECK (jsonb_typeof("receipt") = 'object'), + CONSTRAINT "knowledge_node_generation_receipts_bytes_ck" + CHECK (octet_length("receipt"::text) <= 8388608), + CONSTRAINT "knowledge_node_generation_receipts_pub_gen_nonzero_ck" + CHECK ("publication_generation_id" <> '00000000-0000-0000-0000-000000000000'::uuid), + FOREIGN KEY ("knowledge_space_id") + REFERENCES "knowledge_spaces" ("id") ON DELETE CASCADE, + FOREIGN KEY ("document_asset_id") + REFERENCES "document_assets" ("id") ON DELETE CASCADE, + FOREIGN KEY ("parse_artifact_id") + REFERENCES "parse_artifacts" ("id") ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS "knowledge_node_generation_receipts_document_idx" + ON "knowledge_node_generation_receipts" + ("knowledge_space_id", "document_asset_id", "publication_generation_id", "parse_artifact_id"); diff --git a/knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.tidb.sql b/knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.tidb.sql new file mode 100644 index 00000000000..10ca6e9185c --- /dev/null +++ b/knowledge-fs/packages/database/migrations/0043_semantic_generation_receipts.tidb.sql @@ -0,0 +1,46 @@ +-- Knowledge Platform schema migration +-- Migration id: 0043_semantic_generation_receipts +-- Dialect: tidb + +CREATE TABLE IF NOT EXISTS `knowledge_node_generation_receipts` ( + `knowledge_space_id` CHAR(36) NOT NULL, + `publication_generation_id` CHAR(36) NOT NULL, + `parse_artifact_id` CHAR(36) NOT NULL, + `document_asset_id` CHAR(36) NOT NULL, + `artifact_hash` VARCHAR(64) NOT NULL, + `document_chunk_count` INT NOT NULL, + `stored_node_count` INT NOT NULL, + `request_fingerprint` VARCHAR(71) NOT NULL, + `response_fingerprint` VARCHAR(71) NOT NULL, + `prompt_response_fingerprint` VARCHAR(71) NOT NULL, + `receipt` JSON NOT NULL, + PRIMARY KEY (`knowledge_space_id`, `publication_generation_id`, `parse_artifact_id`), + CONSTRAINT `knowledge_node_generation_receipts_counts_ck` CHECK ( + `document_chunk_count` >= 0 AND `stored_node_count` >= 0 + AND `stored_node_count` <= `document_chunk_count` + ), + CONSTRAINT `knowledge_node_generation_receipts_hashes_ck` CHECK ( + `artifact_hash` REGEXP '^[a-f0-9]{64}$' + AND `request_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$' + AND `response_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$' + AND `prompt_response_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$' + ), + CONSTRAINT `knowledge_node_generation_receipts_json_ck` + CHECK (JSON_TYPE(`receipt`) = 'OBJECT'), + CONSTRAINT `knowledge_node_generation_receipts_bytes_ck` + CHECK (OCTET_LENGTH(CAST(`receipt` AS CHAR)) <= 8388608), + CONSTRAINT `knowledge_node_generation_receipts_pub_gen_nonzero_ck` CHECK ( + `publication_generation_id` REGEXP '^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$' + AND `publication_generation_id` <> '00000000-0000-0000-0000-000000000000' + ), + FOREIGN KEY (`knowledge_space_id`) + REFERENCES `knowledge_spaces` (`id`) ON DELETE CASCADE, + FOREIGN KEY (`document_asset_id`) + REFERENCES `document_assets` (`id`) ON DELETE CASCADE, + FOREIGN KEY (`parse_artifact_id`) + REFERENCES `parse_artifacts` (`id`) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS `knowledge_node_generation_receipts_document_idx` + ON `knowledge_node_generation_receipts` + (`knowledge_space_id`, `document_asset_id`, `publication_generation_id`, `parse_artifact_id`); diff --git a/knowledge-fs/packages/database/src/migration-artifacts.generated.ts b/knowledge-fs/packages/database/src/migration-artifacts.generated.ts index df3e79412c7..103995934b8 100644 --- a/knowledge-fs/packages/database/src/migration-artifacts.generated.ts +++ b/knowledge-fs/packages/database/src/migration-artifacts.generated.ts @@ -87,4 +87,6 @@ export const migrationArtifacts = [ { content: "-- Knowledge Platform schema migration\n-- Migration id: 0041_logical_document_availability\n-- Dialect: tidb\n-- Adds document-scoped availability without mutating source or physical index state.\n\nALTER TABLE `logical_documents`\n ADD COLUMN IF NOT EXISTS `enabled` BOOLEAN NOT NULL DEFAULT TRUE,\n ADD COLUMN IF NOT EXISTS `disabled_at` TIMESTAMP NULL,\n ADD COLUMN IF NOT EXISTS `disabled_by_subject_id` VARCHAR(255) NULL;\n\nALTER TABLE `logical_documents`\n ADD CONSTRAINT `logical_documents_availability_ck`\n CHECK (\n (`enabled` AND `disabled_at` IS NULL AND `disabled_by_subject_id` IS NULL)\n OR (NOT `enabled` AND `disabled_at` IS NOT NULL AND `disabled_by_subject_id` IS NOT NULL)\n );\n", path: "packages/database/migrations/0041_logical_document_availability.tidb.sql" }, { content: "-- Knowledge Platform schema migration\n-- Migration id: 0042_workflow_failed_retrieval_capture\n-- Dialect: postgres\n-- Workflow empty-retrieval events retain their admitted Capability provenance and frozen scope.\n\nALTER TABLE \"failed_queries\"\n ADD COLUMN IF NOT EXISTS \"capability_grant_id\" UUID;\n\nALTER TABLE \"failed_queries\"\n DROP CONSTRAINT IF EXISTS \"failed_queries_permission_binding_ck\";\n\nALTER TABLE \"failed_queries\"\n ADD CONSTRAINT \"failed_queries_permission_binding_ck\" CHECK (\n (\"tenant_id\" IS NULL AND \"capability_grant_id\" IS NULL\n AND \"requested_by_subject_id\" IS NULL AND \"access_channel\" IS NULL\n AND \"permission_snapshot_id\" IS NULL AND \"permission_snapshot_revision\" IS NULL\n AND \"required_permission_scope\" IS NULL AND \"revision\" IS NULL)\n OR (\"tenant_id\" IS NOT NULL AND \"capability_grant_id\" IS NOT NULL\n AND \"requested_by_subject_id\" IS NULL AND \"access_channel\" IS NULL\n AND \"permission_snapshot_id\" IS NULL AND \"permission_snapshot_revision\" IS NULL\n AND \"required_permission_scope\" IS NOT NULL\n AND jsonb_typeof(\"required_permission_scope\") = 'array'\n AND \"revision\" IS NOT NULL AND \"revision\" >= 1)\n OR (\"tenant_id\" IS NOT NULL AND \"capability_grant_id\" IS NULL\n AND \"requested_by_subject_id\" IS NOT NULL\n AND \"access_channel\" IS NOT NULL\n AND \"access_channel\" IN ('interactive', 'service_api', 'mcp', 'agent')\n AND \"permission_snapshot_id\" IS NOT NULL\n AND \"permission_snapshot_revision\" IS NOT NULL\n AND \"permission_snapshot_revision\" >= 1\n AND \"required_permission_scope\" IS NOT NULL\n AND jsonb_typeof(\"required_permission_scope\") = 'array'\n AND \"revision\" IS NOT NULL AND \"revision\" >= 1)\n );\n\nDO $kfs_0042_failed_query_capability_fk$\nBEGIN\n IF NOT EXISTS (\n SELECT 1 FROM pg_constraint\n WHERE conname = 'failed_queries_capability_grant_fk'\n AND conrelid = 'failed_queries'::regclass\n ) THEN\n ALTER TABLE \"failed_queries\"\n ADD CONSTRAINT \"failed_queries_capability_grant_fk\"\n FOREIGN KEY (\"tenant_id\", \"knowledge_space_id\", \"capability_grant_id\")\n REFERENCES \"capability_grants\" (\"tenant_id\", \"knowledge_space_id\", \"grant_id\")\n ON DELETE RESTRICT;\n END IF;\nEND\n$kfs_0042_failed_query_capability_fk$;\n\nCREATE INDEX IF NOT EXISTS \"failed_queries_capability_grant_idx\"\n ON \"failed_queries\" (\"tenant_id\", \"knowledge_space_id\", \"capability_grant_id\");\n", path: "packages/database/migrations/0042_workflow_failed_retrieval_capture.postgres.sql" }, { content: "-- Knowledge Platform schema migration\n-- Migration id: 0042_workflow_failed_retrieval_capture\n-- Dialect: tidb\n-- Workflow empty-retrieval events retain their admitted Capability provenance and frozen scope.\n\nALTER TABLE `failed_queries`\n ADD COLUMN IF NOT EXISTS `capability_grant_id` CHAR(36) NULL;\n\nSET @fq_0042_binding_ck_exists = (\n SELECT COUNT(*) FROM information_schema.tidb_check_constraints\n WHERE constraint_schema = DATABASE()\n AND table_name = 'failed_queries'\n AND constraint_name = 'failed_queries_permission_binding_ck'\n);\nSET @fq_0042_binding_ck_drop_sql = IF(\n @fq_0042_binding_ck_exists > 0,\n 'ALTER TABLE `failed_queries` DROP CONSTRAINT `failed_queries_permission_binding_ck`',\n 'SELECT 1'\n);\nPREPARE fq_0042_binding_ck_drop_stmt FROM @fq_0042_binding_ck_drop_sql;\nEXECUTE fq_0042_binding_ck_drop_stmt;\nDEALLOCATE PREPARE fq_0042_binding_ck_drop_stmt;\n\nALTER TABLE `failed_queries`\n MODIFY COLUMN `permission_binding_complete` TINYINT GENERATED ALWAYS AS (\n CASE WHEN\n (`tenant_id` IS NULL AND `capability_grant_id` IS NULL\n AND `requested_by_subject_id` IS NULL AND `access_channel` IS NULL\n AND `permission_snapshot_id` IS NULL AND `permission_snapshot_revision` IS NULL\n AND `required_permission_scope` IS NULL AND `revision` IS NULL)\n OR (`tenant_id` IS NOT NULL AND `capability_grant_id` IS NOT NULL\n AND `requested_by_subject_id` IS NULL AND `access_channel` IS NULL\n AND `permission_snapshot_id` IS NULL AND `permission_snapshot_revision` IS NULL\n AND `required_permission_scope` IS NOT NULL\n AND JSON_TYPE(`required_permission_scope`) = 'ARRAY'\n AND `revision` IS NOT NULL AND `revision` >= 1)\n OR (`tenant_id` IS NOT NULL AND `capability_grant_id` IS NULL\n AND `requested_by_subject_id` IS NOT NULL\n AND `access_channel` IN ('interactive', 'service_api', 'mcp', 'agent')\n AND `permission_snapshot_id` IS NOT NULL\n AND `permission_snapshot_revision` IS NOT NULL\n AND `permission_snapshot_revision` >= 1\n AND `required_permission_scope` IS NOT NULL\n AND JSON_TYPE(`required_permission_scope`) = 'ARRAY'\n AND `revision` IS NOT NULL AND `revision` >= 1)\n THEN 1 ELSE 0\n END\n ) VIRTUAL;\n\nALTER TABLE `failed_queries`\n ADD CONSTRAINT `failed_queries_permission_binding_ck`\n CHECK (`permission_binding_complete` = 1);\n\nSET @fq_0042_capability_fk_exists = (\n SELECT COUNT(*) FROM information_schema.table_constraints\n WHERE table_schema = DATABASE()\n AND table_name = 'failed_queries'\n AND constraint_name = 'failed_queries_capability_grant_fk'\n);\nSET @fq_0042_capability_fk_sql = IF(\n @fq_0042_capability_fk_exists = 0,\n 'ALTER TABLE `failed_queries` ADD CONSTRAINT `failed_queries_capability_grant_fk` FOREIGN KEY (`tenant_id`, `knowledge_space_id`, `capability_grant_id`) REFERENCES `capability_grants` (`tenant_id`, `knowledge_space_id`, `grant_id`) ON DELETE RESTRICT',\n 'SELECT 1'\n);\nPREPARE fq_0042_capability_fk_stmt FROM @fq_0042_capability_fk_sql;\nEXECUTE fq_0042_capability_fk_stmt;\nDEALLOCATE PREPARE fq_0042_capability_fk_stmt;\n\nCREATE INDEX IF NOT EXISTS `failed_queries_capability_grant_idx`\n ON `failed_queries` (`tenant_id`, `knowledge_space_id`, `capability_grant_id`);\n", path: "packages/database/migrations/0042_workflow_failed_retrieval_capture.tidb.sql" }, + { content: "-- Knowledge Platform schema migration\n-- Migration id: 0043_semantic_generation_receipts\n-- Dialect: postgres\n\nCREATE TABLE IF NOT EXISTS \"knowledge_node_generation_receipts\" (\n \"knowledge_space_id\" UUID NOT NULL,\n \"publication_generation_id\" UUID NOT NULL,\n \"parse_artifact_id\" UUID NOT NULL,\n \"document_asset_id\" UUID NOT NULL,\n \"artifact_hash\" VARCHAR(64) NOT NULL,\n \"document_chunk_count\" INTEGER NOT NULL,\n \"stored_node_count\" INTEGER NOT NULL,\n \"request_fingerprint\" VARCHAR(71) NOT NULL,\n \"response_fingerprint\" VARCHAR(71) NOT NULL,\n \"prompt_response_fingerprint\" VARCHAR(71) NOT NULL,\n \"receipt\" JSONB NOT NULL,\n PRIMARY KEY (\"knowledge_space_id\", \"publication_generation_id\", \"parse_artifact_id\"),\n CONSTRAINT \"knowledge_node_generation_receipts_counts_ck\" CHECK (\n \"document_chunk_count\" >= 0 AND \"stored_node_count\" >= 0\n AND \"stored_node_count\" <= \"document_chunk_count\"\n ),\n CONSTRAINT \"knowledge_node_generation_receipts_hashes_ck\" CHECK (\n \"artifact_hash\" ~ '^[a-f0-9]{64}$'\n AND \"request_fingerprint\" ~ '^sha256:[a-f0-9]{64}$'\n AND \"response_fingerprint\" ~ '^sha256:[a-f0-9]{64}$'\n AND \"prompt_response_fingerprint\" ~ '^sha256:[a-f0-9]{64}$'\n ),\n CONSTRAINT \"knowledge_node_generation_receipts_json_ck\"\n CHECK (jsonb_typeof(\"receipt\") = 'object'),\n CONSTRAINT \"knowledge_node_generation_receipts_bytes_ck\"\n CHECK (octet_length(\"receipt\"::text) <= 8388608),\n CONSTRAINT \"knowledge_node_generation_receipts_pub_gen_nonzero_ck\"\n CHECK (\"publication_generation_id\" <> '00000000-0000-0000-0000-000000000000'::uuid),\n FOREIGN KEY (\"knowledge_space_id\")\n REFERENCES \"knowledge_spaces\" (\"id\") ON DELETE CASCADE,\n FOREIGN KEY (\"document_asset_id\")\n REFERENCES \"document_assets\" (\"id\") ON DELETE CASCADE,\n FOREIGN KEY (\"parse_artifact_id\")\n REFERENCES \"parse_artifacts\" (\"id\") ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS \"knowledge_node_generation_receipts_document_idx\"\n ON \"knowledge_node_generation_receipts\"\n (\"knowledge_space_id\", \"document_asset_id\", \"publication_generation_id\", \"parse_artifact_id\");\n", path: "packages/database/migrations/0043_semantic_generation_receipts.postgres.sql" }, + { content: "-- Knowledge Platform schema migration\n-- Migration id: 0043_semantic_generation_receipts\n-- Dialect: tidb\n\nCREATE TABLE IF NOT EXISTS `knowledge_node_generation_receipts` (\n `knowledge_space_id` CHAR(36) NOT NULL,\n `publication_generation_id` CHAR(36) NOT NULL,\n `parse_artifact_id` CHAR(36) NOT NULL,\n `document_asset_id` CHAR(36) NOT NULL,\n `artifact_hash` VARCHAR(64) NOT NULL,\n `document_chunk_count` INT NOT NULL,\n `stored_node_count` INT NOT NULL,\n `request_fingerprint` VARCHAR(71) NOT NULL,\n `response_fingerprint` VARCHAR(71) NOT NULL,\n `prompt_response_fingerprint` VARCHAR(71) NOT NULL,\n `receipt` JSON NOT NULL,\n PRIMARY KEY (`knowledge_space_id`, `publication_generation_id`, `parse_artifact_id`),\n CONSTRAINT `knowledge_node_generation_receipts_counts_ck` CHECK (\n `document_chunk_count` >= 0 AND `stored_node_count` >= 0\n AND `stored_node_count` <= `document_chunk_count`\n ),\n CONSTRAINT `knowledge_node_generation_receipts_hashes_ck` CHECK (\n `artifact_hash` REGEXP '^[a-f0-9]{64}$'\n AND `request_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$'\n AND `response_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$'\n AND `prompt_response_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$'\n ),\n CONSTRAINT `knowledge_node_generation_receipts_json_ck`\n CHECK (JSON_TYPE(`receipt`) = 'OBJECT'),\n CONSTRAINT `knowledge_node_generation_receipts_bytes_ck`\n CHECK (OCTET_LENGTH(CAST(`receipt` AS CHAR)) <= 8388608),\n CONSTRAINT `knowledge_node_generation_receipts_pub_gen_nonzero_ck` CHECK (\n `publication_generation_id` REGEXP '^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'\n AND `publication_generation_id` <> '00000000-0000-0000-0000-000000000000'\n ),\n FOREIGN KEY (`knowledge_space_id`)\n REFERENCES `knowledge_spaces` (`id`) ON DELETE CASCADE,\n FOREIGN KEY (`document_asset_id`)\n REFERENCES `document_assets` (`id`) ON DELETE CASCADE,\n FOREIGN KEY (`parse_artifact_id`)\n REFERENCES `parse_artifacts` (`id`) ON DELETE CASCADE\n);\n\nCREATE INDEX IF NOT EXISTS `knowledge_node_generation_receipts_document_idx`\n ON `knowledge_node_generation_receipts`\n (`knowledge_space_id`, `document_asset_id`, `publication_generation_id`, `parse_artifact_id`);\n", path: "packages/database/migrations/0043_semantic_generation_receipts.tidb.sql" }, ] as const satisfies readonly MigrationArtifact[]; diff --git a/knowledge-fs/packages/database/src/migration-file.test.ts b/knowledge-fs/packages/database/src/migration-file.test.ts index 75186fc1279..ac5dcdc3faf 100644 --- a/knowledge-fs/packages/database/src/migration-file.test.ts +++ b/knowledge-fs/packages/database/src/migration-file.test.ts @@ -144,6 +144,8 @@ describe("migration file rendering", () => { "packages/database/migrations/0041_logical_document_availability.tidb.sql", "packages/database/migrations/0042_workflow_failed_retrieval_capture.postgres.sql", "packages/database/migrations/0042_workflow_failed_retrieval_capture.tidb.sql", + "packages/database/migrations/0043_semantic_generation_receipts.postgres.sql", + "packages/database/migrations/0043_semantic_generation_receipts.tidb.sql", ]); const workflowCapturePostgres = artifacts.find( (artifact) => @@ -892,6 +894,7 @@ describe("migration file rendering", () => { "packages/database/migrations/0040_knowledge_space_metadata.postgres.sql", "packages/database/migrations/0041_logical_document_availability.postgres.sql", "packages/database/migrations/0042_workflow_failed_retrieval_capture.postgres.sql", + "packages/database/migrations/0043_semantic_generation_receipts.postgres.sql", ]); expect( getPendingMigrationArtifacts({ @@ -938,6 +941,7 @@ describe("migration file rendering", () => { "0040_knowledge_space_metadata", "0041_logical_document_availability", "0042_workflow_failed_retrieval_capture", + "0043_semantic_generation_receipts", ], dialect: "postgres", }), diff --git a/knowledge-fs/packages/database/src/schema.test.ts b/knowledge-fs/packages/database/src/schema.test.ts index a303ae4feda..0647cd8c670 100644 --- a/knowledge-fs/packages/database/src/schema.test.ts +++ b/knowledge-fs/packages/database/src/schema.test.ts @@ -49,6 +49,7 @@ describe("database schema catalog", () => { "knowledge_fs_leases", "retrieval_execution_leases", "knowledge_nodes", + "knowledge_node_generation_receipts", "index_projections", "index_projection_fts_postings", "tidb_fts_posting_backfills", @@ -119,6 +120,39 @@ describe("database schema catalog", () => { ]); }); + it("models immutable semantic generation receipts with bounded JSON and cascade ownership", () => { + const schema = getDatabaseSchema(); + const table = findTable(schema, "knowledge_node_generation_receipts"); + + expect(table.primaryKey).toEqual([ + "knowledge_space_id", + "publication_generation_id", + "parse_artifact_id", + ]); + expect(table.checkConstraints?.map((constraint) => constraint.name)).toEqual( + expect.arrayContaining([ + "knowledge_node_generation_receipts_counts_ck", + "knowledge_node_generation_receipts_hashes_ck", + "knowledge_node_generation_receipts_json_ck", + "knowledge_node_generation_receipts_bytes_ck", + "knowledge_node_generation_receipts_pub_gen_nonzero_ck", + ]), + ); + expect(table.foreignKeys).toEqual( + expect.arrayContaining([ + expect.objectContaining({ referencedTable: "knowledge_spaces", onDelete: "CASCADE" }), + expect.objectContaining({ referencedTable: "document_assets", onDelete: "CASCADE" }), + expect.objectContaining({ referencedTable: "parse_artifacts", onDelete: "CASCADE" }), + ]), + ); + expect(findIndex(schema, "knowledge_node_generation_receipts_document_idx").columns).toEqual([ + "knowledge_space_id", + "document_asset_id", + "publication_generation_id", + "parse_artifact_id", + ]); + }); + it("models durable bulk task history with exact authorization provenance", () => { const schema = getDatabaseSchema(); const table = findTable(schema, "bulk_operations"); diff --git a/knowledge-fs/packages/database/src/schema.ts b/knowledge-fs/packages/database/src/schema.ts index a595331f542..ead327181ec 100644 --- a/knowledge-fs/packages/database/src/schema.ts +++ b/knowledge-fs/packages/database/src/schema.ts @@ -2306,6 +2306,80 @@ const tables = [ ), ], }, + { + name: "knowledge_node_generation_receipts", + checkConstraints: [ + publicationGenerationCheck( + "knowledge_node_generation_receipts_pub_gen_nonzero_ck", + "publication_generation_id", + false, + ), + { + expression: { + postgres: + '"document_chunk_count" >= 0 AND "stored_node_count" >= 0 AND "stored_node_count" <= "document_chunk_count"', + tidb: "`document_chunk_count` >= 0 AND `stored_node_count` >= 0 AND `stored_node_count` <= `document_chunk_count`", + }, + name: "knowledge_node_generation_receipts_counts_ck", + }, + { + expression: { + postgres: + "\"artifact_hash\" ~ '^[a-f0-9]{64}$' AND \"request_fingerprint\" ~ '^sha256:[a-f0-9]{64}$' AND \"response_fingerprint\" ~ '^sha256:[a-f0-9]{64}$' AND \"prompt_response_fingerprint\" ~ '^sha256:[a-f0-9]{64}$'", + tidb: "`artifact_hash` REGEXP '^[a-f0-9]{64}$' AND `request_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$' AND `response_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$' AND `prompt_response_fingerprint` REGEXP '^sha256:[a-f0-9]{64}$'", + }, + name: "knowledge_node_generation_receipts_hashes_ck", + }, + { + expression: { + postgres: "jsonb_typeof(\"receipt\") = 'object'", + tidb: "JSON_TYPE(`receipt`) = 'OBJECT'", + }, + name: "knowledge_node_generation_receipts_json_ck", + }, + { + expression: { + postgres: 'octet_length("receipt"::text) <= 8388608', + tidb: "OCTET_LENGTH(CAST(`receipt` AS CHAR)) <= 8388608", + }, + name: "knowledge_node_generation_receipts_bytes_ck", + }, + ], + foreignKeys: [ + { + columns: ["knowledge_space_id"], + onDelete: "CASCADE", + referencedColumns: ["id"], + referencedTable: "knowledge_spaces", + }, + { + columns: ["document_asset_id"], + onDelete: "CASCADE", + referencedColumns: ["id"], + referencedTable: "document_assets", + }, + { + columns: ["parse_artifact_id"], + onDelete: "CASCADE", + referencedColumns: ["id"], + referencedTable: "parse_artifacts", + }, + ], + columns: [ + idColumn("knowledge_space_id"), + idColumn("publication_generation_id"), + idColumn("parse_artifact_id"), + idColumn("document_asset_id"), + varcharColumn("artifact_hash", 64), + integerColumn("document_chunk_count"), + integerColumn("stored_node_count"), + varcharColumn("request_fingerprint", 71), + varcharColumn("response_fingerprint", 71), + varcharColumn("prompt_response_fingerprint", 71), + jsonColumn("receipt"), + ], + primaryKey: ["knowledge_space_id", "publication_generation_id", "parse_artifact_id"], + }, { name: "index_projections", checkConstraints: [ @@ -6746,6 +6820,18 @@ const indexes = [ postgres: "GIN", }, }, + { + columns: [ + "knowledge_space_id", + "document_asset_id", + "publication_generation_id", + "parse_artifact_id", + ], + name: "knowledge_node_generation_receipts_document_idx", + purpose: + "Delete immutable semantic-generation receipts for one tombstoned document without a table scan", + tableName: "knowledge_node_generation_receipts", + }, { columns: ["knowledge_space_id", "id", "version"], name: "document_assets_space_id_version_uq", diff --git a/knowledge-fs/pnpm-lock.yaml b/knowledge-fs/pnpm-lock.yaml index 1bf4a245b95..e6535d38184 100644 --- a/knowledge-fs/pnpm-lock.yaml +++ b/knowledge-fs/pnpm-lock.yaml @@ -172,6 +172,9 @@ importers: sharp: specifier: 0.35.3 version: 0.35.3(@types/node@22.19.18) + unicode-segmenter: + specifier: 0.15.0 + version: 0.15.0 zod: specifier: ^3.24.1 version: 3.25.76 diff --git a/knowledge-fs/scripts/semantic-compilation-rollout.mjs b/knowledge-fs/scripts/semantic-compilation-rollout.mjs new file mode 100644 index 00000000000..34362582958 --- /dev/null +++ b/knowledge-fs/scripts/semantic-compilation-rollout.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node +import { readFile } from "node:fs/promises"; + +const textDecoder = new TextDecoder(); +const mode = process.env.SEMANTIC_ROLLOUT_MODE?.trim() || "static"; +const supportedModes = new Set(["backfill", "canary", "preflight", "rollback", "static"]); +const mutatingModes = new Set(["backfill", "canary", "rollback"]); +const apiBase = normalizeBaseUrl(process.env.SEMANTIC_ROLLOUT_API_BASE ?? "http://127.0.0.1:8788"); +const token = process.env.SEMANTIC_ROLLOUT_AUTH_TOKEN?.trim() || "dev-token"; +const maxJsonBytes = positiveInteger( + process.env.SEMANTIC_ROLLOUT_MAX_JSON_BYTES ?? "1048576", + "SEMANTIC_ROLLOUT_MAX_JSON_BYTES", +); +const maxPolls = positiveInteger( + process.env.SEMANTIC_ROLLOUT_MAX_POLLS ?? "120", + "SEMANTIC_ROLLOUT_MAX_POLLS", +); +const pollIntervalMs = nonnegativeInteger( + process.env.SEMANTIC_ROLLOUT_POLL_INTERVAL_MS ?? "2000", + "SEMANTIC_ROLLOUT_POLL_INTERVAL_MS", +); + +if (!supportedModes.has(mode)) { + throw new Error(`Unsupported SEMANTIC_ROLLOUT_MODE=${mode}`); +} + +const staticEvidence = await verifyStaticEvidence(); +if (mode === "static") { + printResult({ mode, staticEvidence }); + process.exit(0); +} + +const knowledgeSpaceId = requiredUuid( + process.env.SEMANTIC_ROLLOUT_SPACE_ID, + "SEMANTIC_ROLLOUT_SPACE_ID", +); +if (mutatingModes.has(mode)) { + assertMutationConfirmation(mode, knowledgeSpaceId); +} + +const preflight = await runPreflight(knowledgeSpaceId); +if (mode === "preflight") { + printResult({ knowledgeSpaceId, mode, preflight, staticEvidence }); + process.exit(0); +} + +if (mode === "canary") { + const documentIds = requiredUuidList( + process.env.SEMANTIC_ROLLOUT_DOCUMENT_IDS, + "SEMANTIC_ROLLOUT_DOCUMENT_IDS", + ); + const result = await reindexDocuments({ documentIds, knowledgeSpaceId }); + await verifyOutlines(knowledgeSpaceId, documentIds); + const retrieval = await verifyRetrievalIfConfigured(knowledgeSpaceId); + printResult({ knowledgeSpaceId, mode, preflight, result, retrieval, staticEvidence }); + process.exit(0); +} + +if (mode === "backfill") { + const result = await reindexDocuments({ all: true, knowledgeSpaceId }); + const documentIds = result.items + .map((item) => item?.asset?.id) + .filter((value) => typeof value === "string"); + await verifyOutlines(knowledgeSpaceId, documentIds); + const retrieval = await verifyRetrievalIfConfigured(knowledgeSpaceId); + printResult({ knowledgeSpaceId, mode, preflight, result, retrieval, staticEvidence }); + process.exit(0); +} + +const rollback = await rollbackDocument(knowledgeSpaceId); +printResult({ knowledgeSpaceId, mode, preflight, rollback, staticEvidence }); + +async function verifyStaticEvidence() { + const expectedMigrationId = "0043_semantic_generation_receipts"; + const paths = [ + `packages/database/migrations/${expectedMigrationId}.postgres.sql`, + `packages/database/migrations/${expectedMigrationId}.tidb.sql`, + ]; + for (const path of paths) { + const source = await readFile(new URL(`../${path}`, import.meta.url), "utf8"); + if ( + !source.includes(expectedMigrationId) || + !source.includes("knowledge_node_generation_receipts") + ) { + throw new Error(`Semantic receipt migration evidence is incomplete in ${path}`); + } + } + const registry = await readFile( + new URL("../packages/database/src/migration-artifacts.generated.ts", import.meta.url), + "utf8", + ); + if (!registry.includes(expectedMigrationId)) { + throw new Error( + "Generated migration registry does not contain semantic receipt migration 0043", + ); + } + return { migrationId: expectedMigrationId, registry: "present" }; +} + +async function runPreflight(spaceId) { + const encoded = encodeURIComponent(spaceId); + const [health, settings, documents, tasks] = await Promise.all([ + requestJson("/health", { expectedStatus: 200, method: "GET" }), + requestJson(`/knowledge-spaces/${encoded}/settings`, { + expectedStatus: 200, + method: "GET", + }), + requestJson(`/knowledge-spaces/${encoded}/documents?limit=100`, { + expectedStatus: 200, + method: "GET", + }), + requestJson(`/knowledge-spaces/${encoded}/background-tasks?limit=50`, { + expectedStatus: 200, + method: "GET", + }), + ]); + if (!health || typeof health !== "object") throw new Error("KnowledgeFS health is invalid"); + if (health.components?.database === false || health.components?.objectStorage === false) { + throw new Error("KnowledgeFS durable dependencies are unhealthy"); + } + if (!settings || typeof settings !== "object") + throw new Error("KnowledgeFS settings are invalid"); + if (!Array.isArray(documents.items)) throw new Error("KnowledgeFS document list is invalid"); + if (!Array.isArray(tasks.items)) throw new Error("KnowledgeFS background-task list is invalid"); + const activeFailures = tasks.items.filter( + (task) => task?.state === "failed" && task?.operation === "document_reindex", + ); + return { + activeReindexFailures: activeFailures.length, + configurationState: settings.configurationState ?? settings.configuration_state ?? "unknown", + documentCount: documents.items.length, + healthComponents: health.components ?? {}, + }; +} + +async function reindexDocuments(input) { + const body = input.all ? { all: true } : { documentIds: input.documentIds }; + const result = await requestJson( + `/knowledge-spaces/${encodeURIComponent(input.knowledgeSpaceId)}/documents/bulk/reindex`, + { + body: JSON.stringify(body), + expectedStatus: 202, + headers: { "content-type": "application/json" }, + method: "POST", + }, + ); + if (!Array.isArray(result.items) || typeof result.bulkJobId !== "string") { + throw new Error("Bulk reindex response is invalid"); + } + const rejected = result.items.filter((item) => item?.status !== "queued"); + if (rejected.length > 0) { + throw new Error(`Bulk reindex rejected ${rejected.length} document(s)`); + } + for (const item of result.items) { + const statusUrl = requiredString(item.statusUrl, "bulk reindex statusUrl"); + await pollTask(statusUrl); + } + return { + bulkJobId: result.bulkJobId, + documentsQueued: result.items.length, + items: result.items, + }; +} + +async function rollbackDocument(spaceId) { + const documentId = requiredUuid( + process.env.SEMANTIC_ROLLOUT_ROLLBACK_DOCUMENT_ID, + "SEMANTIC_ROLLOUT_ROLLBACK_DOCUMENT_ID", + ); + const targetRevision = positiveInteger( + process.env.SEMANTIC_ROLLOUT_ROLLBACK_REVISION, + "SEMANTIC_ROLLOUT_ROLLBACK_REVISION", + ); + const encodedSpace = encodeURIComponent(spaceId); + const encodedDocument = encodeURIComponent(documentId); + const current = await requestJson( + `/knowledge-spaces/${encodedSpace}/logical-documents/${encodedDocument}`, + { expectedStatus: 200, method: "GET" }, + ); + const expectedActiveRevision = positiveInteger( + current.activeRevision, + "logical document activeRevision", + ); + const expectedRowVersion = nonnegativeInteger(current.rowVersion, "logical document rowVersion"); + const task = await requestJson( + `/knowledge-spaces/${encodedSpace}/documents/${encodedDocument}/revisions/${targetRevision}/rollback`, + { + body: JSON.stringify({ expectedActiveRevision, expectedRowVersion }), + expectedStatus: 202, + headers: { "content-type": "application/json" }, + method: "POST", + }, + ); + const taskId = requiredUuid(task.id, "rollback task id"); + await pollTask( + `/knowledge-spaces/${encodedSpace}/documents/${encodedDocument}/processing-tasks/${encodeURIComponent(taskId)}`, + ); + const restored = await requestJson( + `/knowledge-spaces/${encodedSpace}/logical-documents/${encodedDocument}`, + { expectedStatus: 200, method: "GET" }, + ); + if (restored.activeRevision !== targetRevision) { + throw new Error( + `Rollback completed without activating revision ${targetRevision}; observed ${String(restored.activeRevision)}`, + ); + } + return { documentId, fromRevision: expectedActiveRevision, taskId, toRevision: targetRevision }; +} + +async function verifyOutlines(spaceId, documentIds) { + for (const documentId of documentIds) { + const outline = await requestJson( + `/knowledge-spaces/${encodeURIComponent(spaceId)}/documents/${encodeURIComponent(documentId)}/outline`, + { expectedStatus: 200, method: "GET" }, + ); + if (!Array.isArray(outline.nodes) || outline.nodes.length === 0) { + throw new Error(`Semantic outline is empty for document ${documentId}`); + } + if ( + outline.nodes.some( + (node) => !Array.isArray(node?.sectionPath) || !Array.isArray(node?.sourceNodeIds), + ) + ) { + throw new Error(`Semantic outline provenance is incomplete for document ${documentId}`); + } + } +} + +async function verifyRetrievalIfConfigured(spaceId) { + const query = process.env.SEMANTIC_ROLLOUT_QUERY?.trim(); + if (!query) return { checked: false }; + const result = await requestJson( + `/knowledge-spaces/${encodeURIComponent(spaceId)}/retrieval-tests`, + { + body: JSON.stringify({ includeText: true, mode: "research", query }), + expectedStatus: 200, + headers: { "content-type": "application/json" }, + method: "POST", + }, + ); + if (!Array.isArray(result.items)) throw new Error("Retrieval verification response is invalid"); + const minimumItems = nonnegativeInteger( + process.env.SEMANTIC_ROLLOUT_MIN_RETRIEVAL_ITEMS ?? "1", + "SEMANTIC_ROLLOUT_MIN_RETRIEVAL_ITEMS", + ); + if (result.items.length < minimumItems) { + throw new Error( + `Retrieval verification returned ${result.items.length} item(s); expected at least ${minimumItems}`, + ); + } + return { checked: true, itemCount: result.items.length, mode: result.mode ?? "research" }; +} + +async function pollTask(path) { + for (let poll = 1; poll <= maxPolls; poll += 1) { + const task = await requestJson(path, { expectedStatus: 200, method: "GET" }); + const terminalState = task.state ?? task.stage; + if (["completed", "published", "ready", "smoke_eval_passed"].includes(terminalState)) { + return task; + } + if (["canceled", "failed"].includes(terminalState)) { + throw new Error(`Rollout task ${path} ended in ${terminalState}: ${task.errorMessage ?? ""}`); + } + if (poll < maxPolls) await delay(pollIntervalMs); + } + throw new Error(`Rollout task ${path} exceeded SEMANTIC_ROLLOUT_MAX_POLLS=${maxPolls}`); +} + +async function requestJson(path, options) { + const response = await fetch(new URL(path, apiBase), { + body: options.body, + headers: { authorization: `Bearer ${token}`, ...(options.headers ?? {}) }, + method: options.method, + }); + const payload = await readBoundedJson(response); + const expected = Array.isArray(options.expectedStatus) + ? options.expectedStatus + : [options.expectedStatus]; + if (!expected.includes(response.status)) { + throw new Error( + `${options.method} ${path} returned ${response.status}: ${JSON.stringify(payload)}`, + ); + } + return payload; +} + +async function readBoundedJson(response) { + if (!response.body) return {}; + const reader = response.body.getReader(); + const chunks = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > maxJsonBytes) { + throw new Error( + `Rollout response exceeded SEMANTIC_ROLLOUT_MAX_JSON_BYTES=${maxJsonBytes}`, + ); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + const text = textDecoder.decode(bytes); + return text ? JSON.parse(text) : {}; +} + +function assertMutationConfirmation(selectedMode, spaceId) { + if (process.env.SEMANTIC_ROLLOUT_APPLY !== "1") { + throw new Error(`SEMANTIC_ROLLOUT_APPLY=1 is required for ${selectedMode}`); + } + const expected = `semantic:${selectedMode}:${spaceId}`; + if (process.env.SEMANTIC_ROLLOUT_CONFIRM !== expected) { + throw new Error(`SEMANTIC_ROLLOUT_CONFIRM must equal ${expected}`); + } +} + +function requiredUuid(value, name) { + const normalized = requiredString(value, name); + if ( + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu.test(normalized) + ) { + throw new Error(`${name} must be a UUID`); + } + return normalized; +} + +function requiredUuidList(value, name) { + const items = requiredString(value, name) + .split(",") + .map((item) => requiredUuid(item.trim(), name)); + return [...new Set(items)]; +} + +function requiredString(value, name) { + if (typeof value !== "string" || !value.trim()) throw new Error(`${name} is required`); + return value.trim(); +} + +function positiveInteger(value, name) { + const parsed = typeof value === "number" ? value : Number.parseInt(value ?? "", 10); + if (!Number.isSafeInteger(parsed) || parsed < 1) throw new Error(`${name} must be at least 1`); + return parsed; +} + +function nonnegativeInteger(value, name) { + const parsed = typeof value === "number" ? value : Number.parseInt(value ?? "", 10); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new Error(`${name} must be a non-negative integer`); + } + return parsed; +} + +function normalizeBaseUrl(value) { + const normalized = requiredString(value, "SEMANTIC_ROLLOUT_API_BASE"); + return normalized.endsWith("/") ? normalized : `${normalized}/`; +} + +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function printResult(value) { + console.log(JSON.stringify(value, null, 2)); +} diff --git a/knowledge-fs/scripts/semantic-compilation-rollout.test.mjs b/knowledge-fs/scripts/semantic-compilation-rollout.test.mjs new file mode 100644 index 00000000000..fdf05c5ec09 --- /dev/null +++ b/knowledge-fs/scripts/semantic-compilation-rollout.test.mjs @@ -0,0 +1,186 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { test } from "node:test"; + +const rootPackage = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8")); +const scriptUrl = new URL("./semantic-compilation-rollout.mjs", import.meta.url); +const spaceId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c40"; +const documentId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c41"; + +test("static rollout evidence is safe and includes migration 0043", async () => { + const result = await runScript({ SEMANTIC_ROLLOUT_MODE: "static" }); + assert.equal(result.code, 0, result.stderr); + const payload = JSON.parse(result.stdout); + assert.equal(payload.mode, "static"); + assert.equal(payload.staticEvidence.migrationId, "0043_semantic_generation_receipts"); +}); + +test("mutating rollout modes require an exact space-scoped confirmation", async () => { + const result = await runScript({ + SEMANTIC_ROLLOUT_DOCUMENT_IDS: documentId, + SEMANTIC_ROLLOUT_MODE: "canary", + SEMANTIC_ROLLOUT_SPACE_ID: spaceId, + }); + assert.notEqual(result.code, 0); + assert.match(result.stderr, /SEMANTIC_ROLLOUT_APPLY=1 is required/u); +}); + +test("preflight is read-only, bounded, and reports durable state", async (context) => { + const requests = []; + const server = createServer((request, response) => { + requests.push({ method: request.method, url: request.url }); + response.setHeader("content-type", "application/json"); + if (request.url === "/health") { + response.end(JSON.stringify({ components: { database: true, objectStorage: true } })); + return; + } + if (request.url?.endsWith("/settings")) { + response.end(JSON.stringify({ configurationState: "active" })); + return; + } + if (request.url?.includes("/documents?")) { + response.end(JSON.stringify({ items: [{ id: documentId }] })); + return; + } + if (request.url?.includes("/background-tasks?")) { + response.end(JSON.stringify({ items: [] })); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + context.after(() => new Promise((resolve) => server.close(resolve))); + const address = server.address(); + assert(address && typeof address === "object"); + + const result = await runScript({ + SEMANTIC_ROLLOUT_API_BASE: `http://127.0.0.1:${address.port}`, + SEMANTIC_ROLLOUT_MODE: "preflight", + SEMANTIC_ROLLOUT_SPACE_ID: spaceId, + }); + assert.equal(result.code, 0, result.stderr); + const payload = JSON.parse(result.stdout); + assert.equal(payload.preflight.configurationState, "active"); + assert.equal(payload.preflight.documentCount, 1); + assert.deepEqual( + requests.map((request) => request.method), + ["GET", "GET", "GET", "GET"], + ); +}); + +test("canary reindexes only explicit documents and verifies the published outline", async (context) => { + const requests = []; + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + requests.push({ body, method: request.method, url: request.url }); + response.setHeader("content-type", "application/json"); + if (request.url === "/health") { + response.end(JSON.stringify({ components: { database: true, objectStorage: true } })); + return; + } + if (request.url?.endsWith("/settings")) { + response.end(JSON.stringify({ configurationState: "active" })); + return; + } + if (request.url?.includes("/documents?")) { + response.end(JSON.stringify({ items: [{ id: documentId }] })); + return; + } + if (request.url?.includes("/background-tasks?")) { + response.end(JSON.stringify({ items: [] })); + return; + } + if (request.method === "POST" && request.url?.endsWith("/documents/bulk/reindex")) { + response.statusCode = 202; + response.end( + JSON.stringify({ + bulkJobId: "bulk-canary", + items: [ + { + asset: { id: documentId }, + status: "queued", + statusUrl: "/jobs/canary", + }, + ], + }), + ); + return; + } + if (request.url === "/jobs/canary") { + response.end(JSON.stringify({ stage: "published" })); + return; + } + if (request.url?.endsWith(`/documents/${documentId}/outline`)) { + response.end( + JSON.stringify({ + nodes: [{ sectionPath: ["Canary"], sourceNodeIds: ["node-1"] }], + }), + ); + return; + } + response.statusCode = 404; + response.end(JSON.stringify({ error: "not found" })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + context.after(() => new Promise((resolve) => server.close(resolve))); + const address = server.address(); + assert(address && typeof address === "object"); + + const result = await runScript({ + SEMANTIC_ROLLOUT_API_BASE: `http://127.0.0.1:${address.port}`, + SEMANTIC_ROLLOUT_APPLY: "1", + SEMANTIC_ROLLOUT_CONFIRM: `semantic:canary:${spaceId}`, + SEMANTIC_ROLLOUT_DOCUMENT_IDS: documentId, + SEMANTIC_ROLLOUT_MAX_POLLS: "1", + SEMANTIC_ROLLOUT_MODE: "canary", + SEMANTIC_ROLLOUT_SPACE_ID: spaceId, + }); + assert.equal(result.code, 0, result.stderr); + const payload = JSON.parse(result.stdout); + assert.equal(payload.result.documentsQueued, 1); + const mutation = requests.find((request) => request.method === "POST"); + assert.deepEqual(JSON.parse(mutation?.body ?? "{}"), { documentIds: [documentId] }); +}); + +test("package scripts expose explicit rollout phases and keep their tests in check", () => { + assert.equal( + rootPackage.scripts["semantic:rollout:static"], + "node scripts/semantic-compilation-rollout.mjs", + ); + assert.match( + rootPackage.scripts["semantic:rollout:preflight"], + /SEMANTIC_ROLLOUT_MODE=preflight/u, + ); + assert.match(rootPackage.scripts["semantic:rollout:canary"], /SEMANTIC_ROLLOUT_MODE=canary/u); + assert.match(rootPackage.scripts["semantic:rollout:backfill"], /SEMANTIC_ROLLOUT_MODE=backfill/u); + assert.match(rootPackage.scripts["semantic:rollout:rollback"], /SEMANTIC_ROLLOUT_MODE=rollback/u); + assert.equal( + rootPackage.scripts["semantic:rollout:test"], + "node --test scripts/semantic-compilation-rollout.test.mjs", + ); + assert.match(rootPackage.scripts.check, /semantic:rollout:test/u); +}); + +function runScript(extraEnv) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [scriptUrl.pathname], { + env: { ...process.env, ...extraEnv }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("close", (code) => resolve({ code, stderr, stdout })); + }); +}