diff --git a/.github/workflows/knowledge-fs-ci.yml b/.github/workflows/knowledge-fs-ci.yml index e349990f6a0..532f0c41690 100644 --- a/.github/workflows/knowledge-fs-ci.yml +++ b/.github/workflows/knowledge-fs-ci.yml @@ -106,6 +106,7 @@ jobs: - 'docker/envs/core-services/knowledge-fs.env.example' - 'docker/envs/core-services/knowledge-fs-unstructured-service.defaults' - 'docker/envs/core-services/knowledge-fs-unstructured.env.example' + - 'docker/knowledge-fs-unstructured-sandbox.compose.yaml' - 'docker/generate_docker_compose' - 'docs/design/knowledge-fs*' - '.github/dependabot.yml' @@ -195,6 +196,12 @@ jobs: - name: Install KnowledgeFS dependencies run: pnpm install --frozen-lockfile + - name: Install PDF metadata inspector + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends poppler-utils + pdfinfo -v + - name: Scan KnowledgeFS secrets run: pnpm security:secrets @@ -227,6 +234,12 @@ jobs: working-directory: . run: uv sync --project api --locked --dev + - name: Test dedicated parser sandbox + working-directory: . + env: + PYTHONPATH: knowledge-fs/services/unstructured-sandbox + run: uv run --project api python -m unittest discover -s knowledge-fs/services/unstructured-sandbox/tests -t knowledge-fs/services/unstructured-sandbox -v + - name: Collect Dify KnowledgeFS gate targets working-directory: . run: | diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 75e6dbae43b..32c6a32cbc8 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,7 +1,7 @@ { "schemaVersion": 5, - "subtreeTree": "d8b34a54bfe2744c76c84fc04ddc137d98a886d5", - "openapiSha256": "eba6f0e32eb27fd68ac20021c46b5f647217005fdb85522e7e4de6f0a1afc9a8", + "subtreeTree": "b7565ce4a39cd86563d41848bfb4d0672ca0a67b", + "openapiSha256": "f8cd6ad1e8ca2e62ceea2fc8a6f80f241135e23a2064d3a5f4ab45e25baf00b1", "capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", "productOperationManifestSha256": "0b472db40cfc89ca16127db9a334891bb291b4f07e77128e5ec7521ae581a3a7", diff --git a/docker/README.md b/docker/README.md index c6a3de9d904..0cba9208e7f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -69,7 +69,13 @@ defaults. The optional `knowledge-fs-unstructured` profile starts one isolated parser service named `knowledge_fs_unstructured` for every KnowledgeFS remote format. Its tracked `knowledge-fs-unstructured-service.defaults` additionally enables bounded page parallelism for -PDFs; an optional copied `knowledge-fs-unstructured.env` can override those service values. The +PDFs and a 25-million-pixel pre-allocation limit per page at the pinned 350 DPI; an optional copied +`knowledge-fs-unstructured.env` can override those service values. Do not raise the DPI or pixel +limit, or disable the guard: the KnowledgeFS client estimates page rasters at 350 DPI and rejects +oversized or unverifiable page geometry before sending it to the parser. Oversized pages need +smaller page dimensions or tiling before import; reducing compressed file size alone does not +help. Existing installations need the new KnowledgeFS image and a recreated parser service to +activate both guards. The existing `unstructured` profile remains unchanged for Dify's legacy ETL, so KnowledgeFS tuning cannot alter its PDF or Office parsing behavior. The copied `knowledge-fs.env` pairs every PDF and structurally/byte-heavy remote document with the longer diff --git a/docker/envs/core-services/knowledge-fs-unstructured-service.defaults b/docker/envs/core-services/knowledge-fs-unstructured-service.defaults index eadcebdc4ee..411f024ba02 100644 --- a/docker/envs/core-services/knowledge-fs-unstructured-service.defaults +++ b/docker/envs/core-services/knowledge-fs-unstructured-service.defaults @@ -2,6 +2,10 @@ # PDFs; the same service also handles standard and heavy Office, mail, EPUB, ODT, and RTF inputs. # Operator values in knowledge-fs-unstructured.env are loaded afterwards and take precedence. # The non-.env suffix is intentional so this required file remains tracked in clean checkouts. +# Keep the pinned image's 350 DPI output and align its pre-allocation guard with the client. +# Raising DPI above 350 invalidates the client's raster-size estimate; do not disable the guard. +PDF_RENDER_DPI=350 +PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000 UNSTRUCTURED_PARALLEL_MODE_ENABLED=true UNSTRUCTURED_PARALLEL_MODE_URL=http://127.0.0.1:8000/general/v0/general UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE=6 diff --git a/docker/envs/core-services/knowledge-fs-unstructured.env.example b/docker/envs/core-services/knowledge-fs-unstructured.env.example index 9177715b5b9..8b6833b7ced 100644 --- a/docker/envs/core-services/knowledge-fs-unstructured.env.example +++ b/docker/envs/core-services/knowledge-fs-unstructured.env.example @@ -8,7 +8,13 @@ # # KnowledgeFS owns durable attempt retries. Keep the child retry loop disabled so one parser # failure cannot multiply retries at both layers. +# PDF geometry, not compressed bytes, determines raster memory. The client preflights at 350 DPI; +# do not raise PDF_RENDER_DPI above 350 or raise/disable the 25-million-pixel per-page guard. +# These are provider process settings: this pinned HTTP API does not accept a pdf_image_dpi +# request parameter. Oversized pages must be resized or tiled before import. # ------------------------------------------------------------------ +PDF_RENDER_DPI=350 +PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000 UNSTRUCTURED_PARALLEL_MODE_ENABLED=true UNSTRUCTURED_PARALLEL_MODE_URL=http://127.0.0.1:8000/general/v0/general UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE=6 diff --git a/docker/envs/core-services/knowledge-fs.env.example b/docker/envs/core-services/knowledge-fs.env.example index be86ba8620c..b3479ef68a6 100644 --- a/docker/envs/core-services/knowledge-fs.env.example +++ b/docker/envs/core-services/knowledge-fs.env.example @@ -104,6 +104,9 @@ KNOWLEDGE_QUERY_IMAGE_EXPANSION_TIMEOUT_MS=8000 # The bundled profile uses this internal service name. Override it with an external # Unstructured-compatible endpoint when the deployment owns that provider instead. UNSTRUCTURED_API_URL=http://knowledge_fs_unstructured:8000 +# Change with parser image/model/output-policy upgrades to invalidate raw parse checkpoints. +# Leave empty for an unknown/custom backend; do not reuse a revision across semantic changes. +UNSTRUCTURED_BACKEND_REVISION= UNSTRUCTURED_API_KEY= # This is the process-wide ceiling shared by all remote parser requests. The nested heavy limit # covers every PDF plus structurally/byte-heavy Office, email, EPUB, ODT, and RTF request. diff --git a/docker/knowledge-fs-unstructured-sandbox.compose.yaml b/docker/knowledge-fs-unstructured-sandbox.compose.yaml new file mode 100644 index 00000000000..7310f0350ce --- /dev/null +++ b/docker/knowledge-fs-unstructured-sandbox.compose.yaml @@ -0,0 +1,35 @@ +# Opt-in only. Run the pinned-image golden gate before applying to production. +# Merge with docker-compose.yaml; this replaces the EXISTING dedicated service. +services: + knowledge_fs_unstructured: + build: + context: ../knowledge-fs/services/unstructured-sandbox + dockerfile: Dockerfile + image: knowledge-fs-unstructured-sandbox:local + init: true + read_only: true + pids_limit: 192 + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:rw,nosuid,nodev,size=1073741824,mode=1777 + # Override the original mount by target, keeping parser outputs off host storage. + volumes: + - type: bind + source: ./volumes/knowledge-fs-unstructured + target: /app/data + read_only: true + environment: + UNSTRUCTURED_PARALLEL_MODE_ENABLED: "false" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthcheck', timeout=3).read(4096)"] + interval: 30s + timeout: 5s + retries: 3 + deploy: + resources: + limits: + cpus: "4.0" + memory: 6G + pids: 192 diff --git a/knowledge-fs/.harness/changes/2026-09-05-incremental-structured-admission.md b/knowledge-fs/.harness/changes/2026-09-05-incremental-structured-admission.md new file mode 100644 index 00000000000..473a790bee6 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-incremental-structured-admission.md @@ -0,0 +1,18 @@ +# Incremental structured-text admission + +## Changes and rationale + +- JSONL scans one line slice at a time instead of allocating a document-wide `split()` array. Its existing record bound now stops the scan before the tail is materialized; scalar/null/array records, exact numeric lexemes, blank lines and CRLF behavior are preserved. +- Native XML receives a SAX-only structure admission pass before the authoritative `fast-xml-parser` object projection. Tag depth, element/attribute/text nodes and processing instructions are bounded without building a DOM, with 64 KiB cooperative scan chunks. Existing post-projection byte/node limits remain defense in depth. No new dependency or XML semantic projection change is introduced. +- CSV already used `csv-parse`'s `on_record` callback, kept no duplicate parser result array, and stopped when `maxRows` was exceeded. A regression verifies that this limit wins before an invalid later record; no unnecessary CSV rewrite was made. + +## Verification + +- New helper tests were introduced before implementation. Integration verifies over-deep XML never invokes `XMLParser.parse`; line/row guards preserve values and prevent later invalid records being decoded. Comments, CDATA, namespaces, self-closing elements and chunk-boundary text are covered. +- Full parser package before the final indentation regression: 520 passed (including contemporaneous sandbox protocol tests). Coverage: 97.13% statements/lines, 91.48% branches, 97.97% functions; new SAX/line helper is 100% lines/functions and 92.3% branches. +- The final XML regression verifies indentation-only whitespace does not consume semantic nodes, matching the authoritative parser's trimming behavior. RED→GREEN confirmed; the focused structural-admission suite passed all 16 tests afterward. +- Parser typecheck and Biome formatting passed. + +## Limits + +Input bytes are still loaded through the existing explicit upload/parser byte caps; this change is bounded structural/record admission, not a claim of arbitrary-size file streaming. Process isolation provides external interruption of synchronous library work. SAX admission is not a substitute for the authoritative projection or the post-projection expansion limits. diff --git a/knowledge-fs/.harness/changes/2026-09-05-isolated-parser-boundary-review.md b/knowledge-fs/.harness/changes/2026-09-05-isolated-parser-boundary-review.md new file mode 100644 index 00000000000..3fa87578ca0 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-isolated-parser-boundary-review.md @@ -0,0 +1,23 @@ +# Independent isolated parser lifecycle review + +Security, resource accounting and failure handling were reviewed separately from the initial process-isolation implementation. + +## Fixes + +- Shared executor validates supplied byte counts as finite, non-negative safe integers before reserving input memory; `NaN` or negative counts cannot corrupt admission accounting. +- A worker message followed by nonzero exit or signal termination is rejected. Receiving a result is not proof that the isolated operation completed cleanly. +- Cancellation preserves all legal abort reasons, including falsy `null`, `false` and `0`, instead of confusing them with absence of failure. +- Native IPC responses validate the success artifact against the existing core contract and validate error packet shape/bounds, producing a typed terminal response failure on malformed output. +- The internal native-isolation wrapper accepts only serializable options; function-valued callback options fail up front rather than throwing during IPC. Standalone native parser factories retain their original callback API. + +## Verification + +- Fourteen new regression tests were written and failed before these fixes (one existing Infinity-over-limit path already rejected correctly). +- New boundary tests plus existing real-child and lifecycle tests: 28 passed. +- API app TypeScript check passed. Biome check/format passed for the four touched implementation/test files. + +## Reviewed trade-offs + +- Response contract validation happens after the child has enforced its 32 MiB serialized output limit; this costs a bounded parent-side validation/copy, accepted for boundary correctness. +- A 256 MiB V8 heap limit is not an OS RSS sandbox. Child process termination, input/output byte caps, and image pixel limits are distinct protections and must be described as such. No claim of a hard per-child native-memory ceiling is made. +- Worker queue slots are still held until actual child close. No early abort release or remote retry behavior was introduced. diff --git a/knowledge-fs/.harness/changes/2026-09-05-multimodal-budget-completeness.md b/knowledge-fs/.harness/changes/2026-09-05-multimodal-budget-completeness.md new file mode 100644 index 00000000000..6cd8c9b199f --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-multimodal-budget-completeness.md @@ -0,0 +1,27 @@ +# Multimodal compilation budgets, capability parity, and isolated image analysis + +## Changes and rationale + +- Capture an immutable `document-media-v1` execution plan. Explicit text-only capabilities now disable extraction, rasterization, and visual indexing while preserving original element/source references. Both synchronous upload/source compilation and durable jobs use the same parser-hint builder. The synchronous path resolves two tenant/space-scoped profile heads concurrently and freezes their capability snapshots before parsing; durable jobs retain attempt-frozen profiles. The synchronous PDF path now preserves the same external-raster/provider-fallback behavior as durable compilation. +- Remote media fetches are sequential and document-bounded: 100 unique URL attempts, 32 MiB aggregate delivered/reserved network bytes, 60-second absolute deadline, 10 MiB per image. Failed/null attempts count; unknown consumed bytes remain conservatively reserved. Identical URL fetches are deduplicated without collapsing element ids, captions, or source-URI hashes. Parent cancellation propagates; transient transport/storage failures preserve existing retry semantics instead of silently completing a document. +- All newly materialized original and generated image bytes share a 64 MiB document budget. Image decoding reserves at most 20 MP per image and 100 MP per document (unknown headers reserve the per-image ceiling); variant generation also has a 60-second document deadline. Optional media omitted by a limit or terminal decoder failure leave originals available and persist bounded `parseCoverage.media` reasons and `multimodalAssets` counters. Existing provider/archive completeness is preserved, never promoted to complete without evidence. +- Allowlisted local images require both lexical and canonical (`realpath`) containment. Roots resolve once per document. Opens use `O_NOFOLLOW`, reads are bounded through the same file descriptor, and size changes during reading are rejected. The default empty allowlist still prohibits local reads. +- Production image variants run in a one-request Node child: two concurrent children, eight queued requests, 64 MiB reserved input, 50 MiB per input (matching the existing local-file ceiling), 128 MiB V8 heap, 384 MiB sampled Linux RSS guard, and 30-second worker deadline. Parent cancellation/resource termination releases admission only after process close through the shared executor. IPC responses validate shape, names, image bytes, dimensions, two-variant maximum, and 16 MiB aggregate output. Worker execution diagnostics (input/output bytes, elapsed time, peak RSS) persist on generated variant references. +- The default preview remains 320 px; a separate bounded 2048 px `analysis` variant supplies vision embedding, enrichment/OCR, and answer models. Legacy objects without an analysis variant fall back to the original, not the preview. New assets rejected by count/byte/pixel/decode/deadline limits carry a bounded `analysisUnavailable.reason` marker: all image-model consumers skip them without dropping original download/preview references or text indexing. The optional marker survives manifest, candidate, node, and descriptor projection, and enrichment cannot clear it or claim successful visual embedding. Overlong skipped inline data URIs remain recoverable in the parse artifact but become a hash and parse-element reference in manifests, preventing a secondary URI-schema failure or duplicate inline payload. Explicit preferred-variant overrides are preserved. Analysis is generated first; the thumbnail is derived from that bounded bitmap. No extra environment variables or database migration are required. +- Malformed image-worker IPC and unknown worker/dependency failures remain operational errors; only known input/pixel/output-limit rejections become partial media. This prevents a broken deployment from silently publishing documents with omitted images. + +## Verification + +- TDD reproduced missing failed-attempt limits, duplicate downloads, aggregate-byte bypass, hung fetches, ignored cancellation, local symlink escape, ignored profile flags, missing synchronous PDF fallback, thumbnail-as-model-input, derived-byte budget bypass, missing worker execution telemetry, and malformed IPC acceptance before their corresponding fixes. +- Targeted API verification: 115 tests passed across media budgets/extraction, immutable plans, image variants, synchronous/durable compilation, and answer-provider tests. Four focused behavior modules reached 99.78% lines/statements, 92.08% branches, and 100% functions. +- Targeted API-app verification: 62 tests passed across actual child-process decoding, IPC validation, multimodal configuration, embedding/enrichment/answer wiring. Two final validation guard regressions were separately rerun after implementation. +- API and API-app TypeScript checks passed. Repository-wide check/build/lint and Docker bundle verification are owned by the parent completion pass; no production deploy or commit was performed here. +- Final safety follow-up: TDD reproduced rejected-original model fallback, marker loss across manifest/candidate projection, malformed IPC misclassification, and overlong skipped inline-URI manifest failure. The final focused API pass includes 113 tests across 12 files (99.06% lines, 92.7% branches, 100% functions for extractor/manifest/candidate modules); image protocol, actual child, and enrichment checks include 46 passing app tests. Existing unmarked artifacts remain compatible. + +## Risks and scope + +- A sampled RSS guard is not a kernel per-process memory cgroup; whole-container limits remain the final memory boundary. Image inputs retain byte/pixel guards before native decoding. PDF crop/thumbnail work on bounded Poppler-generated PNGs remains in the existing rasterizer, not the untrusted-codec child. +- One child per image adds startup overhead (the local two-process decoding regression took roughly 0.3 seconds). It avoids decoder state retention; do not enable an unbounded worker pool to reduce this overhead. Real representative throughput/golden-set evaluation remains part of the parent evaluation pass. +- Custom isolated preview variant names must be 1–64 ASCII letters, digits, hyphens, or underscores and are rejected at startup otherwise. Native generator-only callers retain their existing API. Analysis dimensions are bounded to 4096; normal defaults remain 320/2048. +- Preserved references and explicit partial status do not make unsupported SVG/EMF/WMF/SmartArt resources newly renderable. Archive diagnostics report those separately. No claim is made that an unknown provider text/table extraction is complete. +- Legacy synchronous ingestion freezes only the media capability decision here; its later embedding resolver can still observe a changed manifest. Atomic model/profile publication and retry fencing remain guarantees of the durable path, not the legacy synchronous compiler. The upload HTTP abort signal now reaches parsing/materialization; legacy source materializer calls have no request signal in their existing interface. diff --git a/knowledge-fs/.harness/changes/2026-09-05-native-markup-content-fidelity.md b/knowledge-fs/.harness/changes/2026-09-05-native-markup-content-fidelity.md new file mode 100644 index 00000000000..44c5956b84f --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-native-markup-content-fidelity.md @@ -0,0 +1,55 @@ +# Preserve native Markdown and HTML content while bounding traversal + +## Problem + +Markdown blockquotes and ordinary HTML blocks were omitted. A paragraph beginning +with an image lost its remaining text, while other mixed paragraphs duplicated +the image syntax and placed all images before the paragraph. HTML traversal +ignored bare container text, returned before finding images in paragraphs/lists, +and selected only the first figure image. Deep HTML could overflow recursive +traversal before a useful input error was returned. + +## Changes + +- Visit Markdown blockquotes in document order, sharing section context. Keep + ordinary paragraph/list/table projections; mixed Markdown image/HTML blocks use + an inert HTML tree to preserve surrounding text and image placement. +- Keep static HTML text for both Markdown and MDX. No JSX evaluation, script + execution or URL fetch occurs in this parser. Exclude script/style/noscript + subtrees from text and image discovery, including nested table/heading paths. +- Preserve bare HTML text and ordered inline runs, paragraph/list images, all + figure images and captions, plus images associated with headings/tables. + Tables retain the existing normalized matrix element followed by their image + assets; this is not new cell-level image-anchor support. +- Preserve per-placement captions/titles when the same Markdown URI is repeated. +- Apply the shared maximum DOM depth (128) and node count (250,000) before + projection; classify violations as non-retryable `ParserResourceLimitError`. + Budget inspection, title search, image search and text collection use explicit + stacks. Block projection runs only after that bounded-depth inspection. +- Update the existing regression that deliberately expected ordinary Markdown + HTML content to be discarded. The parent change owns final parser version and + fingerprint updates. + +## Verification + +- TDD: all initial ten regression tests failed before implementation and passed + after it. Additional tests first exposed and then verified heading/table image + extraction, repeated-URI placement metadata and excluded-subtree image handling. +- Fifteen new tests pass, including 5,000 nested divs and a shallow 250,001-node + document; no network/provider was used. Fourteen selected existing markup/image + regressions also pass. Parser package typecheck passes. +- A broader run of the existing parser file plus the first ten new tests had + 111 passes and one concurrent structured-XML expectation failure (score now + remains a string); the parent structured-data fix owns that expectation. +- The parent task owns full package coverage/build/lint and contract-lock + regeneration. Shared index formatting was deliberately left for the parent to + avoid colliding with concurrent structured/provider changes. The dedicated test + file is formatted. No commit, deployment or production mutation was performed. + +## Remaining boundaries + +The DOM budget is enforced after htmlparser2 constructs the DOM, before our +traversal. It prevents projection/stack blowups but does not claim a streaming, +pre-allocation DOM parser; input-byte admission still bounds the input. Marked +lexing precedes token traversal checks. A terminable CPU worker and streaming +parser-admission limits remain part of the broader resource-isolation work. diff --git a/knowledge-fs/.harness/changes/2026-09-05-office-admission-regression-fixtures.md b/knowledge-fs/.harness/changes/2026-09-05-office-admission-regression-fixtures.md new file mode 100644 index 00000000000..eaa24eafdf6 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-office-admission-regression-fixtures.md @@ -0,0 +1,40 @@ +# Office admission regression fixtures and compatibility boundary + +## Changes + +- Replace successful remote DOCX/XLSX transport test inputs that were arbitrary + bytes or fabricated ZIP headers with small actual deflated OOXML packages. + The API app uses a fixed, fflate-generated DOCX ZIP encoded as base64 to avoid + adding a production or test dependency solely for a fixture. +- Update default parser-version expectations and the Markdown artifact digest + golden for the explicitly bumped versions. Custom parser versions and + coordinator fingerprints are not changed. +- Separate successful archive-image extraction from unsafe archive admission. + DOCX/PPTX/XLSX/ODT/EPUB archives with `../` member paths must fail with a + terminal input error before the provider is called. +- Preserve malformed ZIP/XML inputs as rejection regressions. The prior + expectation that provider text remains usable for damaged spreadsheet archives + is intentionally superseded by preflight admission. +- Keep the original unsafe worksheet metadata fixture in an explicit rejection + case. A second case uses valid worksheet declarations and coordinates while + retaining malformed/external optional image relationships and verifies the + original fail-soft image anchoring and fallback behavior. + +## Compatibility boundary + +Damaged archives/XML, unsafe member paths, or missing/external worksheet +relationships no longer enter remote Office parsing. These affect safe resource +inspection and are not treated as optional image metadata. Optional image/drawing +relationship failures still leave admitted document text usable; no file is +rewritten before being sent to the provider. + +## Verification + +The first integrated run exposed 22 failures from obsolete version assertions, +fake archives, and the explicitly tightened admission policy. Both relevant files +now pass all 124 tests. An existing deadline mock hung only in the full suite; it +now retains its Request objects for the lifetime of the simulated transport and +checks an already-aborted signal, matching real fetch behavior. No timeout was +extended, assertion removed, or production deadline logic changed for this test. +Both parser and API-app typechecks pass. Scoped Biome and git whitespace checks +pass. The parent task owns final broader-suite validation and integration notes. diff --git a/knowledge-fs/.harness/changes/2026-09-05-parser-format-contracts-and-coverage.md b/knowledge-fs/.harness/changes/2026-09-05-parser-format-contracts-and-coverage.md new file mode 100644 index 00000000000..647246a3818 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-parser-format-contracts-and-coverage.md @@ -0,0 +1,37 @@ +# Parser format contracts, encoding, provenance and coverage + +## What changed + +- A shared document format registry now drives native/remote routing, upload MIME aliases and source filename MIME inference. The existing 25 upload extensions are unchanged; internal YAML/NDJSON support is not added to uploads. Inherited object keys such as `constructor` and `__proto__` cannot become formats or crash admission. +- Native text decoding is strict UTF-8, or UTF-16LE/BE with a BOM. Malformed sequences, unsupported UTF-32 BOMs and unmarked binary NUL text fail with terminal `provider_input` rather than indexing replacement characters. +- Properties documents have a dedicated semantic projection: comments omitted, continued lines joined, separators and escapes decoded, Unicode surrogate pairs validated, and original source line/key/value retained. This follows Java's character-reader properties grammar; there is deliberately no heuristic legacy code-page detection. +- WebVTT produces cue text with identifier, start/end milliseconds, settings and raw cue payload; header/NOTE/STYLE/REGION blocks are not indexed as speech. Malformed cues fail explicitly instead of silently losing content. +- Native structured syntax failures, row limits and element limits retain terminal input classification. Oversized native markup no longer bypasses native admission by silently being sent to the remote service; explicit OCR/layout/language routes remain available. +- Unstructured accepts a deployment-owned `backendRevision` (bounded to 256 characters). It participates in checkpoint fingerprint, artifact hash and metadata. Client semantic versions advance to native Markdown/MDX 4, HTML 5, structured 4, Unstructured 12. An external provider without a configured revision is explicitly `external-unversioned`, not falsely attested as the pinned image. +- `requiresImages` remains tri-state in fingerprints/hashes: legacy-auto (`undefined`) and explicit text-only (`false`) cannot coalesce or reuse one another's archive-media output. A concurrent parse regression verifies distinct artifacts, not just distinct policy strings. +- Archive fallback honors `requiresImages=false`. Unsupported media (including SVG), count/byte omissions, Office chart/diagram visuals and inspection failures produce bounded `archiveMediaReport` references and `parseCoverage.media` reasons. Reports preserve up to 4,096 resource paths (1,024 characters each), explicitly reporting truncation. Unknown provider text/table/media completeness remains `unknown`; it is never guessed complete. +- Artifact-level coverage/provenance metadata is included in structural output admission, not just element arrays. + +## Why + +Prevent silent encoding/content corruption, format-policy drift, generic misleading compilation failures, cache reuse across provider semantic upgrades, and silent partial multimodal extraction. Keep byte/pixel/normalization protections from the first hardening phase unchanged. + +## Verification + +- TDD reproductions failed first for UTF-16, malformed UTF-8, properties, VTT, backend revision hashes, native overflow routing, unsupported/archive image gating, omitted charts/images, missing source MIME aliases, and prototype-key filename admission. +- `pnpm --filter @knowledge/parsers test:coverage`: 489 passed; statements/lines 97.04%, branches 91.34%, functions 97.91%. +- `pnpm --filter @knowledge/parsers typecheck`: passed. +- Correctly scoped upload/source tests: 80 passed. Initial root-directory Vitest invocation omitted the API package's Dify object-storage setup and consequently failed two source imports. The correctly scoped API command loads `packages/api/vitest.config.ts`; this was a test-invocation issue, not a compilation regression. Full API verification belongs to the integration pass. +- Biome formatting/check applied only to owned modified files; full workspace verification belongs to the integration pass. + +## Scope and risks + +- No live provider, production deploy, migration, commit or push was performed by this slice. Pinned-provider real-file golden execution requires an available Docker/service runtime. +- Parse coverage is a truthful report, not support for rasterizing previously unsupported SVG/EMF/WMF/Office charts. Original document bytes and archive-path provenance remain available; omitted visual content is explicitly partial. +- Existing raw checkpoints correctly invalidate on the new semantic versions. Transport/admission-only limit changes are not misrepresented as provider content revisions. +- Strict malformed text/VTT handling intentionally rejects content that previously appeared to import successfully while indexing corrupt/control text. Normal UTF-8 and existing native format content stay covered by the parser regression suite. + +## Grammar references + +- [Java Properties character-reader grammar](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/Properties.html#load(java.io.Reader)) +- [WebVTT specification](https://www.w3.org/TR/webvtt1/) diff --git a/knowledge-fs/.harness/changes/2026-09-05-parser-hardening-phase-one.md b/knowledge-fs/.harness/changes/2026-09-05-parser-hardening-phase-one.md new file mode 100644 index 00000000000..d49f43f0505 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-parser-hardening-phase-one.md @@ -0,0 +1,48 @@ +# Parser hardening — first implementation phase + +## Scope and intent + +Implement the first phase of the 2026-09-05 parser audit: bound structural amplification, preserve native document content, and prevent incompatible size-based routing. This is not a claim that every parser or every resource-exhaustion vector is now sandboxed. Existing PDF safeguards and the MediaBox consistency follow-up remain in their dedicated change notes. + +## Changes + +- Office/ODF/EPUB admission streams actual ZIP member expansion before contacting Unstructured, within the existing shared single-flight/admission lifetime. It checks local/central entry agreement, duplicate/unsafe paths, XML structure, worksheet coordinates, shared strings, and logical worksheet references. A repeated worksheet relationship consumes its repeated dense-cell cost. Input bytes and provider partition options are unchanged for admitted files. +- Inspection has explicit defaults: 4,096 members, 512 MiB total actual expansion, 64 MiB total XML / 16 MiB per XML part, depth 128, 1 million XML nodes, 256 logical worksheets, 100,000 rows / 16,384 columns, 250,000 dense cells per sheet / 500,000 per workbook, 500,000 actual cells, 200,000 shared strings, 30-second inspection deadline. Column formatting ranges do not inflate occupied cell span. Standard EPUB 2 XHTML PUBLIC identifiers are allowed without loading DTDs; internal subsets and arbitrary declarations are not. +- Shared native budgets bound decoded structure (250,000 nodes / depth 128), projected output (32 MiB), table width (4,096) and cells (500,000). HTML expansion and table text bytes share a parse-local document-wide counter, including separate Markdown HTML blocks and separate provider table elements; independent parses do not share counters. Final/provider metadata uses a separate 1 million-node budget to accommodate the existing 20,000-element limit and ordinary coordinate metadata. Limits fail explicitly; they never silently truncate content. +- JSON/JSONL retain numeric source tokens, including large integers, precise decimals, exponent notation and negative zero. Node 22's native `JSON.parse` reviver source preserves own `__proto__` fields safely. No new runtime dependency is added. JSONL scalar/null/array records retain their values and source order. +- CSV uses array decoding and own-property record construction; duplicate headers receive collision-free stable names in amortized linear naming work, special property names are preserved, header-only documents retain headers, and absent object fields never read inherited properties. +- Sparse heterogeneous records stay sparse instead of expanding into the union-key Cartesian product. Ordinary homogeneous record tables retain their representation. Table text is byte-budgeted before joins, including header-only output and repeated labels. +- XML attributes and lexical tag values are retained. Untyped XML numeric-looking text is deliberately represented as a string, preventing loss of leading-zero identifiers. +- Structured files stay on the native structured parser when configured. The API uses the same admitted input-byte limit for structured and remote paths (15 MiB default; existing override capped at 50 MiB), so 10–15 MiB JSON/CSV uploads no longer switch to an incompatible remote format. +- Native parsing checks an already-aborted request before decoding. Shared remote normalization uses the coordinator-owned signal, not the first caller's signal, preserving remaining consumers of identical work. +- Content/layout changes advance default parser identities: Markdown/MDX v3, HTML v4, structured v3, Unstructured v11. Existing indexes are not rewritten automatically; subsequent parse/re-index operations use the new policy identity. + +The markup fidelity, CJK spatial-index/heading budget, raw provider-response guard, synchronous deadline detection, HTML span budgets, and PDF geometry work are detailed in adjacent change notes. + +## Compatibility and security boundaries + +- Normal-file content and transport contracts are regression-tested, but newly imposed cost limits intentionally reject exceptionally large or ambiguous structures even when the file format itself permits them. Error classification is terminal input failure before remote work; no retry storm and no remote-acceptance ambiguity is introduced for admission failures. +- Previously permissive test fixtures containing path traversal, malformed ZIP/XML, invalid coordinates or unsafe worksheet relationships are now explicit rejection cases. Safe media-anchor failures remain fail-soft inside a valid admitted archive. +- Standard EPUB declaration support was checked against upstream Unstructured/Pandoc source and archive-based tests; this is not a pinned-Pandoc production round trip. +- Native numeric lexeme support was verified directly on Node **22.0.0**, the minimum major version used by the deployment images, as well as the development runtime. + +## Verification + +Tests were added and observed failing before behavioral fixes. Focused tests cover ordinary and adversarial inputs, source-byte preservation, special JSON/CSV property keys, cancellation/single-flight behavior, size-routing boundaries, and real Poppler page/crop geometry. + +- Final parser suite: **442/442**, coverage **96.81% statements/lines, 91.09% branches, 97.8% functions**; no coverage threshold was lowered. +- Node **22.0.0**: 50 JSON/structured integration tests pass on the deployed major's initial release, not just the newer development runtime. +- API package regression suite: **5,061 passed / 3 existing skipped**, 426 passed test files / 1 existing skipped file. Focused real Poppler rendering/crop tests: **60/60**. API parser configuration/preflight tests: **55/55**. +- Full **`pnpm check` passes**, including all workspace tests, configured CI coverage gates, retrieval/phase-4 regressions, OpenAPI export, migration artifact checks and Compose/static smoke checks. The repository's existing CI coverage command excludes the API package; its complete test suite and focused PDF tests still run as recorded above. +- Full workspace typecheck and **`pnpm build` pass**; backend-scoped Biome passes. `git diff --check` passes. Contract pin/export validation uses an isolated temporary Git index so the user's staging area is not changed. +- Full `pnpm lint` was attempted and remains blocked by pre-existing, untouched Admin/test/generated OpenAPI formatting/file-size findings. Those unrelated files were not reformatted as part of parser work. +- Local diagnostic: the audit's 4,881-byte, 300-record sparse JSON now produces **3,979 bytes** of text (about 4 ms in one bounded run), instead of the audited 1,137,189-character dense projection. This is a regression probe, not a production benchmark. +- Production provider/worker redeployment and live ingestion E2E were not run. Real-pinned-provider corpus benchmarks and server verification remain required before calling the entire optimization roadmap complete. + +## Remaining roadmap (not completed by this phase) + +- Nested EML/MSG attachments do not yet share recursive admission/depth/budget accounting; attachment behavior is not disabled as a shortcut. +- Legacy DOC/PPT/XLS conversion and native library image decoding still require provider-side process isolation, hard memory/CPU limits and killable work. Format preflight cannot prove arbitrary binary parser safety. +- Native JSON/HTML/XML and Markdown tokenization still allocate in the API process within byte/structure bounds. The new deadline check detects synchronous overruns; it cannot preempt JavaScript execution. XML's structural check is post-decoding; move recursive native work to bounded workers in a later phase. +- Unified encoding policy, complete/partial visual-enrichment reporting, aggregate remote-image budgets, provider-version fingerprinting, cross-process resource admission and production corpus benchmarks remain later phases. +- No database migration, model configuration, production deployment, commit or push is performed by this implementation turn. diff --git a/knowledge-fs/.harness/changes/2026-09-05-parser-hardening-remaining-phases.md b/knowledge-fs/.harness/changes/2026-09-05-parser-hardening-remaining-phases.md new file mode 100644 index 00000000000..ac9b42e8953 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-parser-hardening-remaining-phases.md @@ -0,0 +1,145 @@ +# Parser hardening: remaining phases + +## Scope and sequence + +Follow-up to `2026-09-05-parser-hardening-phase-one.md` and the all-format parser audit. +Existing uncommitted phase-one changes are preserved. No production deployment or migration +is part of this implementation turn. + +1. Execution contracts: shared format registry, strict text decoding, properties/VTT semantics; + bounded cancellable native child processes; opt-in dedicated Unstructured request isolation + and nested mail/archive admission. +2. Completeness/reuse: immutable media execution plan, sync/durable capability parity, + aggregate remote media budgets, explicit skipped/partial coverage, backend semantic revision. +3. Evaluation: real native golden/resource benchmark, separate preview/analysis variants, + dedicated provider image golden gate and resource/cancellation tests. + +## Native execution boundary + +The Node API wraps native parsers in one shared admission controller (2 active processes, +32 queued requests, 128 MiB total reserved input, 50 MiB absolute input ceiling further +restricted by the existing API input setting). Each request receives a fresh process with a +256 MiB V8 old-generation ceiling and a 600-second wall deadline including admission. +Linux additionally samples resident memory every 50 ms and terminates the child above +512 MiB (this sampled check is not a hard RSS quota). The same lifecycle executor isolates +Sharp image variant work with separate 128 MiB V8 / 384 MiB sampled RSS limits, a 30-second +task deadline, 2 active / 8 queued tasks and 64 MiB reserved input. Each document also has +an aggregate image/variant work budget; independent task limits cannot reset that budget. +Cancellation/timeout sends SIGKILL; admission is released only on process close. The process +receives no provider credentials or inherited NODE_OPTIONS. Serialized output is capped at +32 MiB; peak RSS, parser elapsed time and input/output bytes are added to artifact metadata, +not semantic artifact identity. This is process isolation plus bounded allocation, **not** a +claim that V8 heap size is a hard OS RSS limit or that Node processes form a security sandbox. + +Production builds include a compiled `native-parser-worker.mjs` (no production tsx). The image +bundle smoke now executes a native parsing request in the child, in addition to Sharp/Poppler. +It also executes the compiled image-variant worker. Worker responses are validated at the +parent boundary. Unexplained process exits and malformed IPC are operational response +failures, not falsely labelled malformed user documents. Known format rejection remains +an input failure; cancellation retains the original reason, including falsy reasons. + +Unsafe or budget-rejected original images are retained with a bounded optional +`analysisUnavailable` marker. This marker survives manifest/node/candidate projection and +prevents embedding, enrichment and answer providers from bypassing rejection via an original +image fallback. Legacy artifacts without this optional field retain their previous behavior. + +## Verification + +- RED: missing isolated adapter; API native route had no process metadata; backend revision + did not affect API policy fingerprint; image bundle did not include/probe the worker. +- Native process regression tests: real parser content/hash equivalence, real CPU-bound child + killed on abort, shared slot lifetime, queue/byte admission, deadline, crash and typed errors. +- API parser options tests include configured backend semantic identity and >10 MiB JSON. +- Final native lifecycle/protocol/memory focused gate: 43 tests, 100% statements/lines/functions + and 96.35% branches across the four implementation modules. Individual module branches + are also at least 90%; no threshold was relaxed. +- Real native golden benchmark: 10 format families × 2 bounded fixture sizes × 3 repetitions; + all required Unicode markers preserved. Measures p50/p95, child peak RSS and output amplification. + Markers are **not** a proxy for OCR accuracy or retrieval recall. Development tsx startup is + included in latency; do not compare these values to production compiled worker latency. + +## Release gates / environment limits + +Local Docker CLI exists but the daemon socket is unavailable. Actual pinned Unstructured +image compatibility, legacy binary golden fixtures and multi-replica production capacity are +not validated by local mocked/unit tests. The new provider sandbox must remain opt-in until +its real-image contract gate passes. Do not mark this deployment/quality gate complete merely +because source tests pass. Final verification and remaining external gates will be recorded below. + +## Measured local evaluation + +- Node 22.23.2 / macOS arm64 native adapter: 10 format families × 10/1,000 rows × 3 runs; + 60 actual isolated parses, required Unicode markers all preserved. Across the 20 cases, + p50 140–175 ms, p95 142–197 ms and peak child RSS approximately 99–112 MiB. These are + development tsx-startup-inclusive measurements, not compiled production throughput or OCR + quality estimates. +- Both compiled workers were separately exercised on Node 22.0.0, including exact large + JSON integer preservation and real Sharp analysis/thumbnail output. The full API dependency + graph already contains undici 8.10.0, whose installed engine requires Node >=22.19.0; + full API compatibility is therefore verified on 22.23.2, not asserted for 22.0.0. No runtime + dependency was downgraded or updated during this compatibility check. +- Bounded synthetic PDF thumbnail comparison: repeated two-Poppler rendering measured + 95–103 ms, versus 73–83 ms for a shared main bitmap plus Sharp thumbnail (296 ms cold + run). The output was **not pixel equivalent** (12.54 dB PSNR), and the derived thumbnail + was larger (7,152 vs 6,115 bytes). The production two-render policy remains unchanged; + broader OCR, non-Latin, chart and memory measurements are required before replacement. + +## Final local gates + +- Full `pnpm check` **passed** after final fixes, including workspace typechecking/tests, + configured CI coverage gates, retrieval and phase-4 evaluations, hermetic OpenAPI export, + migration artifacts, Compose configuration and static smoke/workflow contracts. +- Parser package: **521 tests passed**, 97.13% statements/lines, 91.50% branches, + 97.97% functions. API package: **5,124 passed, 3 pre-existing skipped** (430 passed files, + one skipped). API application: **432 tests passed** (57 files). Existing CI aggregate + coverage excludes the API package; focused changed-module coverage is recorded separately, + without lowering thresholds or treating unrelated baseline coverage as newly verified. +- Media focused gate: 113 tests, 99.06% lines, 92.70% branches, 100% functions before final + boundary refinements. Final marker/manifest/enhancer regression gate: 25 passed, followed + by the complete API suite above. It covers count/byte-skipped large inline sources, + whitespace-prefixed data URIs, recovery provenance and truthful missing/unsupported states. +- Dedicated provider runtime: **80 tests**, including actual disposable process/descendant + cancellation, file-locked conversion budgets, sticky signal-death rejection, atomic + publication and real oxmsg fixture admission; 96.90% statements and 92.05% branches. + Ruff passes. Both opt-in Compose overlays were configuration-validated. +- `pnpm build`, production esbuild with both worker entry points, backend Biome and + `git diff --check` pass. Final compiled-worker smoke on Node 22.23.2 preserves a large + integer, returns a typed invalid-JSON error, and generates both actual Sharp variants. +- Full `pnpm lint` still reports the known unrelated Admin/test/generated-artifact + formatting and generated OpenAPI size baseline; backend-scoped lint is clean. Those + unrelated files were not reformatted to manufacture a green global result. +- OpenAPI was regenerated for the additive optional `analysisUnavailable` contract. Contract + lock generation/validation uses a temporary Git index, preserving the user's staging area. + +## Delivery status and outstanding acceptance + +The remaining implementation slices are delivered: shared format/native admission contracts, +bounded child execution, nested mail and converted-product admission, capability-aware media +plans, aggregate image work limits, explicit completeness/provenance, backend revision identity, +incremental JSONL/XML admission, preview/analysis separation and reproducible local benchmarks. +Details live in the adjacent format, media and provider change notes. + +Not yet accepted: actual pinned Unstructured image build/HTTP golden corpus, read-only image +compatibility, real OCR/table/image recall and localization, production cost-weighted/multi-replica +capacity and a staging canary. Docker is unavailable locally, and no staging deployment was +authorized or performed. Hard per-request cgroup/storage quotas require deployment-level +delegation; process sampling is not equivalent. The legacy synchronous path still has its +documented non-atomic model-head decision; durable production jobs use frozen profiles. + +The provider override replaces the existing dedicated service only, never a second per-format +service, and stays opt-in. Existing documents are not automatically re-indexed. No database +migration, production environment mutation, service restart, commit or push occurred. + +## Submission follow-up + +The subsequent user request authorizes committing and pushing this change. The remote branch +had advanced to `ad43bf7eda`, so the local parser change was replayed on that tip without +rewriting any published commit. Only the generated contract lock conflicted; it was regenerated +against the combined tree. Independent review confirmed that the remote namespace-preview +authentication, website-import recovery, scheduler diagnostics and CI changes remain intact. + +After integration, full `pnpm check`, `pnpm build`, production esbuild, backend Biome and secret +scanning pass. API regression now has 5,135 passing tests and the same 3 existing skips; parser +and API-app totals remain 521 and 432. The unmodified provider process suite was not needlessly +rerun. Root `.turbo/`, `artifacts/` and `tmp/` are excluded from the commit. The real-image and +deployment acceptance gates above remain open; committing does not enable the sandbox override. diff --git a/knowledge-fs/.harness/changes/2026-09-05-pdf-mediabox-budget-consistency.md b/knowledge-fs/.harness/changes/2026-09-05-pdf-mediabox-budget-consistency.md new file mode 100644 index 00000000000..ac5ddf713fd --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-pdf-mediabox-budget-consistency.md @@ -0,0 +1,54 @@ +# Use the actual Poppler MediaBox for local raster admission + +## Why + +The local rasterizer estimated allocation from `pdfinfo`'s `Page size`, which is +the CropBox, while its existing `pdftoppm` command renders the MediaBox. A tiny +CropBox therefore did not bound a larger backing bitmap. A safe 144 x 144 point +MediaBox with a 72 x 72 CropBox reproduced this: with a 10,000-pixel budget the +old implementation rendered 20,736 pixels and only then failed in Sharp. + +## Changes + +- Read the requested page's resolved MediaBox endpoints, subtracting nonzero + origins, and reject missing, duplicate, incomplete, non-finite or reversed + geometry before starting `pdftoppm`. +- Preserve MediaBox rendering and the established displayed-page coordinate + contract; do not switch to `-cropbox`. Page rotation only swaps the dimensions + used for area/long-edge admission. Actual rendered dimensions still drive + provider pixel, displayed-point and relative crop mapping. +- Account for `pdfinfo` rounding both box endpoints to two decimal places: use + conservative dimension bounds and the worst-case aspect ratio. The second + real regression (100.004 x 100 points) otherwise passed the rounded estimate + and exceeded its allocation budget by one pixel column. +- Preserve the earlier page-pixel/long-edge caps, shared page bitmap/session, + cancellation, cleanup and provider-fallback semantics. Ordinary uncapped pages + retain their rendering parameters. A page exactly at a rounded budget boundary + can be downscaled slightly more conservatively (typically one pixel). + +## Verification + +- TDD red: five real small-PDF tests failed before the MediaBox fix, including + the original mismatch and four rotations. They passed after the fix. +- TDD red: the fractional 100.004-point regression then failed against the first + MediaBox implementation; conservative rounding fixed it. +- 70 focused rasterizer tests pass, including eight real Poppler tests, nine + rejected geometry cases, existing admission/cancellation/cleanup tests, and + content-colour checks for nonzero origins with 0/90/180/270-degree rotation. + Both relative and provider-style pixel/displayed-point crops preserve colour + and position after downscaling. No unsafe-size raster was generated. +- Focused production-file coverage: statements/lines 95.46%, branches 92.98%, + functions 98.61%. API package typecheck passes. Scoped Biome and diff checks + pass after formatting. +- The parent implementation task owns full-workspace build/lint/test and contract + lock regeneration. They were not repeated for this isolated parallel slice. + No commit, deployment, database migration or production request was performed. + +## Remaining limits + +This closes the CropBox/MediaBox allocation mismatch; it is not an arbitrary PDF +object/embedded-image memory sandbox. The API preflight and provider-native pixel +guard remain complementary. Crops retain the pre-existing displayed-page frame; +raw bottom-left PDF user-space coordinates without frame metadata are not newly +supported. Changes to provider page-frame conventions require explicit contract +tests rather than silently switching rendering boxes. diff --git a/knowledge-fs/.harness/changes/2026-09-05-pdf-page-raster-safety.md b/knowledge-fs/.harness/changes/2026-09-05-pdf-page-raster-safety.md new file mode 100644 index 00000000000..2018e5c0c59 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-pdf-page-raster-safety.md @@ -0,0 +1,99 @@ +# Bound PDF page rasters before parser allocation + +## Problem and evidence + +A user-provided 247,902-byte, single-page PDF has a 2267.72 × 5102.36 point +MediaBox/CropBox (80 × 180 cm). File-byte and document-concurrency admission did +not bound its bitmap allocation. The pinned Unstructured image contains +unstructured 0.22.18 and unstructured-inference 1.6.11, defaults to 350 DPI, and +would render this page to 11024 × 24804 = 273,439,296 pixels. A single RGB buffer +alone is approximately 820 MB before image copies and inference allocations. +Its existing renderer ceiling was one billion pixels per page. + +The pinned API does not expose `pdf_image_dpi` through its form/partition +whitelist, so sending a per-request DPI field cannot fix this issue. + +## Changes + +- The production Node parser injects a PDF preflight into the existing shared + admission and single-flight operation, before transport starts. A single + `pdfinfo` invocation checks all pages and their resolved size, MediaBox and + CropBox. It admits at most 10,000 pages, 25 million pixels per page, and 10,000 + pixels per edge at 350 DPI. Unknown, incomplete or unsafe geometry fails closed. +- Inspection has a 10-second subprocess deadline, 4 MiB output limit, private + temporary directory and 0600 source file. Cancellation kills and joins the + child before cleanup. Non-PDF inputs do not incur filesystem/subprocess work. +- Dedicated parser defaults and local Compose retain 350 DPI and set + `PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000`. The pinned PDFium renderer enforces + this before bitmap allocation across hi_res, OCR/auto fallback and image-block + extraction. This also covers geometry differences between PDF engines. +- The pinned service exposes its raster guard as HTTP 500. An exact bounded + JSON-detail classifier turns only that known rejection into a non-retryable + provider input error before inline retries. Error inspection is limited to + 4 KiB/one second; other HTTP failures retain their retry semantics. +- Local Poppler crop generation now applies both edge and pixel budgets before + rendering, including short-edge rounding. Coordinate mapping still uses the + actual scaled page dimensions. Ordinary pages retain their original rendering + parameters. +- Docker build/runtime checks verify `pdfinfo`; CI installs Poppler for actual + PDF geometry integration tests. Deployment documentation covers both guards. + +## Compatibility and operations + +This is resource admission, not an output transformation. Original files, parser +strategies, text extraction and multimodal behavior of admitted files are +unchanged. Existing safe raw checkpoints remain reusable; budget-only checks do +not change parser output fingerprints. Platform-neutral callers constructing the +low-level parser client must inject an appropriate `requestPreflight`; the Node +API does so by default, and the dedicated service has its own allocation guard. + +Oversized pages are rejected rather than silently converted to text-only parsing. +To import the sample banner, export it with smaller physical page dimensions or +tile its canvas into smaller pages. Splitting only between existing giant pages +or compressing the file further does not reduce this risk. Automatic layout- +preserving tiling/normalization is follow-up work, not implemented here. + +Existing deployments need the new API image and a recreated dedicated parser with +the updated defaults. Operator env files load afterwards: remove any conflicting +DPI/pixel-limit overrides. Kubernetes users must apply the same limits to their +external Unstructured service. No database migration is required. + +Geometry limits do not bound every embedded bitmap decoder, pathological PDF +object graph or model allocation. Container memory limits and workload admission +remain necessary. Poppler metadata inspection is bounded in duration/output, not +an OS-level memory sandbox. + +## Verification + +- TDD: unsafe input previously reached fetch; new tests fail before preflight + integration and pass afterwards. Provider raster rejection previously retried + as HTTP 500; the regression now observes exactly one request and no retry. +- Actual supplied PDF through `createApiDocumentParser`: rejected as + `provider_input`, retryable=false, providerCalls=0, about 13 ms locally. No + request was sent to an external parser. Bounded local rendering at 1 MP + completed in about 314 ms and produced a 163,694-byte PNG. +- Preflight: 39 tests, including real Poppler inherited boxes and an oversized + non-first page; lines/statements 99%, branches 94.89%, functions 100%. +- Parser package: 182 tests; lines/statements 96.23%, branches 90.30%, functions + 96.86%. Local rasterizer: 53 tests including scaled crop-coordinate fidelity. +- API full suite: 5,044 passed, 3 existing skips; API-app full suite: 349 passed. + Local rasterizer focused coverage: lines/statements 94.86%, branches 92.61%, + functions 98.61%. Workspace build/type checks (12 packages) and API production + bundle passed. Backend lint (1,122 files), workflow/secret tests (28), Compose + tests (14), image artifact tests (3), OpenAPI export tests (2), actual secret + scan and local/Dify Compose configuration validation all passed. +- Full Docker image builds/live unsafe-file execution are intentionally not used + for this diagnosis. No production configuration was changed. Read-only server + inspection confirmed the two PDF limits are not configured yet; current + OOMKilled=false with RestartCount=10 is not proof of the historical restart cause. +- The omnibus `pnpm check` (including Docker builds and live stack smoke tests) + and whole-workspace frontend lint were not run for this backend-only change; + the affected full package suites, coverage, backend lint, build and deployment + artifact gates above were run instead. Dify backend logic and database schema + are unchanged; only the KnowledgeFS subtree contract lock needs regeneration. + +## Upstream evidence + +- https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/partition/utils/config.py +- https://github.com/Unstructured-IO/unstructured-inference/blob/v1.6.11/unstructured_inference/config.py +- https://github.com/Unstructured-IO/unstructured-inference/blob/v1.6.11/unstructured_inference/inference/pdf_image.py diff --git a/knowledge-fs/.harness/changes/2026-09-05-unstructured-normalization-safety.md b/knowledge-fs/.harness/changes/2026-09-05-unstructured-normalization-safety.md new file mode 100644 index 00000000000..df12d8a2b49 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-unstructured-normalization-safety.md @@ -0,0 +1,57 @@ +# Bounded Unstructured response normalization + +## What changed + +- Admit raw provider JSON before Zod cloning and layout normalization: scan JSON depth before decoding, then enforce raw element count, tree nodes, and response-byte budgets. Invalid or excessive provider output remains a non-retryable `provider_response_invalid` error. +- Replace repeated full-document CJK glyph scans with a conservative page-local spatial grid. Only neighboring buckets are examined, consumed candidates are removed, and exact adjacency plus the existing deterministic distance/tie ordering are preserved. +- Apply an explicit comparison budget to dense/adversarial buckets instead of silently omitting evidence or allowing quadratic work to continue. +- Bound heading ancestry depth and cumulative emitted section-path items before their repeated expansion. +- Detect request deadline overruns using monotonic elapsed time as well as the timeout callback. Synchronous work cannot report success merely because it delayed the event-loop timer. +- Bound HTML `colspan` / `rowspan` expansion before each cell assignment, including empty cells and the logical width of sparse carried rows. A parse-local budget spans all native HTML tables, HTML tokens within Markdown/MDX, and provider `text_as_html` tables. It also charges the extra padding needed to scan ragged rows as rectangular tables. Independent parses never share this counter. Bound dense header/headerless scans and joined header bytes before projection, and compute maximum width without variadic argument expansion. +- Charge projected table text bytes against the same document-local state before joining header-only labels or constructing data-row strings. Ordinary Markdown tables and HTML tables share this byte counter, preventing repeated individually-admitted tables from allocating many output-sized strings before the final artifact guard. Temporary flattened-header bytes remain a separate admission check and are not double charged. + +Defaults: + +| Resource | Limit | +|---|---:| +| Raw provider elements | 50,000 | +| Raw provider / final artifact tree nodes | 1,000,000 | +| JSON structural depth | 128 | +| Raw response bytes | Existing configured `maxResponseBytes` (default 32 MiB) | +| Heading depth | 64 | +| Cumulative emitted section-path items | 100,000 | +| Vertical candidate comparisons | 1,000,000 | +| Expanded HTML table columns | 4,096 | +| Cumulative expanded HTML table cells / dense traversal cells, per document | 500,000 | +| Flattened HTML header bytes | 32 MiB | +| Cumulative projected table text bytes, per document | 32 MiB | + +The existing final `maxElements` remains unchanged (default 20,000). Native structured-record budgets remain separate. All over-budget cases fail explicitly; no truncation or empty-artifact fallback was introduced. + +## Why + +The all-format audit reproduced two distinct post-response amplification paths: repeated glyph scans across unrelated pages, and quadratic ancestry-array expansion. An HTTP byte cap alone did not constrain either path. Element filtering and final limits were also too late to protect schema and normalization work. + +The implementation preserves transport-owned cancellation and single-flight. Shared operations must not inspect the first caller's cancelled signal while serving another active caller. + +## Verification + +- TDD: five existing-behavior tests failed before the normalization/deadline implementation (parent-chain depth, category depth, cumulative path expansion, dense candidate work budget, delayed-timer deadline); three raw-response tests failed before raw admission (filtered raw elements, hidden metadata tree nodes, excessive JSON depth). +- Dedicated tests cover local candidate counts, page separation, negative/spatial bucket boundaries, large finite coordinates, tie ordering, coordinate-system compatibility, geometry overflow, boundary admission, and rejection classification. +- The 17 normalization/grid tests plus 5 raw-admission tests passed. +- HTML table TDD: four existing-behavior regressions failed before implementation (empty overwide colspan, cumulative empty cells, carried rowspans, and sparse carried logical width). Four follow-up regressions confirmed separate HTML / Markdown / MDX / provider tables could otherwise reset this counter, and one further test caught aggregate ragged-row padding. Nineteen dedicated tests now pass, including independent-parse isolation, exact-boundary admission, unchanged multirow-header/carry output, pre-projection traversal limits, and pre-join header bytes. These tests use small mocked budgets to exercise the real parser paths without large allocations. +- Document table-byte TDD: five regressions failed before the shared byte counter (HTML headers/data rows, Markdown and MDX tables, and provider tables). Ten dedicated tests now pass, including ordinary Markdown mixed with HTML tokens, exact cumulative byte admission, and independent native/provider parses. +- Independent JSON/resource helper tests also exposed and verified root-agent fixes for prototype-shaped JSON fields and invalid numeric budget options. Together these five dedicated test files contain 90 passing tests. +- Four focused helper modules measured 100% statement/line/function coverage and 97.79% branch coverage; each exceeded 90% branch coverage. Parser TypeScript checking passed. +- The original parser suite passed after the localized changes; full workspace verification remains the responsibility of the parent task's integrated verification after concurrent parser edits finish. +- After the final parse-local table cell/byte budget changes, all 442 parser tests, parser TypeScript checking, and `git diff --check` passed. Broader workspace checks remain with the parent task. +- Bounded diagnostic only: a simulated 2,352,895-byte response containing 12,000 cross-page CJK glyphs retained all 12,000 output elements in 221 ms locally, compared with about 2.2 seconds in the audit's previous implementation. These timings are not a production benchmark or SLA, and no OOM-scale inputs or network provider requests were run. + +## Remaining boundaries + +- Monotonic checks recognize overruns but cannot forcibly interrupt JavaScript currently executing on the same event loop. Hard CPU/RSS cancellation still requires worker/subprocess isolation. +- Raw response buffering and native `JSON.parse` allocation still occur in the API process, within the existing byte/depth bounds. The structural pass runs after decoding but before schema cloning and normalization. +- The spatial grid is conservative, not a claim of constant time for all inputs. Dense or extreme-geometry inputs can exhaust the explicit comparison budget and be rejected. +- New finite structural limits can reject unusually deep or complex otherwise-readable documents. This is explicit resource admission, not a promise that all previously accepted files remain accepted. +- The raw JSON tree budget does not inspect the internal semantics of strings. HTML table expansion now has its own pre-expansion budget; future nested content decoders must also enforce their own structural limits. +- No commit, push, deployment, or database migration was performed by this subtask. diff --git a/knowledge-fs/.harness/changes/2026-09-05-unstructured-request-isolation-gate.md b/knowledge-fs/.harness/changes/2026-09-05-unstructured-request-isolation-gate.md new file mode 100644 index 00000000000..b7ff41a3eb3 --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-09-05-unstructured-request-isolation-gate.md @@ -0,0 +1,61 @@ +# Optional Unstructured request isolation and release gates + +## What / why + +- Added an opt-in, single-service supervisor derived from the existing pinned Unstructured + image, with bounded shared admission, temporary request spooling, disposable process groups, + disconnect/deadline cleanup, inherited Linux limits, and sampled aggregate resource checks. +- Added shared nested EML/MSG and archive/XML/Excel admission. Implicit XLSX coordinates are + supported; renamed MSG content is detected through OLE properties rather than trusting `.xls`. +- Added Docker overrides for the existing dedicated service only. Base deployment files and + legacy Dify Unstructured are unchanged; unverified runtime is not enabled automatically. +- Added a tiny real-image golden runner and a licensed, hash-checked upstream MSG fixture. No fixtures are actual + user documents; no dangerous OOM payload is sent to an unbounded parser. +- Added client classification for the exact versioned supervisor contract. Confirmed 413/422 + resource/input rejections become nonretryable `provider_input`; confirmed killed-worker 504 + becomes nonretryable, non-ambiguous `provider_timeout`. Unknown errors and 429/503 preserve + existing behavior. Shared bounded error-body reading also preserves the existing PDF guard. +- Added version-scoped executable adapters for DOC/PPT LibreOffice and RTF/EPUB/ODT Pandoc + conversions. Products are admitted against the root/mail shared locked budget before + atomic publication or stdout release. No upstream Python monkeypatch or raw fallback exists. +- Inspected original ASGI lifecycle requirements; image build asserts there are no startup, + shutdown or custom lifespan hooks before permitting direct request replay. + +## TDD / verification + +- Initial admission and worker tests failed for absent implementation; targeted sparse Excel, + nested attachment budget, implicit coordinate, and renamed MSG regressions were then fixed. +- A deterministic yielding semaphore reproduced burst over-admission (20 reservations with + capacity 1); an atomic pre-await capacity reservation fixes it. +- A blocked response-send test reproduced indefinite admission retention; delivery is now + bounded by disconnect and the same document deadline, without emitting a second response. +- Conversion TDD covered byte/structure rejection, concurrent shared-budget spend, exact CLI + shapes, HTML spans, lifecycle deadlines and bounded output. A fake converter creating the + destination during execution exposed an overwrite race; publish now uses atomic link-if-absent. +- Independent review found signal-killed converters could leave the sticky ledger clear. + Regression failed for both missing rejection and wrong 422 mapping; negative statuses now + become sticky `worker_resource_limit` 413. A provider that catches the rejection cannot + publish an apparent success. Ordinary positive converter error codes remain unchanged. +- 80 Python tests pass, including real child/descendant process cancellation and cleanup. + Subprocess-aware coverage: 96.90% statements, 92.05% branches, 95.61% combined. +- 37 TypeScript response-guard tests pass; parser package typecheck passes. +- `ruff check` and formatting pass for the new service. Local merged Compose config passes. +- Full workspace verification is performed by the coordinating change; these focused commands + were not repeatedly rerun without relevant edits. + +## Remaining gates / honest limits + +Docker daemon is unavailable on this machine (`/Users/jyong/.orbstack/run/docker.sock` missing). +The real pinned image has therefore **not been built, started, or golden-tested**. The pinned +public MSG fixture is now bundled with its full Apache license and SHA-256 attribution; actual +local oxmsg validates its 30-byte attachment, while full HTTP output remains an image gate. + +Converted DOCX/PPTX/HTML structural re-admission is implemented and tested with real child +processes plus generated products. Its image-specific discovery/CLI compatibility still needs +the real golden gate. Hard aggregate request-specific cgroups are not present; +RSS/CPU/PID/tmp sampling can overshoot, and a fast-detaching unobserved descendant is not proven +captured. HTTP PDF self-parallelism is disabled inside this optional worker to avoid admission +deadlock; throughput must be benchmarked before promotion. These are release gates, not completed +production protections. See `services/unstructured-sandbox/README.md`. + +No production deployment, database change, user-file upload, commit, or push occurred. diff --git a/knowledge-fs/apps/api/Dockerfile b/knowledge-fs/apps/api/Dockerfile index 8c63c738147..4cbf6d70fff 100644 --- a/knowledge-fs/apps/api/Dockerfile +++ b/knowledge-fs/apps/api/Dockerfile @@ -63,16 +63,20 @@ ENV NODE_ENV=production \ WORKDIR /workspace # PDF image elements are rasterized outside Unstructured. Keep Poppler in the -# final image and fail the build if the executable is not usable on the target -# platform. Deployments can set KNOWLEDGE_PDF_RASTERIZER=off as a kill switch. +# final image and verify both the renderer and metadata inspector. pdfinfo is also +# required by remote PDF admission, even when local image rasterization is off. RUN apt-get update \ && apt-get install --yes --no-install-recommends poppler-utils \ && rm -rf /var/lib/apt/lists/* \ && command -v pdftoppm >/dev/null \ - && pdftoppm -v >/dev/null 2>&1 + && pdftoppm -v >/dev/null 2>&1 \ + && command -v pdfinfo >/dev/null \ + && pdfinfo -v >/dev/null 2>&1 COPY --from=builder /workspace/apps/api/dist/server.mjs ./server.mjs COPY --from=builder /workspace/apps/api/dist/migrate.mjs ./migrate.mjs +COPY --from=builder /workspace/apps/api/dist/native-parser-worker.mjs ./native-parser-worker.mjs +COPY --from=builder /workspace/apps/api/dist/image-variant-worker.mjs ./image-variant-worker.mjs COPY --from=builder /runtime/node_modules ./node_modules # Fail the image build if the target platform's sharp binary or libvips payload diff --git a/knowledge-fs/apps/api/package.json b/knowledge-fs/apps/api/package.json index 827d81b9b5a..9140c5976ea 100644 --- a/knowledge-fs/apps/api/package.json +++ b/knowledge-fs/apps/api/package.json @@ -4,7 +4,7 @@ "type": "module", "scripts": { "build": "tsc --noEmit", - "build:prod": "esbuild src/server.ts src/migrate.ts --bundle --platform=node --format=esm --target=node22 --external:sharp --banner:js=\"import { createRequire as __knowledgeCreateRequire } from 'node:module';const require = __knowledgeCreateRequire(import.meta.url);\" --outdir=dist --out-extension:.js=.mjs", + "build:prod": "esbuild src/server.ts src/migrate.ts src/native-parser-worker.ts src/image-variant-worker.ts --bundle --platform=node --format=esm --target=node22 --external:sharp --banner:js=\"import { createRequire as __knowledgeCreateRequire } from 'node:module';const require = __knowledgeCreateRequire(import.meta.url);\" --outdir=dist --out-extension:.js=.mjs", "dev": "NODE_ENV=development node --env-file-if-exists=../../infra/local/.env --import tsx --watch src/server.ts", "start": "node dist/server.mjs", "test": "vitest run --passWithNoTests", diff --git a/knowledge-fs/apps/api/src/image-variant-protocol.test.ts b/knowledge-fs/apps/api/src/image-variant-protocol.test.ts new file mode 100644 index 00000000000..fcf19da4cca --- /dev/null +++ b/knowledge-fs/apps/api/src/image-variant-protocol.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; +import { isImageInputRejection, validateImageVariantResponse } from "./image-variant-protocol"; +import { imageVariantsFromResponse } from "./isolated-image-variant-generator"; + +const variant = () => ({ + body: new Uint8Array([1]), + contentType: "image/png", + name: "analysis", + height: 12, + width: 24, +}); + +describe("image worker IPC response validation", () => { + it("classifies malformed worker replies as operational errors, not rejected documents", () => { + expect(() => imageVariantsFromResponse({ ok: true, variants: [{}] })).toThrow( + expect.objectContaining({ code: "provider_response_invalid" }), + ); + expect(() => + imageVariantsFromResponse({ ok: false, message: "Input image exceeds pixel limit" }), + ).toThrow(expect.objectContaining({ code: "provider_input" })); + expect(imageVariantsFromResponse({ ok: true, variants: [variant()] })).toHaveLength(1); + }); + it.each([ + "Input image exceeds pixel limit", + "Input buffer contains unsupported image format", + "Sharp image thumbnail output exceeds maxOutputBytes", + "Image variants exceed the output byte budget", + ])("recognizes bounded image rejection: %s", (message) => { + expect(isImageInputRejection(new Error(message))).toBe(true); + }); + it.each([ + new Error("Cannot find package sharp"), + new Error("Unexpected worker state"), + "Input image exceeds pixel limit", + ])("does not silently downgrade operational decoder failures", (error) => { + expect(isImageInputRejection(error)).toBe(false); + }); + it("rejects unbounded or malformed optional execution diagnostics", () => { + expect( + validateImageVariantResponse({ + ok: true, + variants: [{ ...variant(), execution: { wallMs: Number.POSITIVE_INFINITY } }], + }), + ).toBe(false); + }); + it("accepts bounded success and failure messages", () => { + expect(validateImageVariantResponse({ ok: true, variants: [variant()] })).toBe(true); + expect(validateImageVariantResponse({ ok: true, variants: [] })).toBe(true); + expect(validateImageVariantResponse({ ok: false, message: "Image rejected" })).toBe(true); + }); + it.each([ + undefined, + null, + [], + {}, + { ok: 1 }, + { ok: true }, + { ok: true, variants: null }, + { ok: true, variants: [null] }, + { ok: true, variants: [variant(), variant(), variant()] }, + { ok: false, message: 1 }, + { ok: false, message: "" }, + { ok: false, message: "x".repeat(1025) }, + ])("rejects malformed message %j", (message) => { + expect(validateImageVariantResponse(message)).toBe(false); + }); + it.each([ + { body: [1] }, + { body: new Uint8Array() }, + { body: new Uint8Array(16 * 1024 * 1024 + 1) }, + { contentType: "text/plain" }, + { name: "../escape" }, + { name: "x".repeat(65) }, + { width: 0 }, + { height: -1 }, + { width: Number.NaN }, + { height: 1.5 }, + ])("rejects malformed variant field", (patch) => { + expect(validateImageVariantResponse({ ok: true, variants: [{ ...variant(), ...patch }] })).toBe( + false, + ); + }); + it("bounds aggregate variant bytes and permits absent optional dimensions", () => { + expect( + validateImageVariantResponse({ + ok: true, + variants: [ + { ...variant(), body: new Uint8Array(8 * 1024 * 1024 + 1) }, + { ...variant(), body: new Uint8Array(8 * 1024 * 1024) }, + ], + }), + ).toBe(false); + expect( + validateImageVariantResponse({ + ok: true, + variants: [{ body: new Uint8Array([1]), contentType: "image/png", name: "thumbnail" }], + }), + ).toBe(true); + }); +}); diff --git a/knowledge-fs/apps/api/src/image-variant-protocol.ts b/knowledge-fs/apps/api/src/image-variant-protocol.ts new file mode 100644 index 00000000000..8e9df39edec --- /dev/null +++ b/knowledge-fs/apps/api/src/image-variant-protocol.ts @@ -0,0 +1,89 @@ +import type { + GenerateDocumentImageVariantsInput, + GeneratedDocumentImageVariant, + SharpImageThumbnailVariantGeneratorOptions, +} from "../../../packages/api/src/document-image-variant-generator"; + +export interface ImageVariantRequest { + readonly input: Omit; + readonly options: SharpImageThumbnailVariantGeneratorOptions; +} +export type ImageVariantResponse = + | { readonly ok: true; readonly variants: readonly GeneratedDocumentImageVariant[] } + | { readonly ok: false; readonly message: string }; +export const imageVariantMaxInputBytes = 50 * 1024 * 1024; +export const imageVariantMaxOutputBytes = 16 * 1024 * 1024; + +/** Unknown/native initialization errors remain operational failures instead of silently omitting media. */ +export function isImageInputRejection(error: unknown): error is Error { + return ( + error instanceof Error && + /^(?:Input (?:image exceeds pixel limit|buffer (?:contains unsupported image format|has corrupt header))|Sharp image thumbnail output exceeds maxOutputBytes|Image variants exceed the output byte budget)/u.test( + error.message, + ) + ); +} + +export function validateImageVariantResponse(value: unknown): value is ImageVariantResponse { + if (typeof value !== "object" || value === null || Array.isArray(value) || !("ok" in value)) + return false; + if (value.ok === false) + return ( + "message" in value && + typeof value.message === "string" && + value.message.length > 0 && + value.message.length <= 1024 + ); + if ( + value.ok !== true || + !("variants" in value) || + !Array.isArray(value.variants) || + value.variants.length > 2 + ) + return false; + let bytes = 0; + const names = new Set(); + for (const variant of value.variants) { + if ( + typeof variant !== "object" || + variant === null || + !(variant.body instanceof Uint8Array) || + variant.body.byteLength === 0 || + variant.contentType !== "image/png" || + typeof variant.name !== "string" || + !/^[a-zA-Z0-9_-]{1,64}$/u.test(variant.name) || + names.has(variant.name) + ) + return false; + if (variant.execution !== undefined) { + const execution = variant.execution; + if ( + typeof execution !== "object" || + execution === null || + execution.isolation !== "child-process" || + !Number.isSafeInteger(execution.inputBytes) || + execution.inputBytes < 0 || + execution.inputBytes > imageVariantMaxInputBytes || + !Number.isSafeInteger(execution.outputBytes) || + execution.outputBytes < 0 || + execution.outputBytes > imageVariantMaxOutputBytes || + !Number.isFinite(execution.wallMs) || + execution.wallMs < 0 || + !Number.isSafeInteger(execution.peakRssKiB) || + execution.peakRssKiB < 0 + ) + return false; + } + if ( + (variant.width !== undefined && + (!Number.isSafeInteger(variant.width) || variant.width < 1)) || + (variant.height !== undefined && + (!Number.isSafeInteger(variant.height) || variant.height < 1)) + ) + return false; + names.add(variant.name); + bytes += variant.body.byteLength; + if (bytes > imageVariantMaxOutputBytes) return false; + } + return true; +} diff --git a/knowledge-fs/apps/api/src/image-variant-worker.ts b/knowledge-fs/apps/api/src/image-variant-worker.ts new file mode 100644 index 00000000000..c634273f100 --- /dev/null +++ b/knowledge-fs/apps/api/src/image-variant-worker.ts @@ -0,0 +1,44 @@ +import { createSharpImageThumbnailVariantGenerator } from "../../../packages/api/src/document-image-variant-generator"; +import { + type ImageVariantRequest, + type ImageVariantResponse, + imageVariantMaxInputBytes, + imageVariantMaxOutputBytes, + isImageInputRejection, +} from "./image-variant-protocol"; + +process.once("message", async (request: ImageVariantRequest) => { + const started = performance.now(); + let response: ImageVariantResponse; + try { + if ( + !(request.input.body instanceof Uint8Array) || + request.input.body.byteLength > imageVariantMaxInputBytes + ) + throw new Error("Image worker input exceeds its byte budget"); + const variants = await createSharpImageThumbnailVariantGenerator(request.options).generate( + request.input, + ); + if ( + variants.reduce((sum, variant) => sum + variant.body.byteLength, 0) > + imageVariantMaxOutputBytes + ) + throw new Error("Image variants exceed the output byte budget"); + const execution = { + isolation: "child-process" as const, + inputBytes: request.input.body.byteLength, + outputBytes: variants.reduce((sum, variant) => sum + variant.body.byteLength, 0), + wallMs: performance.now() - started, + peakRssKiB: process.resourceUsage().maxRSS, + }; + response = { ok: true, variants: variants.map((variant) => ({ ...variant, execution })) }; + } catch (error) { + if (!isImageInputRejection(error)) throw error; + response = { + ok: false, + message: + error instanceof Error ? error.message.slice(0, 1024) : "Image variant generation failed", + }; + } + process.send?.(response, () => process.disconnect()); +}); diff --git a/knowledge-fs/apps/api/src/isolated-image-variant-generator.test.ts b/knowledge-fs/apps/api/src/isolated-image-variant-generator.test.ts new file mode 100644 index 00000000000..431850d3585 --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-image-variant-generator.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { createIsolatedImageVariantGenerator } from "./isolated-image-variant-generator"; + +describe("isolated image variants", () => { + it("rejects incompatible variant names before starting any image work", () => { + expect(() => createIsolatedImageVariantGenerator({ variantName: "../escape" })).toThrow( + "Image variant name", + ); + }); + it("decodes in a child and returns separate preview and analysis variants", async () => { + const sharp = (await import("sharp")).default; + const body = await sharp({ + create: { width: 640, height: 320, channels: 3, background: "white" }, + }) + .png() + .toBuffer(); + const generator = createIsolatedImageVariantGenerator({ analysisMaxDimension: 512 }); + const variants = await generator.generate({ + body, + contentType: "image/png", + elementId: "figure", + }); + expect(variants.map(({ name, width, height }) => ({ name, width, height }))).toEqual([ + { name: "thumbnail", width: 320, height: 160 }, + { name: "analysis", width: 512, height: 256 }, + ]); + expect(variants[0]?.execution).toMatchObject({ + isolation: "child-process", + inputBytes: body.byteLength, + outputBytes: expect.any(Number), + wallMs: expect.any(Number), + peakRssKiB: expect.any(Number), + }); + }); + it("returns a terminal input error when the decoder rejects pixels", async () => { + const generator = createIsolatedImageVariantGenerator({ maxInputPixels: 100 }); + await expect( + generator.generate({ + body: new TextEncoder().encode( + '', + ), + contentType: "image/svg+xml", + elementId: "too-large", + }), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + }); + it("does not start a decoder after the execution lease was cancelled", async () => { + const controller = new AbortController(); + controller.abort(new Error("lease lost")); + await expect( + createIsolatedImageVariantGenerator().generate({ + body: new Uint8Array([1]), + contentType: "image/png", + elementId: "cancelled", + signal: controller.signal, + }), + ).rejects.toThrow("lease lost"); + }); +}); diff --git a/knowledge-fs/apps/api/src/isolated-image-variant-generator.ts b/knowledge-fs/apps/api/src/isolated-image-variant-generator.ts new file mode 100644 index 00000000000..18da7938f10 --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-image-variant-generator.ts @@ -0,0 +1,75 @@ +import { fork } from "node:child_process"; +import { ProviderInputError, ProviderResponseError } from "@knowledge/parsers"; +import { + type DocumentImageVariantGenerator, + type GeneratedDocumentImageVariant, + type SharpImageThumbnailVariantGeneratorOptions, + createSharpImageThumbnailVariantGenerator, +} from "../../../packages/api/src/document-image-variant-generator"; +import { + type ImageVariantRequest, + type ImageVariantResponse, + imageVariantMaxInputBytes, + validateImageVariantResponse, +} from "./image-variant-protocol"; +import { createIsolatedProcessExecutor } from "./isolated-process-executor"; + +const executor = createIsolatedProcessExecutor({ + maxConcurrency: 2, + maxInputBytes: imageVariantMaxInputBytes, + maxReservedBytes: 64 * 1024 * 1024, + maxRssBytes: 384 * 1024 * 1024, + maxQueued: 8, + timeoutMs: 30_000, + spawn: () => { + const compiled = import.meta.url.endsWith(".mjs"); + return fork( + new URL( + compiled ? "./image-variant-worker.mjs" : "./image-variant-worker.ts", + import.meta.url, + ), + [], + { + execArgv: ["--max-old-space-size=128", ...(compiled ? [] : ["--import", "tsx"])], + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + VIPS_CONCURRENCY: "1", + MALLOC_ARENA_MAX: "2", + }, + serialization: "advanced", + stdio: ["ignore", "ignore", "ignore", "ipc"], + }, + ); + }, +}); + +export function imageVariantsFromResponse( + value: unknown, +): readonly GeneratedDocumentImageVariant[] { + if (!validateImageVariantResponse(value)) + throw new ProviderResponseError("Image worker returned an invalid or over-budget response"); + if (!value.ok) throw new ProviderInputError(value.message); + return value.variants; +} + +export function createIsolatedImageVariantGenerator( + options: SharpImageThumbnailVariantGeneratorOptions = {}, +): DocumentImageVariantGenerator { + createSharpImageThumbnailVariantGenerator(options); // Validate configuration before accepting jobs. + if (options.variantName !== undefined && !/^[a-zA-Z0-9_-]{1,64}$/u.test(options.variantName)) + throw new Error( + "Image variant name must contain 1..64 ASCII letters, digits, hyphens or underscores", + ); + const frozenOptions = Object.freeze({ ...options }); + return { + generate: async ({ signal, ...input }) => { + const response = await executor.execute( + { input, options: frozenOptions }, + input.body.byteLength, + signal, + ); + return imageVariantsFromResponse(response); + }, + }; +} diff --git a/knowledge-fs/apps/api/src/isolated-process-executor.ts b/knowledge-fs/apps/api/src/isolated-process-executor.ts new file mode 100644 index 00000000000..af6175e728b --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-process-executor.ts @@ -0,0 +1,186 @@ +import type { ChildProcess } from "node:child_process"; +import { ProviderError, ProviderInputError, ProviderResponseError } from "@knowledge/parsers"; +import { readProcessRssBytes, superviseProcessMemory } from "./isolated-process-memory"; + +export interface IsolatedProcessOptions { + readonly maxConcurrency?: number; + readonly maxQueued?: number; + readonly maxInputBytes?: number; + readonly maxReservedBytes?: number; + readonly timeoutMs?: number; + readonly maxRssBytes?: number; + readonly readRssBytes?: (pid: number) => Promise; + readonly spawn: () => ChildProcess; +} + +/** Shared bounded process lifecycle; resolve/release only after the worker actually exits. */ +export function createIsolatedProcessExecutor({ + maxConcurrency = 2, + maxQueued = 32, + maxInputBytes = 50 * 1024 * 1024, + maxReservedBytes = 128 * 1024 * 1024, + timeoutMs = 600_000, + maxRssBytes, + readRssBytes = readProcessRssBytes, + spawn, +}: IsolatedProcessOptions) { + for (const value of [maxConcurrency, maxQueued, maxInputBytes, maxReservedBytes, timeoutMs]) { + if (!Number.isSafeInteger(value) || value < 1) + throw new Error("Invalid native parser isolation budget"); + } + if (maxRssBytes !== undefined && (!Number.isSafeInteger(maxRssBytes) || maxRssBytes < 1)) + throw new Error("Invalid worker RSS budget"); + let active = 0; + let reservedBytes = 0; + const queue: Array<() => void> = []; + const acquire = (signal: AbortSignal): Promise<() => void> => + new Promise((resolve, reject) => { + signal.throwIfAborted(); + const enter = () => { + signal.removeEventListener("abort", cancel); + active++; + let released = false; + resolve(() => { + if (released) return; + released = true; + active--; + queue.shift()?.(); + }); + }; + const cancel = () => { + const index = queue.indexOf(enter); + if (index >= 0) queue.splice(index, 1); + reject(signal.reason); + }; + if (active < maxConcurrency) enter(); + else if (queue.length >= maxQueued) + reject( + new ProviderError("Native parser admission queue is full", { + code: "provider_rate_limited", + retryable: true, + }), + ); + else { + queue.push(enter); + signal.addEventListener("abort", cancel, { once: true }); + } + }); + + return { + async execute(request: TRequest, inputBytes: number, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + if (!Number.isSafeInteger(inputBytes) || inputBytes < 0) + throw new ProviderInputError( + "Isolated process input byte count must be a non-negative safe integer", + ); + if (inputBytes > maxInputBytes) + throw new ProviderInputError(`Native parser input exceeds maxInputBytes=${maxInputBytes}`); + if (reservedBytes + inputBytes > maxReservedBytes) + throw new ProviderError("Native parser reserved-input byte budget is full", { + code: "provider_rate_limited", + retryable: true, + }); + reservedBytes += inputBytes; + const controller = new AbortController(); + const cancel = () => controller.abort(signal?.reason); + signal?.addEventListener("abort", cancel, { once: true }); + const timer = setTimeout( + () => + controller.abort( + new ProviderError("Native parser wall-clock budget exceeded", { + code: "provider_timeout", + retryable: false, + requestOutcomeAmbiguous: false, + }), + ), + timeoutMs, + ); + let release: (() => void) | undefined; + try { + release = await acquire(controller.signal); + controller.signal.throwIfAborted(); + return await runChild( + spawn, + request, + controller.signal, + maxRssBytes, + readRssBytes, + ); + } finally { + release?.(); + reservedBytes -= inputBytes; + clearTimeout(timer); + signal?.removeEventListener("abort", cancel); + } + }, + }; +} + +function runChild( + spawn: () => ChildProcess, + request: TRequest, + signal: AbortSignal, + maxRssBytes: number | undefined, + readRssBytes: (pid: number) => Promise, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(); + let response: TResponse | undefined; + let failure: unknown; + let failed = false; + const stopMemory = + child.pid !== undefined && maxRssBytes !== undefined + ? superviseProcessMemory({ + pid: child.pid, + maxRssBytes, + readRssBytes, + stop: (reason) => { + if (failed || signal.aborted) return; + failed = true; + failure = reason; + child.kill("SIGKILL"); + }, + }) + : () => {}; + const cancel = () => { + failed = true; + failure = signal.reason; + child.kill("SIGKILL"); + }; + signal.addEventListener("abort", cancel, { once: true }); + child.once("error", (error) => { + failed = true; + failure = error; + child.kill("SIGKILL"); + }); + child.once("message", (message: TResponse) => { + response = message; + }); + child.once("close", (code: number | null, exitSignal: NodeJS.Signals | null) => { + stopMemory(); + signal.removeEventListener("abort", cancel); + if (failed) reject(failure); + else if (response === undefined || code !== 0 || exitSignal !== null) + reject( + new ProviderResponseError("Isolated worker exited without a valid completed response"), + ); + else resolve(response); + }); + if (signal.aborted) cancel(); + else { + try { + child.send(request as Parameters[0], (error) => { + if (error) { + failed = true; + failure = error; + child.kill("SIGKILL"); + } + }); + } catch (error) { + failed = true; + failure = error; + child.kill("SIGKILL"); + } + } + }); +} diff --git a/knowledge-fs/apps/api/src/isolated-process-memory-reader.test.ts b/knowledge-fs/apps/api/src/isolated-process-memory-reader.test.ts new file mode 100644 index 00000000000..d7eee1e2832 --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-process-memory-reader.test.ts @@ -0,0 +1,84 @@ +import { open } from "node:fs/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + parseProcStatusRss, + readProcessRssBytes, + superviseProcessMemory, +} from "./isolated-process-memory"; + +vi.mock("node:fs/promises", () => ({ open: vi.fn() })); +const originalPlatform = process.platform; +afterEach(() => { + Object.defineProperty(process, "platform", { value: originalPlatform }); + vi.useRealTimers(); + vi.clearAllMocks(); +}); + +describe("bounded Linux RSS reader", () => { + it("parses KiB and handles zombie status without invented values", () => { + expect(parseProcStatusRss("Name: node\nVmRSS:\t1200 kB\n")).toBe(1200 * 1024); + expect(parseProcStatusRss("Name: node\nState: Z\n")).toBeUndefined(); + expect(() => parseProcStatusRss("VmRSS: 99999999999999999 kB\n")).toThrow("counter"); + }); + it("does not attempt procfs on non-Linux adapters", async () => { + Object.defineProperty(process, "platform", { value: "darwin" }); + expect(await readProcessRssBytes(1)).toBeUndefined(); + expect(open).not.toHaveBeenCalled(); + }); + it("uses a bounded read and closes the descriptor", async () => { + Object.defineProperty(process, "platform", { value: "linux" }); + const close = vi.fn(); + const read = vi.fn(async (buffer: Buffer) => ({ bytesRead: buffer.write("VmRSS: 10 kB\n") })); + vi.mocked(open).mockResolvedValue({ read, close } as never); + expect(await readProcessRssBytes(123)).toBe(10240); + expect(open).toHaveBeenCalledWith("/proc/123/status", "r"); + expect(read.mock.calls[0]?.[0].length).toBe(16 * 1024); + expect(close).toHaveBeenCalledOnce(); + }); + it("handles exit races and rejects unavailable or oversized metadata", async () => { + Object.defineProperty(process, "platform", { value: "linux" }); + await expect(readProcessRssBytes(-1)).rejects.toThrow("PID"); + vi.mocked(open).mockRejectedValueOnce(Object.assign(new Error(), { code: "ENOENT" })); + expect(await readProcessRssBytes(123)).toBeUndefined(); + vi.mocked(open).mockRejectedValueOnce(new Error("permission")); + await expect(readProcessRssBytes(123)).rejects.toThrow("permission"); + const close = vi.fn(); + vi.mocked(open).mockResolvedValueOnce({ + read: vi.fn(async () => ({ bytesRead: 16 * 1024 })), + close, + } as never); + await expect(readProcessRssBytes(123)).rejects.toThrow("metadata budget"); + expect(close).toHaveBeenCalledOnce(); + }); + it("never overlaps probes and ignores a late sample after disposal", async () => { + vi.useFakeTimers(); + let resolve: (bytes: number) => void = () => {}; + const readRssBytes = vi.fn( + () => + new Promise((done) => { + resolve = done; + }), + ); + const stop = vi.fn(); + const dispose = superviseProcessMemory({ pid: 1, maxRssBytes: 1, readRssBytes, stop }); + await vi.advanceTimersByTimeAsync(200); + expect(readRssBytes).toHaveBeenCalledOnce(); + dispose(); + resolve(2000); + await Promise.resolve(); + expect(stop).not.toHaveBeenCalled(); + }); + it("fails closed on a live probe error and accepts values below the budget", async () => { + vi.useFakeTimers(); + const stop = vi.fn(); + const readRssBytes = vi + .fn() + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(10) + .mockRejectedValueOnce(new Error()); + const dispose = superviseProcessMemory({ pid: 1, maxRssBytes: 100, readRssBytes, stop }); + await vi.advanceTimersByTimeAsync(100); + expect(stop).toHaveBeenCalledWith(expect.objectContaining({ code: "provider_request_failed" })); + dispose(); + }); +}); diff --git a/knowledge-fs/apps/api/src/isolated-process-memory.test.ts b/knowledge-fs/apps/api/src/isolated-process-memory.test.ts new file mode 100644 index 00000000000..e59f310842a --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-process-memory.test.ts @@ -0,0 +1,24 @@ +import type { ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import { createIsolatedProcessExecutor } from "./isolated-process-executor"; + +describe("isolated parser resident memory supervision", () => { + it("kills the worker when native allocations exceed the sampled RSS budget", async () => { + const child = new EventEmitter() as ChildProcess; + Object.defineProperty(child, "pid", { value: 123 }); + child.send = vi.fn(() => true) as unknown as ChildProcess["send"]; + child.kill = vi.fn(() => { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + return true; + }); + const executor = createIsolatedProcessExecutor({ + spawn: () => child, + timeoutMs: 500, + maxRssBytes: 1000, + readRssBytes: async () => 2000, + }); + await expect(executor.execute({}, 0)).rejects.toThrow("resident-memory"); + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + }); +}); diff --git a/knowledge-fs/apps/api/src/isolated-process-memory.ts b/knowledge-fs/apps/api/src/isolated-process-memory.ts new file mode 100644 index 00000000000..3ae269c48b1 --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-process-memory.ts @@ -0,0 +1,71 @@ +import { open } from "node:fs/promises"; +import { ProviderError, ProviderInputError } from "@knowledge/parsers"; + +export function parseProcStatusRss(status: string): number | undefined { + const value = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; + if (value === undefined) return undefined; // Zombie processes have no VmRSS. + const bytes = Number(value) * 1024; + if (!Number.isSafeInteger(bytes)) throw new Error("Invalid process RSS counter"); + return bytes; +} + +export async function readProcessRssBytes(pid: number): Promise { + if (process.platform !== "linux") return undefined; + if (!Number.isSafeInteger(pid) || pid < 1) throw new Error("Invalid worker PID"); + try { + const file = await open(`/proc/${pid}/status`, "r"); + try { + const buffer = Buffer.alloc(16 * 1024); + const { bytesRead } = await file.read(buffer, 0, buffer.length, 0); + if (bytesRead === buffer.length) throw new Error("Process status exceeds metadata budget"); + return parseProcStatusRss(buffer.toString("utf8", 0, bytesRead)); + } finally { + await file.close(); + } + } catch (error) { + if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") + return undefined; + throw error; + } +} + +/** Sampled stop for native allocations; not a hard cgroup or hostile-code sandbox. */ +export function superviseProcessMemory(input: { + readonly pid: number; + readonly maxRssBytes: number; + readonly readRssBytes: (pid: number) => Promise; + readonly stop: (reason: Error) => void; +}): () => void { + let stopped = false; + let sampling = false; + const sample = async () => { + if (stopped || sampling) return; + sampling = true; + try { + const bytes = await input.readRssBytes(input.pid); + if (!stopped && bytes !== undefined && bytes > input.maxRssBytes) + input.stop( + new ProviderInputError("Document parser worker resident-memory budget exceeded"), + ); + } catch { + if (!stopped) + input.stop( + new ProviderError("Document parser worker memory supervision unavailable", { + code: "provider_request_failed", + retryable: false, + }), + ); + } finally { + sampling = false; + } + }; + const timer = setInterval(() => { + void sample(); + }, 50); + timer.unref(); + void sample(); + return () => { + stopped = true; + clearInterval(timer); + }; +} diff --git a/knowledge-fs/apps/api/src/isolated-process-review.test.ts b/knowledge-fs/apps/api/src/isolated-process-review.test.ts new file mode 100644 index 00000000000..1815aca9d87 --- /dev/null +++ b/knowledge-fs/apps/api/src/isolated-process-review.test.ts @@ -0,0 +1,111 @@ +import type { ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { createNativeStructuredDataParser } from "@knowledge/parsers"; +import { describe, expect, it, vi } from "vitest"; +import { createIsolatedProcessExecutor } from "./isolated-process-executor"; +import { createNativeParserIsolation } from "./native-parser-isolation"; + +function fakeChild() { + const child = new EventEmitter() as ChildProcess; + child.send = vi.fn(() => true) as unknown as ChildProcess["send"]; + child.kill = vi.fn(() => { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + return true; + }); + return child; +} +const input = { + body: new TextEncoder().encode("{}"), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "x.json", + mimeType: "application/json", + version: 1, +}; + +describe("isolated process boundary regressions", () => { + it.each([-1, Number.NaN, Number.POSITIVE_INFINITY, 0.5])( + "rejects invalid input byte accounting %s before spawning", + async (inputBytes) => { + const spawn = vi.fn(() => fakeChild()); + const executor = createIsolatedProcessExecutor({ spawn, timeoutMs: 5 }); + await expect(executor.execute({}, inputBytes)).rejects.toMatchObject({ + code: "provider_input", + }); + expect(spawn).not.toHaveBeenCalled(); + }, + ); + it.each([ + [1, null], + [null, "SIGKILL"], + ])("rejects a response followed by abnormal exit %s/%s", async (code, signal) => { + const child = fakeChild(); + const executor = createIsolatedProcessExecutor({ spawn: () => child }); + const result = expect(executor.execute({}, 1)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + child.emit("message", { result: "not publishable" }); + child.emit("close", code, signal); + await result; + }); + it("classifies a clean exit without a result as a worker protocol failure", async () => { + const child = fakeChild(); + const executor = createIsolatedProcessExecutor({ spawn: () => child }); + const result = expect(executor.execute({}, 1)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + child.emit("close", 0, null); + await result; + }); + it.each([null, {}, { ok: true, artifact: {} }, { ok: false, message: null }])( + "classifies malformed worker output %#", + async (response) => { + const child = fakeChild(); + const parser = createNativeParserIsolation({ spawn: () => child }).wrap( + createNativeStructuredDataParser(), + {}, + ); + const result = expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + child.emit("message", response); + child.emit("close", 0, null); + await result; + }, + ); + it.each(["provider_input", "provider_response_invalid"])( + "preserves the worker's typed %s failure", + async (errorCode) => { + const child = fakeChild(); + const parser = createNativeParserIsolation({ spawn: () => child }).wrap( + createNativeStructuredDataParser(), + {}, + ); + const result = expect(parser.parse(input)).rejects.toMatchObject({ code: errorCode }); + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + child.emit("message", { ok: false, message: "failure", errorCode }); + child.emit("close", 0, null); + await result; + }, + ); + it("rejects non-serializable callback options before accepting work", () => { + const runtimeOptions = { maxInputBytes: 15, now: () => "2026-01-01T00:00:00Z" }; + expect(() => + createNativeParserIsolation().wrap(createNativeStructuredDataParser(), runtimeOptions), + ).toThrow("serializable"); + }); + it.each([null, false, 0])("preserves a falsy abort reason %s", async (reason) => { + const child = fakeChild(); + const executor = createIsolatedProcessExecutor({ spawn: () => child }); + const controller = new AbortController(); + const result = expect(executor.execute({}, 1, controller.signal)).rejects.toBe(reason); + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + controller.abort(reason); + await result; + }); +}); diff --git a/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts b/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts index d2562833cf8..e160512b556 100644 --- a/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts +++ b/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts @@ -119,7 +119,7 @@ describe("createApiMultimodalAnswerOptions", () => { await objectStorage.putObject({ body: new Uint8Array([1, 2, 3]), contentType: "image/png", - key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png", + key: "tenant/spaces/space/documents/doc/assets/chart.png", }); globalThis.fetch = (async (input, init) => { const request = new Request(input, init); diff --git a/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts b/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts index 5eebd7f7832..48e41a651a1 100644 --- a/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts +++ b/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts @@ -37,7 +37,7 @@ describe("createApiMultimodalEnrichmentOptions", () => { await objectStorage.putObject({ body: new Uint8Array([1, 2, 3]), contentType: "image/png", - key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png", + key: "tenant/spaces/space/documents/doc/assets/chart-analysis.png", }); globalThis.fetch = (async (input, init) => { const request = new Request(input, init); @@ -88,6 +88,10 @@ describe("createApiMultimodalEnrichmentOptions", () => { contentType: "image/png", objectKey: "tenant/spaces/space/documents/doc/assets/chart.png", variants: { + analysis: { + contentType: "image/png", + objectKey: "tenant/spaces/space/documents/doc/assets/chart-analysis.png", + }, thumbnail: { contentType: "image/png", objectKey: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png", diff --git a/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts b/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts index d407f509ff0..946fa7e8d6d 100644 --- a/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts +++ b/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts @@ -450,8 +450,9 @@ async function objectBackedImageSource({ readonly objectStorage: KnowledgeGatewayOptions["adapter"]["objectStorage"]; }): Promise<{ base64Data: string; mimeType: string } | undefined> { const assetRef = input.item.assetRef; - const thumbnail = assetRef?.variants?.thumbnail; - const candidate = thumbnail?.objectKey ? thumbnail : assetRef; + if (assetRef?.analysisUnavailable !== undefined) return undefined; + const analysis = assetRef?.variants?.analysis; + const candidate = analysis?.objectKey ? analysis : assetRef; const objectKey = candidate?.objectKey; const contentType = candidate?.contentType; diff --git a/knowledge-fs/apps/api/src/multimodal-options.ts b/knowledge-fs/apps/api/src/multimodal-options.ts index 8bb870c23ee..0900a188151 100644 --- a/knowledge-fs/apps/api/src/multimodal-options.ts +++ b/knowledge-fs/apps/api/src/multimodal-options.ts @@ -2,8 +2,8 @@ import { type DocumentImageVariantGenerator, type DocumentPdfRasterizer, createPopplerPdfRasterizer, - createSharpImageThumbnailVariantGenerator, } from "@knowledge/api"; +import { createIsolatedImageVariantGenerator } from "./isolated-image-variant-generator"; export interface ApiMultimodalEnv { readonly DIFY_ROOT_KNOWLEDGE_DOCUMENT_MATERIALIZATION_MAX_CONCURRENCY_OVERRIDE?: @@ -149,7 +149,14 @@ function imageThumbnailOptions( } return { - documentMultimodalImageVariantGenerator: createSharpImageThumbnailVariantGenerator({ + documentMultimodalImageVariantGenerator: createIsolatedImageVariantGenerator({ + analysisMaxDimension: Math.max( + 2048, + positiveIntegerEnv( + env.KNOWLEDGE_IMAGE_THUMBNAIL_MAX_DIMENSION ?? "320", + "KNOWLEDGE_IMAGE_THUMBNAIL_MAX_DIMENSION", + ), + ), ...(env.KNOWLEDGE_IMAGE_THUMBNAIL_MAX_DIMENSION !== undefined ? { maxDimension: positiveIntegerEnv( diff --git a/knowledge-fs/apps/api/src/native-parser-busy.fixture.mjs b/knowledge-fs/apps/api/src/native-parser-busy.fixture.mjs new file mode 100644 index 00000000000..2f86ec114fa --- /dev/null +++ b/knowledge-fs/apps/api/src/native-parser-busy.fixture.mjs @@ -0,0 +1,5 @@ +// Test-only bounded by the parent process deadline; deliberately cannot service its event loop. +process.once("message", () => { + process.send({ started: true }); + while (true) Math.sqrt(2); +}); diff --git a/knowledge-fs/apps/api/src/native-parser-isolation.test.ts b/knowledge-fs/apps/api/src/native-parser-isolation.test.ts new file mode 100644 index 00000000000..beda306062a --- /dev/null +++ b/knowledge-fs/apps/api/src/native-parser-isolation.test.ts @@ -0,0 +1,244 @@ +import { type ChildProcess, fork } from "node:child_process"; +import { EventEmitter } from "node:events"; +import { createNativeStructuredDataParser } from "@knowledge/parsers"; +import { describe, expect, it, vi } from "vitest"; +import { createNativeParserIsolation } from "./native-parser-isolation"; + +const input = { + body: new TextEncoder().encode('{"value":"preserved"}'), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "example.json", + mimeType: "application/json", + version: 1, +}; + +function childFixture() { + const child = new EventEmitter() as ChildProcess; + child.send = vi.fn(() => true) as unknown as ChildProcess["send"]; + child.kill = vi.fn(() => { + queueMicrotask(() => child.emit("close", null, "SIGKILL")); + return true; + }); + return child; +} + +describe("native parser process isolation", () => { + it("runs the real parser outside the API process and preserves its semantic identity", async () => { + const native = createNativeStructuredDataParser(); + const isolated = createNativeParserIsolation().wrap(native, {}); + const expected = await native.parse(input); + const actual = await isolated.parse(input); + expect(actual.elements.map(({ id: _id, ...element }) => element)).toEqual( + expected.elements.map(({ id: _id, ...element }) => element), + ); + expect(actual.artifactHash).toEqual(expected.artifactHash); + expect(isolated.policyFingerprint?.(input)).toBe(native.policyFingerprint?.(input)); + expect(actual.metadata.parserExecution).toMatchObject({ isolation: "child-process" }); + expect(actual.metadata.parserExecution).toHaveProperty("peakRssKiB"); + }, 15_000); + + it("terminates synchronous work on cancellation before releasing its shared slot", async () => { + const first = childFixture(); + const second = childFixture(); + const spawn = vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second); + const isolation = createNativeParserIsolation({ spawn, maxConcurrency: 1 }); + const parser = isolation.wrap(createNativeStructuredDataParser(), {}); + const controller = new AbortController(); + const aborted = expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow( + "cancelled", + ); + await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce()); + const queuedController = new AbortController(); + const queued = expect( + parser.parse({ ...input, signal: queuedController.signal }), + ).rejects.toThrow("queued"); + queuedController.abort(new Error("queued")); + await queued; + controller.abort(new Error("cancelled")); + await aborted; + expect(first.kill).toHaveBeenCalledWith("SIGKILL"); + expect(spawn).toHaveBeenCalledOnce(); + }); + + it("enforces a real wall deadline on a non-responsive child", async () => { + const child = childFixture(); + const parser = createNativeParserIsolation({ spawn: () => child, timeoutMs: 20 }).wrap( + createNativeStructuredDataParser(), + {}, + ); + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_timeout", + requestOutcomeAmbiguous: false, + retryable: false, + }); + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + }); + + it("never spawns for cancelled or oversized input", async () => { + const spawn = vi.fn(); + const parser = createNativeParserIsolation({ spawn, maxInputBytes: 1 }).wrap( + createNativeStructuredDataParser(), + {}, + ); + await expect(parser.parse(input)).rejects.toMatchObject({ code: "provider_input" }); + await expect( + parser.parse({ ...input, signal: AbortSignal.abort(new Error("cancelled")) }), + ).rejects.toThrow("cancelled"); + expect(spawn).not.toHaveBeenCalled(); + }); + + it("kills an actual CPU-bound process, not just its request promise", async () => { + const controller = new AbortController(); + let pid: number | undefined; + const parser = createNativeParserIsolation({ + timeoutMs: 5_000, + spawn: () => { + const child = fork(new URL("./native-parser-busy.fixture.mjs", import.meta.url), [], { + execArgv: [], + stdio: ["ignore", "ignore", "ignore", "ipc"], + }); + pid = child.pid; + child.once("message", () => controller.abort(new Error("actual CPU cancellation"))); + return child; + }, + }).wrap(createNativeStructuredDataParser(), {}); + await expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow( + "actual CPU cancellation", + ); + expect(pid).toBeTypeOf("number"); + expect(() => process.kill(pid as number, 0)).toThrow(); + }); + + it("bounds aggregate input retention before joining the queue", async () => { + const child = childFixture(); + const spawn = vi.fn(() => child); + const parser = createNativeParserIsolation({ + spawn, + maxReservedBytes: input.body.byteLength, + }).wrap(createNativeStructuredDataParser(), {}); + const controller = new AbortController(); + const first = expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow( + "done", + ); + await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce()); + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_rate_limited", + retryable: true, + }); + controller.abort(new Error("done")); + await first; + }); + + it("bounds queue length and frees the slot on process failure", async () => { + const child = childFixture(); + const spawn = vi.fn(() => child); + const parser = createNativeParserIsolation({ spawn, maxConcurrency: 1, maxQueued: 1 }).wrap( + createNativeStructuredDataParser(), + {}, + ); + const first = expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + }); + await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce()); + const controller = new AbortController(); + const second = expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow( + "queue cancellation", + ); + await expect(parser.parse(input)).rejects.toMatchObject({ code: "provider_rate_limited" }); + controller.abort(new Error("queue cancellation")); + await second; + child.emit("close", 1, null); + await first; + }); + + it("preserves typed format errors and rejects invalid worker budgets", async () => { + for (const maxConcurrency of [0, Number.NaN, 1.1, Number.POSITIVE_INFINITY]) + expect(() => createNativeParserIsolation({ maxConcurrency })).toThrow("Invalid"); + const child = childFixture(); + const parser = createNativeParserIsolation({ spawn: () => child }).wrap( + createNativeStructuredDataParser(), + {}, + ); + const result = expect(parser.parse(input)).rejects.toMatchObject({ + code: "document_parser_unsupported_type", + }); + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + child.emit("message", { + ok: false, + message: "unsupported", + errorCode: "document_parser_unsupported_type", + }); + child.emit("close", 0, null); + await result; + }); + + it.each(["event", "callback", "throw", "input"])( + "cleans up %s worker failures without leaking admission", + async (mode) => { + const child = childFixture(); + if (mode === "callback" || mode === "throw") { + child.send = vi.fn((_message, callback) => { + if (mode === "throw") throw new Error("IPC failed"); + callback(new Error("IPC failed")); + return false; + }) as unknown as ChildProcess["send"]; + } + const parser = createNativeParserIsolation({ spawn: () => child }).wrap( + createNativeStructuredDataParser(), + {}, + ); + const result = expect(parser.parse(input)).rejects.toThrow( + mode === "input" ? "bad input" : "IPC failed", + ); + if (mode === "event" || mode === "input") { + await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce()); + if (mode === "event") child.emit("error", new Error("IPC failed")); + else { + child.emit("message", { ok: false, message: "bad input", errorCode: "provider_input" }); + child.emit("close", 0, null); + } + } + await result; + }, + ); + + it("advances queued work only after the preceding process closes", async () => { + const first = childFixture(); + const second = childFixture(); + const spawn = vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second); + const parser = createNativeParserIsolation({ spawn, maxConcurrency: 1 }).wrap( + createNativeStructuredDataParser(), + {}, + ); + const failed = expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + }); + const artifact = await createNativeStructuredDataParser().parse(input); + const next = parser.parse(input); + await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce()); + first.emit("close", 1, null); + await failed; + await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(2)); + second.emit("message", { ok: true, artifact }); + second.emit("close", 0, null); + expect(await next).toEqual(artifact); + }); + + it("handles cancellation during spawn and rejects accidental remote adapters", async () => { + expect(() => + createNativeParserIsolation().wrap({ kind: "unstructured", parse: vi.fn() }, {}), + ).toThrow("remote"); + const controller = new AbortController(); + const child = childFixture(); + const parser = createNativeParserIsolation({ + spawn: () => { + controller.abort(new Error("spawn cancelled")); + return child; + }, + }).wrap(createNativeStructuredDataParser(), {}); + await expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow( + "spawn cancelled", + ); + expect(child.send).not.toHaveBeenCalled(); + }); +}); diff --git a/knowledge-fs/apps/api/src/native-parser-isolation.ts b/knowledge-fs/apps/api/src/native-parser-isolation.ts new file mode 100644 index 00000000000..028b7119969 --- /dev/null +++ b/knowledge-fs/apps/api/src/native-parser-isolation.ts @@ -0,0 +1,73 @@ +import { type ChildProcess, fork } from "node:child_process"; +import { + type ParserAdapter, + ProviderInputError, + ProviderResponseError, + ProviderUnsupportedFileTypeError, +} from "@knowledge/parsers"; +import { + type IsolatedProcessOptions, + createIsolatedProcessExecutor, +} from "./isolated-process-executor"; +import { + type NativeParserRequest, + type SerializableNativeParserOptions, + decodeNativeParserResponse, +} from "./native-parser-protocol"; + +function spawnNativeParser(): ChildProcess { + const compiled = import.meta.url.endsWith(".mjs"); + return fork( + new URL(compiled ? "./native-parser-worker.mjs" : "./native-parser-worker.ts", import.meta.url), + [], + { + execArgv: ["--max-old-space-size=256", ...(compiled ? [] : ["--import", "tsx"])], + // Native parsers need neither provider credentials nor arbitrary NODE_OPTIONS hooks. + env: { PATH: process.env.PATH, SYSTEMROOT: process.env.SYSTEMROOT }, + serialization: "advanced", + stdio: ["ignore", "ignore", "ignore", "ipc"], + }, + ); +} + +export function createNativeParserIsolation( + options: Omit & { readonly spawn?: () => ChildProcess } = {}, +) { + const executor = createIsolatedProcessExecutor({ + maxRssBytes: 512 * 1024 * 1024, + ...options, + spawn: options.spawn ?? spawnNativeParser, + }); + return { + wrap(parser: ParserAdapter, options: SerializableNativeParserOptions): ParserAdapter { + if (parser.kind === "unstructured") + throw new Error("Native isolation cannot wrap a remote parser"); + if (Object.values(options).some((value) => typeof value === "function")) + throw new Error( + "Native isolation options must be serializable (callbacks are not supported)", + ); + const kind = parser.kind; + const frozenOptions = Object.freeze({ ...options }); + return { + ...parser, + async parse(input) { + const { signal, ...serializedInput } = input; + const payload = await executor.execute( + { input: serializedInput, kind, options: frozenOptions }, + input.body.byteLength, + signal, + ); + const response = decodeNativeParserResponse(payload); + if (!response) + throw new ProviderResponseError("Native parser worker returned an invalid response"); + if (response.ok) return response.artifact; + throw response.errorCode === "document_parser_unsupported_type" + ? new ProviderUnsupportedFileTypeError(response.message) + : response.errorCode === "provider_input" + ? new ProviderInputError(response.message) + : new ProviderResponseError(response.message); + }, + }; + }, + }; +} diff --git a/knowledge-fs/apps/api/src/native-parser-protocol.test.ts b/knowledge-fs/apps/api/src/native-parser-protocol.test.ts new file mode 100644 index 00000000000..a747dd85d7b --- /dev/null +++ b/knowledge-fs/apps/api/src/native-parser-protocol.test.ts @@ -0,0 +1,22 @@ +import { ProviderInputError, ProviderUnsupportedFileTypeError } from "@knowledge/parsers"; +import { describe, expect, it } from "vitest"; +import { decodeNativeParserResponse, encodeNativeParserFailure } from "./native-parser-protocol"; + +describe("native parser failure protocol", () => { + it.each([ + [new ProviderInputError("invalid source"), "provider_input"], + [new ProviderUnsupportedFileTypeError("unsupported"), "document_parser_unsupported_type"], + [new Error("unexpected implementation failure"), "provider_response_invalid"], + [null, "provider_response_invalid"], + ])("preserves only known failure classifications %#", (error, errorCode) => { + const result = encodeNativeParserFailure(error); + expect(result).toMatchObject({ ok: false, errorCode }); + expect(decodeNativeParserResponse(result)).toEqual(result); + }); + it("bounds error text and rejects unrecognized failure codes", () => { + expect(encodeNativeParserFailure(new Error("x".repeat(2000))).message).toHaveLength(1024); + expect( + decodeNativeParserResponse({ ok: false, message: "x", errorCode: "unknown" }), + ).toBeUndefined(); + }); +}); diff --git a/knowledge-fs/apps/api/src/native-parser-protocol.ts b/knowledge-fs/apps/api/src/native-parser-protocol.ts new file mode 100644 index 00000000000..2c547d13301 --- /dev/null +++ b/knowledge-fs/apps/api/src/native-parser-protocol.ts @@ -0,0 +1,68 @@ +import { type ParseArtifact, ParseArtifactSchema } from "@knowledge/core"; +import type { + ParseDocumentInput, + ParserKind, + StructuredDataParserOptions, +} from "@knowledge/parsers"; + +export interface NativeParserRequest { + readonly input: Omit; + readonly kind: Exclude; + readonly options: SerializableNativeParserOptions; +} + +export type SerializableNativeParserOptions = Omit< + StructuredDataParserOptions, + "generateId" | "now" +>; + +export type NativeParserResponse = + | { readonly ok: true; readonly artifact: ParseArtifact } + | { + readonly ok: false; + readonly message: string; + readonly errorCode: + | "provider_input" + | "document_parser_unsupported_type" + | "provider_response_invalid"; + }; + +export function encodeNativeParserFailure( + error: unknown, +): Extract { + const code = + typeof error === "object" && error !== null && "code" in error ? error.code : undefined; + return { + ok: false, + message: + error instanceof Error ? error.message.slice(0, 1024) : "Native document parsing failed", + errorCode: + code === "provider_input" || code === "document_parser_unsupported_type" + ? code + : "provider_response_invalid", + }; +} + +export function decodeNativeParserResponse(value: unknown): NativeParserResponse | undefined { + if (typeof value !== "object" || value === null || !("ok" in value)) return undefined; + if (value.ok === true && "artifact" in value) { + const parsed = ParseArtifactSchema.safeParse(value.artifact); + return parsed.success ? { ok: true, artifact: parsed.data } : undefined; + } + if ( + value.ok === false && + "message" in value && + typeof value.message === "string" && + value.message.length <= 1024 && + "errorCode" in value && + (value.errorCode === "provider_input" || + value.errorCode === "document_parser_unsupported_type" || + value.errorCode === "provider_response_invalid") + ) { + return { ok: false, message: value.message, errorCode: value.errorCode }; + } + return undefined; +} + +// Serialized output is bounded independently from the child's V8 heap ceiling. +export const nativeParserOutputBytes = 32 * 1024 * 1024; diff --git a/knowledge-fs/apps/api/src/native-parser-worker.ts b/knowledge-fs/apps/api/src/native-parser-worker.ts new file mode 100644 index 00000000000..adddad1ffa0 --- /dev/null +++ b/knowledge-fs/apps/api/src/native-parser-worker.ts @@ -0,0 +1,48 @@ +import { + createNativeHtmlParser, + createNativeMarkdownParser, + createNativeStructuredDataParser, +} from "@knowledge/parsers"; +import { + type NativeParserRequest, + type NativeParserResponse, + encodeNativeParserFailure, + nativeParserOutputBytes, +} from "./native-parser-protocol"; + +// A single immutable request per process. No providers, credentials or document filesystem IO. +process.once("message", async (request: NativeParserRequest) => { + const started = performance.now(); + let response: NativeParserResponse; + try { + const parser = + request.kind === "native-html" + ? createNativeHtmlParser(request.options) + : request.kind === "native-markdown" + ? createNativeMarkdownParser(request.options) + : createNativeStructuredDataParser(request.options); + const artifact = await parser.parse(request.input); + const outputBytes = Buffer.byteLength(JSON.stringify(artifact)); + if (outputBytes > nativeParserOutputBytes) + throw new Error("Native parser output exceeds 32 MiB"); + response = { + ok: true, + artifact: { + ...artifact, + metadata: { + ...artifact.metadata, + parserExecution: { + isolation: "child-process", + inputBytes: request.input.body.byteLength, + outputBytes, + wallMs: performance.now() - started, + peakRssKiB: process.resourceUsage().maxRSS, + }, + }, + }, + }; + } catch (error) { + response = encodeNativeParserFailure(error); + } + process.send?.(response, () => process.disconnect()); +}); diff --git a/knowledge-fs/apps/api/src/parser-options.test.ts b/knowledge-fs/apps/api/src/parser-options.test.ts index 881bc4fe855..68de6aeb5d7 100644 --- a/knowledge-fs/apps/api/src/parser-options.test.ts +++ b/knowledge-fs/apps/api/src/parser-options.test.ts @@ -1,4 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; + +import { ProviderInputError } from "@knowledge/parsers"; import type { Dispatcher } from "undici"; @@ -7,40 +9,109 @@ import { createApiUnstructuredConcurrencyOptions, createNodeUnstructuredFetch, } from "./parser-options"; +import { createPdfParserPreflight } from "./pdf-parser-preflight"; + +// Parser routing/timeout tests use tiny transport fixtures. Real PDF inspection, child process +// limits and page geometry are exercised independently in pdf-parser-preflight.test.ts. +vi.mock("./pdf-parser-preflight", () => ({ + createPdfParserPreflight: vi.fn(() => ({ + check: vi.fn(async () => {}), + policyFingerprint: "pdf-preflight-test", + })), +})); const encoder = new TextEncoder(); function ordinaryDocx(): Uint8Array { - const filename = encoder.encode("word/document.xml"); - const localHeader = new Uint8Array(30); - const centralDirectory = new Uint8Array(46 + filename.byteLength); - const endOfCentralDirectory = new Uint8Array(22); - const localView = new DataView(localHeader.buffer); - const centralView = new DataView(centralDirectory.buffer); - const endView = new DataView(endOfCentralDirectory.buffer); - - localView.setUint32(0, 0x04034b50, true); - centralView.setUint32(0, 0x02014b50, true); - centralView.setUint32(20, 1, true); - centralView.setUint32(24, 1, true); - centralView.setUint16(28, filename.byteLength, true); - centralDirectory.set(filename, 46); - endView.setUint32(0, 0x06054b50, true); - endView.setUint16(8, 1, true); - endView.setUint16(10, 1, true); - endView.setUint32(12, centralDirectory.byteLength, true); - endView.setUint32(16, localHeader.byteLength, true); - - const body = new Uint8Array( - localHeader.byteLength + centralDirectory.byteLength + endOfCentralDirectory.byteLength, + // Real deflated OOXML ZIP, generated with fflate.zipSync (fixed mtime), containing + // [Content_Types].xml, _rels/.rels and a one-paragraph word/document.xml. + return new Uint8Array( + Buffer.from( + "UEsDBBQAAAAIAACYn090JJxTuwAAAD4BAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbJWQuQ7CMAyGX6XKiqgRAwNquwArMPACVuq2EbkUm+vtSTk6sDHa//FZrk6PSFzcnfVcq0EkrgFYD+SQyxDJZ6ULyaHkMfUQUZ+xJ1guFivQwQt5mcvYoZpqSx1erBS7e16zCb5WiSyrYvM2jqxaYYzWaJSsw9W3P5T5h1Dm5MvDg4k8ywYFTXW4UkqmpeKISfboch3cQmqhDfriMqIcjX/xQtcZTVN+bIspaGI2vne2nBSHxn/vgNfbmidQSwMEFAAAAAgAAJifT2F7L0OIAAAA8gAAAAsAAABfcmVscy8ucmVsc43POQ7CMBAF0KtEPkAmUFCg2BVNWsQFLHu8iHjReBBwe1xQEERBOYve15/PuGqOJbcQaxseac1NisBcjwDNBEy6jaVi7hdXKGnuI3mo2ly1R9hP0wHo0xBqYw6LlYIWuxPD5VnxH7s4Fw2eirklzPwj4uujy5o8shT3Qhbsez12VoCaYVNRvQBQSwMEFAAAAAgAAJifT/fkxAV3AAAAowAAABEAAAB3b3JkL2RvY3VtZW50LnhtbDWNXQ6DIAyAr2I8wGr2sAfiuMLOwIApiW1JYUFvb4nx5evf13ZuJrD/Y6Q67LhRMe09rrVmA1D8GtGVB+dIOvuxoKtaygKNJWRhH0tJtOAGz2l6AbpEo9WTXw5Hj7lDOqr9SEjk5BjufzP0dqcaSpWV16omt2ZPUEsBAhQAFAAAAAgAAJifT3QknFO7AAAAPgEAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECFAAUAAAACAAAmJ9PYXsvQ4gAAADyAAAACwAAAAAAAAAAAAAAAADsAAAAX3JlbHMvLnJlbHNQSwECFAAUAAAACAAAmJ9P9+TEBXcAAACjAAAAEQAAAAAAAAAAAAAAAACdAQAAd29yZC9kb2N1bWVudC54bWxQSwUGAAAAAAMAAwC5AAAAQwIAAAAA", + "base64", + ), ); - body.set(localHeader); - body.set(centralDirectory, localHeader.byteLength); - body.set(endOfCentralDirectory, localHeader.byteLength + centralDirectory.byteLength); - return body; } describe("createApiDocumentParser", () => { + it("includes the configured provider semantic revision in checkpoint identity", () => { + const input = { + body: ordinaryDocx(), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "a.docx", + mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + version: 1, + }; + const make = (revision: string) => + createApiDocumentParser({ + env: { + UNSTRUCTURED_API_URL: "https://parser.example.test", + UNSTRUCTURED_BACKEND_REVISION: revision, + }, + }); + expect(make("parser-policy-v1").policyFingerprint?.(input)).not.toBe( + make("parser-policy-v2").policyFingerprint?.(input), + ); + }); + it("keeps a structured upload above 10 MiB on the native parser within the 15 MiB admission limit", async () => { + const fetch = vi.fn(async () => new Response("[]")); + const parser = createApiDocumentParser({ + env: { UNSTRUCTURED_API_URL: "https://unstructured.example.test" }, + fetch, + }); + const artifact = await parser.parse({ + body: encoder.encode(`${" ".repeat(10 * 1024 * 1024)}{"id":9007199254740993}`), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "large.json", + mimeType: "application/json", + version: 1, + }); + expect(artifact.metadata.routedParser).toBe("native-structured"); + expect(artifact.metadata.parserExecution).toMatchObject({ isolation: "child-process" }); + expect(artifact.elements[0]?.text).toContain("9007199254740993"); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("applies a configured input limit to native structured parsing even without a remote endpoint", async () => { + const parser = createApiDocumentParser({ env: { UNSTRUCTURED_MAX_INPUT_BYTES: "8" } }); + await expect( + parser.parse({ + body: encoder.encode('{"value":123}'), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "data.json", + mimeType: "application/json", + version: 1, + }), + ).rejects.toThrow("maxInputBytes=8"); + }); + + it("always runs PDF safety inspection before the configured remote transport", async () => { + const check = vi.fn(async () => { + throw new ProviderInputError("PDF page 1 exceeds the raster pixel budget"); + }); + vi.mocked(createPdfParserPreflight).mockReturnValueOnce({ + check, + policyFingerprint: "pdf-preflight-test", + }); + const fetch = vi.fn(async () => new Response("[]")); + const parser = createApiDocumentParser({ + env: { UNSTRUCTURED_API_URL: "https://unstructured.example.test" }, + fetch, + }); + + await expect( + parser.parse({ + body: encoder.encode("%PDF-1.7"), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "80x180cm.pdf", + mimeType: "application/pdf", + version: 1, + }), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + expect(check).toHaveBeenCalledOnce(); + expect(fetch).not.toHaveBeenCalled(); + }); + it("resolves parser lane widths and preserves the legacy heavy alias", () => { expect(createApiUnstructuredConcurrencyOptions({})).toEqual({ heavyMaxConcurrency: 2, @@ -136,7 +207,7 @@ describe("createApiDocumentParser", () => { expect(fetchCalls).toBe(0); }); - it("routes an admitted 11 MiB structured upload to the remote parser", async () => { + it("preserves native CSV semantics for an admitted 11 MiB structured upload", async () => { let fetchCalls = 0; const parser = createApiDocumentParser({ env: { UNSTRUCTURED_API_URL: "https://unstructured.example.test" }, @@ -147,18 +218,19 @@ describe("createApiDocumentParser", () => { }); const artifact = await parser.parse({ - body: new Uint8Array(11 * 1024 * 1024), + body: encoder.encode(`${"\n".repeat(11 * 1024 * 1024)}name\nAda`), documentAssetId: "00000000-0000-4000-8000-000000000020", filename: "large.csv", mimeType: "text/csv", version: 1, }); - expect(fetchCalls).toBe(1); + expect(fetchCalls).toBe(0); expect(artifact).toMatchObject({ - metadata: { routeReason: "native-size-limit", routedParser: "unstructured" }, - parser: "unstructured", + metadata: { routeReason: "structured-file-type", routedParser: "native-structured" }, + parser: "native-structured", }); + expect(artifact.elements[0]?.text).toBe("name: Ada"); }); it("routes complex documents to the configured Unstructured API", async () => { diff --git a/knowledge-fs/apps/api/src/parser-options.ts b/knowledge-fs/apps/api/src/parser-options.ts index bbf04a4400d..6ec46629c29 100644 --- a/knowledge-fs/apps/api/src/parser-options.ts +++ b/knowledge-fs/apps/api/src/parser-options.ts @@ -8,6 +8,9 @@ import { } from "@knowledge/parsers"; import { Agent, type Dispatcher, fetch as undiciFetch } from "undici"; +import { createNativeParserIsolation } from "./native-parser-isolation"; +import { createPdfParserPreflight } from "./pdf-parser-preflight"; + const defaultUnstructuredRequestTimeoutMs = 600_000; const defaultUnstructuredMaxConcurrency = 2; const defaultUnstructuredMaxInputBytes = 15 * 1024 * 1024; @@ -34,6 +37,7 @@ export interface ApiParserEnv { readonly NODE_ENV?: string | undefined; readonly UNSTRUCTURED_API_KEY?: string | undefined; readonly UNSTRUCTURED_API_URL?: string | undefined; + readonly UNSTRUCTURED_BACKEND_REVISION?: string | undefined; readonly UNSTRUCTURED_DEFAULT_LANGUAGE?: string | undefined; readonly UNSTRUCTURED_HEAVY_MAX_CONCURRENCY?: string | undefined; readonly UNSTRUCTURED_HEAVY_REQUEST_TIMEOUT_MS?: string | undefined; @@ -100,11 +104,14 @@ export function createApiDocumentParser({ env, ...(fetchImpl ? { fetch: fetchImpl } : {}), }); + const nativeOptions = Object.freeze({ maxInputBytes: resolveParserMaxInputBytes(env) }); + const isolation = createNativeParserIsolation({ maxInputBytes: nativeOptions.maxInputBytes }); return createParserRouter({ - html: createNativeHtmlParser(), - markdown: createNativeMarkdownParser(), - structured: createNativeStructuredDataParser(), + html: isolation.wrap(createNativeHtmlParser(nativeOptions), nativeOptions), + markdown: isolation.wrap(createNativeMarkdownParser(nativeOptions), nativeOptions), + structured: isolation.wrap(createNativeStructuredDataParser(nativeOptions), nativeOptions), + maxNativeInputBytes: nativeOptions.maxInputBytes, unstructured, }); } @@ -157,6 +164,10 @@ function createApiUnstructuredParser({ const concurrency = createApiUnstructuredConcurrencyOptions(env); return createUnstructuredParserClient({ + ...(env.UNSTRUCTURED_BACKEND_REVISION?.trim() + ? { backendRevision: env.UNSTRUCTURED_BACKEND_REVISION.trim() } + : {}), + requestPreflight: createPdfParserPreflight(), ...(env.UNSTRUCTURED_DEFAULT_LANGUAGE?.trim() ? { defaultLanguage: env.UNSTRUCTURED_DEFAULT_LANGUAGE.trim() } : {}), @@ -179,14 +190,7 @@ function createApiUnstructuredParser({ ), } : {}), - maxInputBytes: - env.UNSTRUCTURED_MAX_INPUT_BYTES === undefined - ? defaultUnstructuredMaxInputBytes - : parseBoundedPositiveInteger( - env.UNSTRUCTURED_MAX_INPUT_BYTES, - "UNSTRUCTURED_MAX_INPUT_BYTES", - maxUnstructuredInputBytes, - ), + maxInputBytes: resolveParserMaxInputBytes(env), ...(env.UNSTRUCTURED_MAX_RETRIES !== undefined ? { maxRetries: parseNonNegativeInteger( @@ -207,6 +211,16 @@ function createApiUnstructuredParser({ }); } +function resolveParserMaxInputBytes(env: ApiParserEnv): number { + return env.UNSTRUCTURED_MAX_INPUT_BYTES === undefined + ? defaultUnstructuredMaxInputBytes + : parseBoundedPositiveInteger( + env.UNSTRUCTURED_MAX_INPUT_BYTES, + "UNSTRUCTURED_MAX_INPUT_BYTES", + maxUnstructuredInputBytes, + ); +} + export function createNodeUnstructuredFetch({ createDispatcher = (options) => new Agent(options), fetch: fetchImpl = undiciFetch as unknown as typeof fetch, diff --git a/knowledge-fs/apps/api/src/pdf-parser-preflight.test.ts b/knowledge-fs/apps/api/src/pdf-parser-preflight.test.ts new file mode 100644 index 00000000000..20aa4d8f3bc --- /dev/null +++ b/knowledge-fs/apps/api/src/pdf-parser-preflight.test.ts @@ -0,0 +1,344 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { type ParseDocumentInput, ProviderInputError } from "@knowledge/parsers"; +import { afterEach, describe, expect, it } from "vitest"; + +import { createPdfParserPreflight } from "./pdf-parser-preflight"; + +const directories: string[] = []; +const pdfInput = (overrides: Partial = {}): ParseDocumentInput => ({ + body: new TextEncoder().encode("%PDF-1.7\n"), + documentAssetId: "asset-pdf", + filename: "document.pdf", + mimeType: "application/pdf", + version: 1, + ...overrides, +}); + +function pageInfo(width: number, height: number, page = 1): string { + return `Page ${page} size: ${width} x ${height} pts\nPage ${page} MediaBox: 0 0 ${width} ${height}\nPage ${page} CropBox: 0 0 ${width} ${height}\n`; +} + +async function temporaryDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pdf-preflight-test-")); + directories.push(directory); + return directory; +} + +async function executableFixture( + source: string, +): Promise<{ directory: string; executable: string }> { + const directory = await temporaryDirectory(); + const executable = join(directory, "pdfinfo-fixture"); + await writeFile(executable, `#!${process.execPath}\n${source}`, { mode: 0o700 }); + return { directory, executable }; +} + +async function waitForFile(path: string): Promise { + for (let attempt = 0; attempt < 200; attempt += 1) { + try { + return await readFile(path, "utf8"); + } catch { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + } + throw new Error("Fixture child did not start"); +} + +function syntheticPdf({ + pages, + inherited = false, +}: { pages: readonly [number, number][]; inherited?: boolean }): Uint8Array { + const firstPage = pages[0] ?? [595, 842]; + const box = ([width, height]: readonly [number, number]) => + `/MediaBox [0 0 ${width} ${height}] /CropBox [0 0 ${width} ${height}]`; + const objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + `<< /Type /Pages /Kids [${pages.map((_, index) => `${index + 3} 0 R`).join(" ")}] /Count ${pages.length} ${inherited ? box(firstPage) : ""} >>`, + ...pages.map((page) => `<< /Type /Page /Parent 2 0 R ${inherited ? "" : box(page)} >>`), + ]; + let body = "%PDF-1.7\n"; + const offsets = [0]; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.byteLength(body)); + body += `${index + 1} 0 obj\n${object}\nendobj\n`; + } + const xref = Buffer.byteLength(body); + body += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + body += offsets + .slice(1) + .map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`) + .join(""); + body += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(body); +} + +afterEach(async () => { + await Promise.all( + directories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })), + ); +}); + +describe("PDF raster preflight", () => { + it.each([ + [595.28, 841.89], + [841.89, 1190.55], + ])("admits ordinary %s by %s point pages", async (width, height) => { + const preflight = createPdfParserPreflight({ + executePdfinfo: async () => `Pages: 1\n${pageInfo(width, height)}`, + }); + await expect(preflight.check(pdfInput())).resolves.toBeUndefined(); + }); + + it("rejects an 80 by 180 cm banner despite its tiny file body", async () => { + const preflight = createPdfParserPreflight({ + executePdfinfo: async () => `Pages: 1\n${pageInfo(2267.72, 5102.36)}`, + }); + await expect(preflight.check(pdfInput())).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + message: expect.stringContaining("page 1"), + }); + }); + + it("checks every page, including an oversized page after a normal first page", async () => { + const preflight = createPdfParserPreflight({ + executePdfinfo: async () => `Pages: 2\n${pageInfo(595, 842)}${pageInfo(2267, 5102, 2)}`, + }); + await expect(preflight.check(pdfInput())).rejects.toThrow("page 2"); + }); + + it("uses the conservative MediaBox even when page size and CropBox are small", async () => { + const output = `Pages: 1\n${pageInfo(595, 842).replace("MediaBox: 0 0 595 842", "MediaBox: -1000 0 2267 5102")}`; + await expect( + createPdfParserPreflight({ executePdfinfo: async () => output }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + }); + + it("also bounds single-edge length when the total pixel area is small", async () => { + await expect( + createPdfParserPreflight({ + executePdfinfo: async () => `Pages: 1\n${pageInfo(3000, 1)}`, + }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + }); + + it("bounds total pixel area even when both individual edges fit", async () => { + await expect( + createPdfParserPreflight({ + executePdfinfo: async () => `Pages: 1\n${pageInfo(1190.55, 1683.78)}`, + }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + }); + + it("checks the CropBox independently of the page size and MediaBox", async () => { + const output = `Pages: 1\n${pageInfo(595, 842).replace("CropBox: 0 0 595 842", "CropBox: 0 0 2267 5102")}`; + await expect( + createPdfParserPreflight({ executePdfinfo: async () => output }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + }); + + it.each([ + "Pages: 0\n", + "Pages: 10001\n", + `Pages: 1\nPages: 1\n${pageInfo(595, 842)}`, + `Pages: 2\n${pageInfo(595, 842)}`, + `Pages: 1\n${pageInfo(595, 842)}${pageInfo(595, 842)}`, + `Pages: 1\n${pageInfo(595, 842, 2)}`, + `Pages: 1\n${pageInfo(595, 842).replace(/Page 1 CropBox:.*\n/, "")}`, + `Pages: 1\n${pageInfo(Number.NaN, 842)}`, + `Pages: 1\n${pageInfo(Number.POSITIVE_INFINITY, 842)}`, + `Pages: 1\n${pageInfo(-595, 842)}`, + `Pages: 1\n${pageInfo(0, 842)}`, + `Pages: 1\n${pageInfo(595, 842).replace("size: 595 x 842 pts", "size: invalid")}`, + `Pages: 1\n${pageInfo(595, 842).replace("CropBox: 0 0 595 842", "CropBox: 0 0 595")}`, + `Pages: 1\n${pageInfo(595, 842).replace("MediaBox: 0 0 595 842", "MediaBox: 10 0 0 842")}`, + "Encrypted: yes\n", + ])("fails closed for incomplete, duplicate or invalid geometry: %s", async (output) => { + await expect( + createPdfParserPreflight({ executePdfinfo: async () => output }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + }); + + it.each([ + { filename: "document.bin", mimeType: "application/pdf", body: new Uint8Array() }, + { filename: "DOCUMENT.PDF", mimeType: "application/octet-stream", body: new Uint8Array() }, + { + filename: "document.docx", + mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + body: new TextEncoder().encode("garbage\n%PDF-1.7\n"), + }, + ])("applies the PDF guard if any identity signal identifies a PDF", async (identity) => { + await expect( + createPdfParserPreflight({ + executePdfinfo: async () => `Pages: 1\n${pageInfo(2267, 5102)}`, + }).check(pdfInput(identity)), + ).rejects.toBeInstanceOf(ProviderInputError); + }); + + it("performs no filesystem or subprocess I/O for non-PDF documents", async () => { + const preflight = createPdfParserPreflight({ + temporaryDirectory: "/does-not-exist/pdf-preflight", + executePdfinfo: async () => { + throw new Error("must not execute"); + }, + }); + await expect( + preflight.check( + pdfInput({ + filename: "document.docx", + mimeType: "application/octet-stream", + body: new TextEncoder().encode("PK"), + }), + ), + ).resolves.toBeUndefined(); + }); + + it.each([{ timeoutMs: 0 }, { timeoutMs: Number.NaN }, { maxOutputBytes: -1 }])( + "refuses invalid inspection limits: %s", + (options) => { + expect(() => createPdfParserPreflight(options)).toThrow("must be a positive safe integer"); + }, + ); + + it("writes the input privately and cleans it up after successful checking", async () => { + const directory = await temporaryDirectory(); + const input = pdfInput(); + const preflight = createPdfParserPreflight({ + temporaryDirectory: directory, + executePdfinfo: async ({ path }) => { + expect((await stat(path)).mode & 0o777).toBe(0o600); + expect(await readFile(path)).toEqual(Buffer.from(input.body)); + return `Pages: 1\n${pageInfo(595, 842)}`; + }, + }); + await preflight.check(input); + expect(await readdir(directory)).toEqual([]); + }); + + it("cleans up its private temporary input when geometry is rejected", async () => { + const directory = await temporaryDirectory(); + await expect( + createPdfParserPreflight({ + temporaryDirectory: directory, + executePdfinfo: async () => "invalid", + }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + expect(await readdir(directory)).toEqual([]); + }); + + it("fails clearly when pdfinfo is missing, rather than silently bypassing the guard", async () => { + const directory = await temporaryDirectory(); + await expect( + createPdfParserPreflight({ + temporaryDirectory: directory, + pdfinfoExecutable: join(directory, "missing"), + }).check(pdfInput()), + ).rejects.toThrow("requires the pdfinfo executable"); + expect(await readdir(directory)).toEqual([]); + }); + + it("preserves an already-aborted parent reason without doing I/O", async () => { + const controller = new AbortController(); + const reason = new Error("lease expired"); + controller.abort(reason); + await expect( + createPdfParserPreflight({ temporaryDirectory: "/does-not-exist/pdf-preflight" }).check( + pdfInput({ signal: controller.signal }), + ), + ).rejects.toBe(reason); + }); + + it("kills the actual subprocess and cleans up when the parent is cancelled", async () => { + const { directory, executable } = await executableFixture( + `const fs = require('node:fs');\nfs.writeFileSync(__dirname + '/child.pid', String(process.pid));\nsetInterval(() => {}, 1000);`, + ); + const controller = new AbortController(); + const reason = new Error("lease lost"); + const result = createPdfParserPreflight({ + temporaryDirectory: directory, + pdfinfoExecutable: executable, + }).check(pdfInput({ signal: controller.signal })); + const rejected = expect(result).rejects.toBe(reason); + const pid = Number(await waitForFile(join(directory, "child.pid"))); + controller.abort(reason); + await rejected; + expect(() => process.kill(pid, 0)).toThrow(); + expect((await readdir(directory)).sort()).toEqual(["child.pid", "pdfinfo-fixture"]); + }); + + it("kills timed-out subprocesses even when they ignore SIGTERM, then cleans up", async () => { + const { directory, executable } = await executableFixture( + "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);", + ); + await expect( + createPdfParserPreflight({ + temporaryDirectory: directory, + pdfinfoExecutable: executable, + timeoutMs: 100, + }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + expect(await readdir(directory)).toEqual(["pdfinfo-fixture"]); + }); + + it("bounds subprocess stdout and cleans up after output overflow", async () => { + const { directory, executable } = await executableFixture( + "process.stdout.write('a'.repeat(4096)); setInterval(() => {}, 1000);", + ); + await expect( + createPdfParserPreflight({ + temporaryDirectory: directory, + pdfinfoExecutable: executable, + maxOutputBytes: 64, + }).check(pdfInput()), + ).rejects.toBeInstanceOf(ProviderInputError); + expect(await readdir(directory)).toEqual(["pdfinfo-fixture"]); + }); + + it("rejects a real subprocess parse failure without exposing its stderr", async () => { + const { directory, executable } = await executableFixture( + "process.stderr.write('private-document-text'); process.exit(1);", + ); + const result = createPdfParserPreflight({ + temporaryDirectory: directory, + pdfinfoExecutable: executable, + }).check(pdfInput()); + await expect(result).rejects.toThrow("Unable to safely inspect PDF"); + await expect(result).rejects.not.toThrow("private-document-text"); + expect(await readdir(directory)).toEqual(["pdfinfo-fixture"]); + }); + + it("uses real Poppler to resolve inherited boxes and detect later oversized pages", async () => { + await promisify(execFile)("pdfinfo", ["-v"]); + const preflight = createPdfParserPreflight(); + await expect( + preflight.check( + pdfInput({ + body: syntheticPdf({ + pages: [ + [595, 842], + [595, 842], + ], + inherited: true, + }), + }), + ), + ).resolves.toBeUndefined(); + await expect( + preflight.check( + pdfInput({ + body: syntheticPdf({ + pages: [ + [595, 842], + [2267, 5102], + ], + }), + }), + ), + ).rejects.toThrow("page 2"); + }); +}); diff --git a/knowledge-fs/apps/api/src/pdf-parser-preflight.ts b/knowledge-fs/apps/api/src/pdf-parser-preflight.ts new file mode 100644 index 00000000000..92256d034fd --- /dev/null +++ b/knowledge-fs/apps/api/src/pdf-parser-preflight.ts @@ -0,0 +1,251 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { type ParseDocumentInput, ProviderInputError } from "@knowledge/parsers"; + +const RASTER_DPI = 350; +const MAX_PAGE_PIXELS = 25_000_000; +const MAX_PAGE_EDGE_PIXELS = 10_000; +const MAX_PAGES = 10_000; +const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024; + +interface PdfinfoExecutionInput { + readonly path: string; + readonly signal?: AbortSignal; +} + +interface PdfParserPreflightOptions { + /** Substitute only the bounded subprocess boundary; useful for deterministic geometry tests. */ + readonly executePdfinfo?: (input: PdfinfoExecutionInput) => Promise; + readonly maxOutputBytes?: number; + readonly pdfinfoExecutable?: string; + readonly temporaryDirectory?: string; + readonly timeoutMs?: number; +} + +export interface PdfParserPreflight { + readonly policyFingerprint: string; + check(input: ParseDocumentInput): Promise; +} + +/** + * Metadata-only admission prevents huge pages from reaching rasterization. Poppler resolves + * inherited boxes without decoding page images. The provider's own pre-render pixel guard is + * still required: PDF engines can disagree on malformed objects and features such as UserUnit. + */ +export function createPdfParserPreflight( + options: PdfParserPreflightOptions = {}, +): PdfParserPreflight { + const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs"); + const maxOutputBytes = positiveInteger( + options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES, + "maxOutputBytes", + ); + const executePdfinfo = + options.executePdfinfo ?? + ((input: PdfinfoExecutionInput) => + runPdfinfo(input, { + executable: options.pdfinfoExecutable ?? "pdfinfo", + maxOutputBytes, + timeoutMs, + })); + + return { + policyFingerprint: `pdf-raster-preflight-v1:dpi=${RASTER_DPI}:pixels=${MAX_PAGE_PIXELS}:edge=${MAX_PAGE_EDGE_PIXELS}:pages=${MAX_PAGES}`, + async check(input) { + if (!isPdf(input)) { + return; + } + input.signal?.throwIfAborted(); + const directory = await mkdtemp( + join(options.temporaryDirectory ?? tmpdir(), "knowledge-fs-pdf-"), + ); + try { + input.signal?.throwIfAborted(); + const path = join(directory, "input.pdf"); + await writeFile(path, input.body, { + flag: "wx", + mode: 0o600, + ...(input.signal ? { signal: input.signal } : {}), + }); + input.signal?.throwIfAborted(); + const output = await executePdfinfo({ + path, + ...(input.signal ? { signal: input.signal } : {}), + }); + input.signal?.throwIfAborted(); + assertSafeGeometry(output); + } catch (error) { + if (input.signal?.aborted) { + throw input.signal.reason; + } + throw error; + } finally { + await rm(directory, { force: true, recursive: true }); + } + }, + }; +} + +function isPdf(input: ParseDocumentInput): boolean { + return ( + input.mimeType.split(";", 1)[0]?.trim().toLowerCase() === "application/pdf" || + input.filename.toLowerCase().endsWith(".pdf") || + Buffer.from( + input.body.buffer, + input.body.byteOffset, + Math.min(input.body.byteLength, 1024), + ).indexOf("%PDF-") !== -1 + ); +} + +function runPdfinfo( + input: PdfinfoExecutionInput, + options: { + readonly executable: string; + readonly maxOutputBytes: number; + readonly timeoutMs: number; + }, +): Promise { + input.signal?.throwIfAborted(); + return new Promise((resolve, reject) => { + const child = execFile( + options.executable, + ["-f", "1", "-l", String(MAX_PAGES + 1), "-box", input.path], + { + encoding: "utf8", + env: { ...process.env, LANG: "C", LC_ALL: "C" }, + killSignal: "SIGKILL", + maxBuffer: options.maxOutputBytes, + timeout: options.timeoutMs, + }, + (error, stdout) => { + input.signal?.removeEventListener("abort", cancel); + if (input.signal?.aborted) { + reject(input.signal.reason); + } else if (error?.code === "ENOENT") { + reject( + new Error("PDF safety inspection requires the pdfinfo executable (poppler-utils)."), + ); + } else if (error) { + // stderr can contain document content; do not include it in errors or their causes. + reject( + new ProviderInputError( + "Unable to safely inspect PDF page geometry within the inspection limits.", + ), + ); + } else { + resolve(stdout); + } + }, + ); + // Wait for the execFile callback after termination so temporary input is not removed while + // a child still uses it, and callers never continue while abandoned work is still running. + const cancel = () => child.kill("SIGKILL"); + input.signal?.addEventListener("abort", cancel, { once: true }); + if (input.signal?.aborted) { + cancel(); + } + }); +} + +type PageGeometry = Map<"size" | "MediaBox" | "CropBox", readonly [number, number]>; + +function assertSafeGeometry(output: string): void { + const pageCounts = [...output.matchAll(/^Pages:\s*(.*?)\s*$/gm)]; + const rawPageCount = pageCounts[0]?.[1] ?? ""; + if (pageCounts.length !== 1 || !/^\d+$/.test(rawPageCount)) { + throw invalidGeometry(); + } + const pageCount = Number(rawPageCount); + if (!Number.isSafeInteger(pageCount) || pageCount < 1 || pageCount > MAX_PAGES) { + throw new ProviderInputError( + `PDF safety inspection supports between 1 and ${MAX_PAGES} pages.`, + ); + } + const pages = new Map(); + for (const match of output.matchAll(/^Page\s+(\d+)\s+(size|MediaBox|CropBox):\s*(.*?)\s*$/gm)) { + const pageNumber = Number(match[1]); + const kind = match[2] as "size" | "MediaBox" | "CropBox"; + const value = match[3] ?? ""; + if (!Number.isSafeInteger(pageNumber) || pageNumber < 1 || pageNumber > pageCount) { + throw invalidGeometry(); + } + const page = pages.get(pageNumber) ?? new Map(); + if (page.has(kind)) { + throw invalidGeometry(); + } + const dimensions = kind === "size" ? parsePageSize(value) : parseBox(value); + page.set(kind, dimensions); + pages.set(pageNumber, page); + } + if (pages.size !== pageCount) { + throw invalidGeometry(); + } + for (const [pageNumber, geometry] of pages) { + if (geometry.size !== 3) { + throw invalidGeometry(); + } + for (const [width, height] of geometry.values()) { + const rasterWidth = Math.ceil((width / 72) * RASTER_DPI); + const rasterHeight = Math.ceil((height / 72) * RASTER_DPI); + if ( + rasterWidth > MAX_PAGE_EDGE_PIXELS || + rasterHeight > MAX_PAGE_EDGE_PIXELS || + rasterWidth * rasterHeight > MAX_PAGE_PIXELS + ) { + throw new ProviderInputError( + `PDF page ${pageNumber} exceeds the safe rasterization budget (${MAX_PAGE_PIXELS} pixels per page, ${MAX_PAGE_EDGE_PIXELS} pixels per edge at ${RASTER_DPI} DPI). Resize the page before importing.`, + ); + } + } + } +} + +function parsePageSize(value: string): readonly [number, number] { + const match = /^(\S+)\s+x\s+(\S+)\s+pts(?:\s+\([^\r\n]*\))?$/.exec(value); + if (!match) { + throw invalidGeometry(); + } + return positiveDimensions(finiteNumber(match[1] ?? ""), finiteNumber(match[2] ?? "")); +} + +function parseBox(value: string): readonly [number, number] { + const coordinates = value.trim().split(/\s+/); + if (coordinates.length !== 4) { + throw invalidGeometry(); + } + const [left = 0, bottom = 0, right = 0, top = 0] = coordinates.map(finiteNumber); + return positiveDimensions(right - left, top - bottom); +} + +function finiteNumber(value: string): number { + const number = Number(value); + if (!/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/.test(value) || !Number.isFinite(number)) { + throw invalidGeometry(); + } + return number; +} + +function positiveDimensions(width: number, height: number): readonly [number, number] { + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + throw invalidGeometry(); + } + return [width, height]; +} + +function invalidGeometry(): ProviderInputError { + return new ProviderInputError( + "Unable to safely inspect PDF: page geometry is incomplete or invalid.", + ); +} + +function positiveInteger(value: number, name: string): number { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`PDF preflight ${name} must be a positive safe integer.`); + } + return value; +} diff --git a/knowledge-fs/apps/api/src/profile-visual-embedding-options.ts b/knowledge-fs/apps/api/src/profile-visual-embedding-options.ts index 8faa6f0c617..84ae18c11c8 100644 --- a/knowledge-fs/apps/api/src/profile-visual-embedding-options.ts +++ b/knowledge-fs/apps/api/src/profile-visual-embedding-options.ts @@ -85,7 +85,7 @@ export function createApiProfileVisualEmbeddingOptions({ "KNOWLEDGE_VISUAL_EMBEDDING_MAX_BATCH_BYTES", ), objectStorage, - preferredVariant: env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT?.trim() || "thumbnail", + preferredVariant: env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT?.trim() || "analysis", provider: imageEmbeddingProviderFactory(profile), }); return lifecycleGate.run(() => objectProvider.embedAssets(input), { signal: input.signal }); diff --git a/knowledge-fs/apps/api/src/visual-embedding-options.test.ts b/knowledge-fs/apps/api/src/visual-embedding-options.test.ts index 5672004d9ec..a4185a6199b 100644 --- a/knowledge-fs/apps/api/src/visual-embedding-options.test.ts +++ b/knowledge-fs/apps/api/src/visual-embedding-options.test.ts @@ -46,7 +46,7 @@ describe("createApiVisualEmbeddingOptions", () => { await adapter.objectStorage.putObject({ body: new Uint8Array([1, 2, 3]), contentType: "image/png", - key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png", + key: "tenant/spaces/space/documents/doc/assets/chart.png", }); globalThis.fetch = (async (input, init) => { const request = new Request(input, init); @@ -123,7 +123,7 @@ describe("createApiVisualEmbeddingOptions", () => { { content: "AQID", content_type: "image", - file_id: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png", + file_id: "tenant/spaces/space/documents/doc/assets/chart.png", }, ], input_type: "document", diff --git a/knowledge-fs/apps/api/src/visual-embedding-options.ts b/knowledge-fs/apps/api/src/visual-embedding-options.ts index ad005260ba4..4c25e0dc8aa 100644 --- a/knowledge-fs/apps/api/src/visual-embedding-options.ts +++ b/knowledge-fs/apps/api/src/visual-embedding-options.ts @@ -130,7 +130,7 @@ export function createApiVisualEmbeddingOptions({ objectStorage, ...(trimmed(env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT) ? { preferredVariant: trimmed(env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT) } - : { preferredVariant: "thumbnail" }), + : { preferredVariant: "analysis" }), provider: imageBytesProvider, }); diff --git a/knowledge-fs/docs/production-deployment.md b/knowledge-fs/docs/production-deployment.md index 8c8c0cd504e..8dd87ea84ba 100644 --- a/knowledge-fs/docs/production-deployment.md +++ b/knowledge-fs/docs/production-deployment.md @@ -92,6 +92,7 @@ the service: | `KNOWLEDGE_BUFFERED_DOCUMENT_UPLOAD_IDLE_TIMEOUT_MS` | Maximum continuous idle interval while reading a direct multipart or small-file fallback body. Defaults to `30000`; expiry cancels the reader, returns 408, and releases admission. | | `KNOWLEDGE_BUFFERED_DOCUMENT_UPLOAD_TOTAL_TIMEOUT_MS` | Generous total body-read deadline for a direct multipart or small-file fallback request. Defaults to `600000`, must be at least the idle timeout, and returns 408 on expiry. | | `UNSTRUCTURED_API_URL` | Parser endpoint for complex formats. | +| `UNSTRUCTURED_BACKEND_REVISION` | Non-secret identity of the actual parser image, models and output policy. Bump it on semantic backend changes to invalidate raw parse checkpoint identity. An unset revision is explicitly `external-unversioned`, not proof of reproducibility. Admission-only limits need no revision bump. | | `UNSTRUCTURED_API_KEY` | Optional parser authentication. | | `UNSTRUCTURED_MAX_CONCURRENCY` | Process-wide limit shared by every remote parser request; defaults to `2`. | | `UNSTRUCTURED_HEAVY_MAX_CONCURRENCY` | Nested limit for every PDF and structurally/byte-heavy Office, email, EPUB, ODT, or RTF request. The bundled parser profile uses `1`; it must not exceed `UNSTRUCTURED_MAX_CONCURRENCY`. The materialization pre-admission lane follows this value but is capped at `KNOWLEDGE_DOCUMENT_MATERIALIZATION_MAX_CONCURRENCY - 1` (minimum `1`) to preserve ordinary-document progress. `UNSTRUCTURED_PDF_MAX_CONCURRENCY` remains a lower-precedence compatibility alias. | @@ -118,7 +119,8 @@ including response consumption. It defaults to `60000`; transport failures and The optional `knowledge-fs-unstructured` Compose profile starts an isolated parser service with six pages per child request, three child workers, and zero child retries. It does not modify Dify's existing `unstructured` service or legacy ETL traffic. The tracked -`knowledge-fs-unstructured-service.defaults` file contains only service-side page-parallel values; +`knowledge-fs-unstructured-service.defaults` file contains service-side page-parallel and PDF +raster-safety values; operator-owned `knowledge-fs-unstructured.env` is loaded afterwards and can override it. The required file deliberately uses a non-`.env` suffix so it remains tracked in clean checkouts. The `.env.example` files remain copy-only templates and are never loaded as runtime configuration. @@ -127,6 +129,30 @@ resource benchmarks; review the same contracts before intentionally changing tha requests are at or below the split size and therefore partition locally instead of recursively spawning more requests. +PDF import also checks page geometry before sending the file to Unstructured. Compressed file +size and page count alone cannot bound raster memory: an 80 × 180 cm single-page PDF needs about +273 million pixels at the pinned parser's 350 DPI even if the file is only 248 KB. The client +checks every page with bounded Poppler metadata inspection, estimates its raster size at 350 DPI, +and rejects pages above 25 million pixels or 10,000 pixels on either side. It also rejects files +whose geometry cannot be safely inspected. Oversized documents must be exported at smaller page +dimensions or tiled into smaller pages before import; splitting only between existing giant pages +does not reduce the per-page risk. The original file, parsing strategy, text, and image extraction +remain unchanged for admitted documents. The client does not silently switch to text-only parsing. + +The isolated service explicitly keeps `PDF_RENDER_DPI=350` (the pinned image's existing default) +and sets `PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000`. The latter is a second guard in the pinned +`unstructured-inference` renderer: it rejects an oversized page before allocating its bitmap. +That renderer is shared by `hi_res` layout inference, PDF OCR (including `auto` fallback), and +image-block extraction. Its PDF rendering uses PDFium; Poppler in this upstream path is used for +metadata inspection. These process settings are not HTTP request parameters: the pinned API does +not expose `pdf_image_dpi`. Do not raise the service DPI above 350, increase the pixel ceiling, or +disable the guard (`0`) without changing and validating the client policy. Operator env overrides +still take precedence, so existing deployments must remove conflicting overrides and recreate +the parser service as well as deploy the new KnowledgeFS image. The Kubernetes baseline does not +deploy Unstructured; configure the same settings on its external parser service. Geometry bounds +address oversized page rasters, not every possible embedded-image decompression or PDF complexity +attack; the service memory limit and workload admission remain necessary. + The KnowledgeFS client keeps a process-wide limit of `2` and adds a heavy-workload nested limit of `1`. Every PDF remains heavy because compressed PDF object streams make a bounded page-count scan unreliable. ZIP-backed remote formats become heavy when their admitted body exceeds 8 MiB or a @@ -146,6 +172,28 @@ own child thread pool. The Kubernetes baseline does not own an Unstructured depl generic client limits. Operators with a different resource envelope must benchmark representative narrative, table, and scanned pages before changing either concurrency limit. +### Document structure admission + +Before sending ZIP-backed Office, ODT or EPUB documents to the service, KnowledgeFS also streams +their actual expanded entries within the same admission slot. This checks XML structure and +spreadsheet cell spans, not just compressed upload size. Default limits include 512 MiB total +expansion, 64 MiB XML (16 MiB per part), XML depth 128, 256 worksheets, and 250,000 dense cells per +worksheet / 500,000 per workbook. Sparse distant cells and repeated worksheet references count +toward the dense-cell cost; harmless whole-column formatting does not. Unsafe paths, damaged +archives/XML, or missing/external worksheet relationships fail before a provider request. Standard +EPUB 2 XHTML declarations and optional image-relationship fallbacks remain supported. + +Native structured parsing shares the admitted upload-byte limit (15 MiB by default), so an upload +above 10 MiB no longer silently switches JSON/CSV to a different parser. Decoded structure, +document-wide table expansion, and projected output have independent finite budgets. Exceeding a +budget is an explicit non-retryable error, not truncated content. Split unusually complex inputs +before retrying; these are resource limits, not the file formats' theoretical maximum sizes. + +These checks do not replace process isolation or memory limits. Legacy DOC/PPT/XLS conversion, +nested mail attachments, native image decoding, and synchronous in-process parsing still require +further isolation work. Existing indexes are unchanged until re-indexing; new parses use the +updated parser policy identity. No new database migration or model setting is required. + ## PDF image rasterization The production API image installs Poppler and verifies `pdftoppm` during the image build. Its image diff --git a/knowledge-fs/infra/local/.env.example b/knowledge-fs/infra/local/.env.example index f5826040507..2bcdf264599 100644 --- a/knowledge-fs/infra/local/.env.example +++ b/knowledge-fs/infra/local/.env.example @@ -14,6 +14,8 @@ UNSTRUCTURED_PORT=8000 # Host-run API processes use the published localhost port. Compose must use the separate internal # service URL below because 127.0.0.1 inside the API container refers to that container itself. UNSTRUCTURED_API_URL=http://127.0.0.1:8000 +# Non-secret identity for the actual parser image/model/output policy; bump on semantic upgrades. +UNSTRUCTURED_BACKEND_REVISION= UNSTRUCTURED_CONTAINER_API_URL=http://unstructured:8000 UNSTRUCTURED_API_KEY= UNSTRUCTURED_PARALLEL_MODE_ENABLED=true diff --git a/knowledge-fs/infra/local/compose.unstructured-sandbox.yaml b/knowledge-fs/infra/local/compose.unstructured-sandbox.yaml new file mode 100644 index 00000000000..5d154582025 --- /dev/null +++ b/knowledge-fs/infra/local/compose.unstructured-sandbox.yaml @@ -0,0 +1,21 @@ +# Explicit local opt-in; retains the same unstructured hostname and API URL. +services: + unstructured: + build: + context: ../../services/unstructured-sandbox + dockerfile: Dockerfile + image: knowledge-fs-unstructured-sandbox:local + init: true + read_only: true + pids_limit: 192 + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + tmpfs: + - /tmp:rw,nosuid,nodev,size=1073741824,mode=1777 + environment: + UNSTRUCTURED_PARALLEL_MODE_ENABLED: "false" + deploy: + resources: + limits: + pids: 192 diff --git a/knowledge-fs/infra/local/compose.yaml b/knowledge-fs/infra/local/compose.yaml index b1c64c26953..5d82781618e 100644 --- a/knowledge-fs/infra/local/compose.yaml +++ b/knowledge-fs/infra/local/compose.yaml @@ -21,6 +21,9 @@ services: unstructured: image: downloads.unstructured.io/unstructured-io/unstructured-api@sha256:0df934a22e4e893cf15e7aeaf35c463ecc75937758a83099aefdc13041619a1d environment: + # Keep geometry admission aligned with the KnowledgeFS client and reject before allocation. + PDF_RENDER_DPI: "350" + PDF_RENDER_MAX_PIXELS_PER_PAGE: "25000000" UNSTRUCTURED_PARALLEL_MODE_ENABLED: ${UNSTRUCTURED_PARALLEL_MODE_ENABLED:-true} UNSTRUCTURED_PARALLEL_MODE_URL: ${UNSTRUCTURED_PARALLEL_MODE_URL:-http://127.0.0.1:8000/general/v0/general} UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE: ${UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE:-6} @@ -81,6 +84,7 @@ services: KNOWLEDGE_DEV_SUBJECT_ID: ${KNOWLEDGE_DEV_SUBJECT_ID:-dev-user} KNOWLEDGE_DEV_TENANT_ID: ${KNOWLEDGE_DEV_TENANT_ID:-tenant-dev} UNSTRUCTURED_API_URL: ${UNSTRUCTURED_CONTAINER_API_URL:-http://unstructured:8000} + UNSTRUCTURED_BACKEND_REVISION: ${UNSTRUCTURED_BACKEND_REVISION:-} UNSTRUCTURED_MAX_CONCURRENCY: ${UNSTRUCTURED_MAX_CONCURRENCY:-2} UNSTRUCTURED_HEAVY_MAX_CONCURRENCY: ${UNSTRUCTURED_HEAVY_MAX_CONCURRENCY:-1} UNSTRUCTURED_MAX_INPUT_BYTES: ${UNSTRUCTURED_MAX_INPUT_BYTES:-15728640} diff --git a/knowledge-fs/package.json b/knowledge-fs/package.json index d14ce2931cd..bc30c38da60 100644 --- a/knowledge-fs/package.json +++ b/knowledge-fs/package.json @@ -13,7 +13,7 @@ "benchmark:ingestion": "node --import tsx scripts/benchmark-ingestion-model-optimizations.mjs", "build": "turbo run build", "capability:export": "node --import tsx scripts/export-capability-v2-operations.mjs", - "check": "pnpm typecheck && pnpm test && pnpm openapi:export:test && pnpm test:coverage:ci && pnpm eval:regression && pnpm eval:phase4 && pnpm swagger:test && pnpm db:migrations:check && pnpm p9:bundle:test && pnpm ci:workflow:test && pnpm compose:middleware:config && pnpm compose:config && pnpm dify:compose:config && pnpm compose:middleware:test && pnpm compose:apps:test && pnpm docker:context:test && pnpm local:happy-path:test && pnpm semantic:rollout:test && pnpm docker:api:bundle-smoke:test && pnpm docker:admin:http-smoke:test && pnpm docker:apps:smoke:test", + "check": "pnpm typecheck && pnpm test && pnpm openapi:export:test && pnpm test:coverage:ci && pnpm eval:regression && pnpm eval:phase4 && pnpm swagger:test && pnpm db:migrations:check && pnpm p9:bundle:test && pnpm ci:workflow:test && pnpm compose:middleware:config && pnpm compose:config && pnpm dify:compose:config && pnpm compose:middleware:test && pnpm compose:apps:test && pnpm docker:context:test && pnpm local:happy-path:test && pnpm semantic:rollout:test && pnpm docker:api:bundle-smoke:test && pnpm docker:admin:http-smoke:test && pnpm docker:apps:smoke:test && pnpm parser:regression:test", "ci:workflow:test": "node --test scripts/github-actions-workflow.test.mjs scripts/secret-scan.test.mjs", "compose:config": "docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml config", "compose:apps:test": "node --test scripts/compose-apps.test.mjs", @@ -61,7 +61,10 @@ "test": "turbo run test", "test:coverage": "turbo run test:coverage", "test:coverage:ci": "turbo run test:coverage --filter=!@knowledge/api", - "typecheck": "turbo run typecheck" + "typecheck": "turbo run typecheck", + "parser:regression:test": "node --test scripts/parser-benchmark.test.mjs scripts/pdf-thumbnail-benchmark.test.mjs", + "benchmark:parsers": "node --import tsx scripts/parser-benchmark.mjs", + "benchmark:pdf-thumbnail": "node --import tsx scripts/pdf-thumbnail-benchmark.mjs" }, "devDependencies": { "@biomejs/biome": "^1.9.4", diff --git a/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts b/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts index e66edc41b60..7d6714ed5db 100644 --- a/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts +++ b/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts @@ -26,6 +26,134 @@ const firstArtifactId = "30000000-0000-4000-8000-000000000001"; const retryArtifactId = "30000000-0000-4000-8000-000000000002"; describe("compileDocumentArtifact canonical artifact", () => { + it("uses provider fallback when synchronous external PDF images cannot be resolved", async () => { + const hints: unknown[] = []; + const deps = pipelineDeps(); + const artifact = await compileDocumentArtifact( + { + asset: { + ...documentAsset(), + mimeType: "application/pdf", + filename: "image.pdf", + metadata: { language: "zh", requiresTables: true }, + }, + body: new TextEncoder().encode("%PDF"), + knowledgeSpaceId, + permissionScope: [], + tenantId: "tenant-1", + traceId: randomUUID(), + }, + { + ...deps, + profileImageExtractionEnabled: true, + documentPdfRasterizer: { + render: async () => { + throw new Error("Missing page cannot render"); + }, + }, + documentParser: { + kind: "unstructured", + parse: async (input) => { + hints.push(input.parserHints); + return ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-07-13T00:00:00.000Z", + documentAssetId, + elements: [ + { + id: "figure", + type: "image", + sectionPath: [], + metadata: input.parserHints?.imagesHandledExternally + ? {} + : { assetRef: { uri: "data:image/png;base64,AQIDBA==" } }, + }, + ], + id: firstArtifactId, + metadata: {}, + parser: "unstructured", + version: 1, + }); + }, + }, + }, + ); + expect(hints).toEqual([ + expect.objectContaining({ + imagesHandledExternally: true, + requiresImages: true, + language: "zh", + requiresTables: true, + }), + expect.objectContaining({ + imagesHandledExternally: false, + requiresImages: true, + language: "zh", + requiresTables: true, + }), + ]); + expect(artifact.elements[0]?.metadata.assetRef).toMatchObject({ + objectKey: expect.any(String), + }); + }); + + it("uses resolved text-only capabilities for synchronous parsing without losing source references", async () => { + const deps = pipelineDeps(); + const artifact = await compileDocumentArtifact( + { + asset: documentAsset(), + body: new Uint8Array([1]), + knowledgeSpaceId, + permissionScope: [], + tenantId: "tenant-1", + traceId: randomUUID(), + }, + { + ...deps, + resolveDocumentMediaCapabilities: async () => ({ + imageExtractionEnabled: false, + visualEmbeddingEnabled: false, + }), + documentMultimodalRemoteAssetFetcher: { + fetch: async () => { + throw new Error("Text-only profile must not download"); + }, + }, + documentParser: { + kind: "native-markdown", + parse: async (input) => { + expect(input.parserHints).toMatchObject({ requiresImages: false }); + return ParseArtifactSchema.parse({ + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-07-13T00:00:00.000Z", + documentAssetId, + elements: [ + { + id: "figure", + type: "image", + sectionPath: [], + metadata: { assetRef: { uri: "https://example.test/image.png" } }, + }, + ], + id: firstArtifactId, + metadata: {}, + parser: "native-markdown", + version: 1, + }); + }, + }, + }, + ); + expect(artifact.elements[0]?.metadata.assetRef).toEqual({ + uri: "https://example.test/image.png", + }); + expect(artifact.metadata).toMatchObject({ + mediaExecutionPlan: { materializeImages: false }, + parseCoverage: { media: { status: "not-requested" } }, + }); + }); it("materializes parser-provided remote image refs before persisting the manifest", async () => { const adapter = createNodePlatformAdapter({ env: {} }); const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 }); @@ -107,7 +235,11 @@ describe("compileDocumentArtifact canonical artifact", () => { ); expect(remoteFetches).toEqual([ - { maxBytes: 10 * 1024 * 1024, url: "https://cdn.example.test/office-linked.png" }, + { + maxBytes: 10 * 1024 * 1024, + signal: expect.any(AbortSignal), + url: "https://cdn.example.test/office-linked.png", + }, ]); expect(artifact.elements[0]?.metadata).toMatchObject({ assetRef: { @@ -342,3 +474,32 @@ function documentAsset(): DocumentAsset { version: 1, }; } + +function pipelineDeps(): Omit { + return { + artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }), + artifactSegments: createInMemoryArtifactSegmentRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxSegments: 10, + }), + documentMultimodalManifests: createInMemoryDocumentMultimodalManifestRepository({ + maxManifests: 4, + }), + generateArtifactSegmentId: randomUUID, + generateKnowledgePathId: randomUUID, + knowledgePaths: createInMemoryKnowledgePathRepository({ maxListLimit: 20, maxPaths: 20 }), + now: () => "2026-07-13T00:00:00.000Z", + objectStorage: createNodePlatformAdapter({ env: {} }).objectStorage, + outlineBuilder: createDocumentOutlineBuilder({ + generateId: randomUUID, + maxElements: 10, + maxNodes: 10, + maxSummaryChars: 1000, + now: () => "2026-07-13T00:00:00.000Z", + }), + outlines: createInMemoryDocumentOutlineRepository({ maxOutlines: 4 }), + synchronousUploadReindexer: null, + traces: createNoopTraceRecorder(), + }; +} diff --git a/knowledge-fs/packages/api/src/document-compilation-pipeline.ts b/knowledge-fs/packages/api/src/document-compilation-pipeline.ts index 03326e1fe07..94016a0e428 100644 --- a/knowledge-fs/packages/api/src/document-compilation-pipeline.ts +++ b/knowledge-fs/packages/api/src/document-compilation-pipeline.ts @@ -18,6 +18,12 @@ import { buildDocumentOutlineKnowledgePath, buildDocumentSectionKnowledgePaths, } from "./document-knowledge-paths"; +import { + type DocumentMediaCapabilityResolver, + createDocumentMediaExecutionPlan, + documentParserHints, + withDocumentMediaExecutionPlan, +} from "./document-media-execution-plan"; import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact"; import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor"; import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor"; @@ -34,6 +40,7 @@ import type { DocumentOutlineRepository } from "./document-outline-repository"; import type { DocumentOutlineSummaryEnhancer } from "./document-outline-summary-enhancer"; import { type DocumentPdfRasterizer, + DocumentPdfRenderError, rasterizeDocumentPdfMultimodalAssets, } from "./document-pdf-rasterizer"; import { sha256Hex } from "./document-upload-utils"; @@ -60,6 +67,7 @@ export interface CompileDocumentArtifactInput { readonly publicationGenerationId?: string | undefined; readonly tenantId: string; readonly traceId: string; + readonly signal?: AbortSignal | undefined; } export interface CompileDocumentArtifactDeps { @@ -89,6 +97,8 @@ export interface CompileDocumentArtifactDeps { readonly synchronousUploadReindexer: IncrementalReindexer | null; readonly traces: TraceRecorder; readonly visualEmbeddingModel?: string | undefined; + readonly profileImageExtractionEnabled?: boolean | undefined; + readonly resolveDocumentMediaCapabilities?: DocumentMediaCapabilityResolver | undefined; } export async function compileDocumentArtifact( @@ -103,6 +113,7 @@ export async function compileDocumentArtifact( publicationGenerationId: requestedPublicationGenerationId, tenantId, traceId, + signal, } = input; const { artifacts, @@ -131,6 +142,8 @@ export async function compileDocumentArtifact( synchronousUploadReindexer, traces, visualEmbeddingModel, + profileImageExtractionEnabled, + resolveDocumentMediaCapabilities, } = deps; const publicationGenerationId = requestedPublicationGenerationId === undefined @@ -149,58 +162,115 @@ export async function compileDocumentArtifact( } : null; let stagedProjectionIds: readonly string[] = []; + signal?.throwIfAborted(); + const mediaCapabilities = await resolveDocumentMediaCapabilities?.({ + knowledgeSpaceId, + tenantId, + signal, + }); + const mediaPlan = createDocumentMediaExecutionPlan({ + profileImageExtractionEnabled: + mediaCapabilities?.imageExtractionEnabled ?? profileImageExtractionEnabled, + hasPdfRasterizer: Boolean(documentPdfRasterizer), + hasImageVariantGenerator: Boolean(documentMultimodalImageVariantGenerator), + visualEmbeddingEnabled: + mediaCapabilities?.visualEmbeddingEnabled ?? Boolean(visualEmbeddingModel), + }); - const artifact = await traceAsync(traces, traceId, "ingestion.parser_parse", () => - documentParser.parse({ - body, - documentAssetId: asset.id, - filename: asset.filename, - mimeType: asset.mimeType, - version: asset.version, - }), - ); - const rasterizedArtifact = await traceAsync(traces, traceId, "ingestion.pdf_rasterize", () => - rasterizeDocumentPdfMultimodalAssets({ - artifact, - documentBody: body, - documentMimeType: asset.mimeType, - knowledgeSpaceId, - ...(documentMultimodalMaxPdfRasterizedAssets - ? { maxRasterizedAssets: documentMultimodalMaxPdfRasterizedAssets } - : {}), - objectStorage, - ...(documentPdfRasterizer ? { rasterizer: documentPdfRasterizer } : {}), - tenantId, - }), + const externalPdfImages = + Boolean(documentPdfRasterizer) && + asset.mimeType.split(";", 1)[0]?.trim().toLowerCase() === "application/pdf"; + const parse = (imagesHandledExternally: boolean) => + traceAsync(traces, traceId, "ingestion.parser_parse", () => + documentParser.parse({ + body, + documentAssetId: asset.id, + filename: asset.filename, + mimeType: asset.mimeType, + parserHints: documentParserHints({ + assetMetadata: asset.metadata, + imagesHandledExternally, + requiresImages: mediaPlan.requestParserImages, + }), + ...(signal ? { signal } : {}), + version: asset.version, + }), + ); + const artifact = await parse(externalPdfImages); + const rasterizedArtifact = await traceAsync( + traces, + traceId, + "ingestion.pdf_rasterize", + async () => { + if (!mediaPlan.materializeImages) return { artifact }; + try { + const result = await rasterizeDocumentPdfMultimodalAssets({ + artifact, + documentBody: body, + documentMimeType: asset.mimeType, + knowledgeSpaceId, + ...(documentMultimodalMaxPdfRasterizedAssets + ? { maxRasterizedAssets: documentMultimodalMaxPdfRasterizedAssets } + : {}), + objectStorage, + ...(documentPdfRasterizer ? { rasterizer: documentPdfRasterizer } : {}), + tenantId, + ...(signal ? { signal } : {}), + }); + if ( + externalPdfImages && + mediaPlan.requestParserImages && + result.rasterizedCount === 0 && + result.unresolvedCount > 0 + ) + return { artifact: await parse(false) }; + return result; + } catch (error) { + signal?.throwIfAborted(); + if ( + !(error instanceof DocumentPdfRenderError) || + !externalPdfImages || + !mediaPlan.requestParserImages + ) + throw error; + return { artifact: await parse(false) }; + } + }, ); const assetExtractionResult = await traceAsync( traces, traceId, "ingestion.multimodal_assets_extract", () => - extractDocumentMultimodalAssets({ - ...(documentMultimodalLocalAssetAllowlist - ? { allowLocalAssetPaths: documentMultimodalLocalAssetAllowlist } - : {}), - artifact: rasterizedArtifact.artifact, - knowledgeSpaceId, - ...(documentMultimodalMaxExtractedAssets - ? { maxExtractedAssets: documentMultimodalMaxExtractedAssets } - : {}), - ...(documentMultimodalMaxLocalAssetBytes - ? { maxLocalAssetBytes: documentMultimodalMaxLocalAssetBytes } - : {}), - ...(documentMultimodalImageVariantGenerator - ? { imageVariantGenerator: documentMultimodalImageVariantGenerator } - : {}), - objectStorage, - ...(documentMultimodalRemoteAssetFetcher - ? { remoteAssetFetcher: documentMultimodalRemoteAssetFetcher } - : {}), - tenantId, - }), + mediaPlan.materializeImages + ? extractDocumentMultimodalAssets({ + ...(documentMultimodalLocalAssetAllowlist + ? { allowLocalAssetPaths: documentMultimodalLocalAssetAllowlist } + : {}), + artifact: rasterizedArtifact.artifact, + knowledgeSpaceId, + ...(documentMultimodalMaxExtractedAssets + ? { maxExtractedAssets: documentMultimodalMaxExtractedAssets } + : {}), + ...(documentMultimodalMaxLocalAssetBytes + ? { maxLocalAssetBytes: documentMultimodalMaxLocalAssetBytes } + : {}), + ...(documentMultimodalImageVariantGenerator + ? { imageVariantGenerator: documentMultimodalImageVariantGenerator } + : {}), + objectStorage, + ...(documentMultimodalRemoteAssetFetcher + ? { remoteAssetFetcher: documentMultimodalRemoteAssetFetcher } + : {}), + tenantId, + ...(signal ? { signal } : {}), + }) + : Promise.resolve({ artifact: rasterizedArtifact.artifact, extractedCount: 0 }), + ); + signal?.throwIfAborted(); + const materializedArtifact = finalizeDocumentMultimodalArtifact( + withDocumentMediaExecutionPlan(assetExtractionResult.artifact, mediaPlan), ); - const materializedArtifact = finalizeDocumentMultimodalArtifact(assetExtractionResult.artifact); const artifactToPersist = ParseArtifactSchema.parse({ ...materializedArtifact, metadata: { @@ -300,7 +370,11 @@ export async function compileDocumentArtifact( projectionVersion: asset.version, ...(publicationGenerationId !== undefined ? { publicationGenerationId } : {}), tenantId, - ...(visualEmbeddingModel ? { visualModel: visualEmbeddingModel } : {}), + ...(visualEmbeddingModel && mediaPlan.visualEmbedding + ? { visualModel: visualEmbeddingModel } + : mediaCapabilities || profileImageExtractionEnabled === false + ? { skipVisual: true as const } + : {}), }), ); if (stagedProjectionPublication && reindexResult.status === "rebuilt") { diff --git a/knowledge-fs/packages/api/src/document-compilation-worker.test.ts b/knowledge-fs/packages/api/src/document-compilation-worker.test.ts index 8721ccec87a..55dc60de622 100644 --- a/knowledge-fs/packages/api/src/document-compilation-worker.test.ts +++ b/knowledge-fs/packages/api/src/document-compilation-worker.test.ts @@ -223,7 +223,14 @@ describe("createDocumentCompilationWorker lease integration", () => { contentType: "text", createdAt: "2026-09-01T12:00:00.000Z", documentAssetId: input.documentAssetId, - elements: [], + elements: [ + { + id: "inline-profile-image", + type: "image", + sectionPath: [], + metadata: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } }, + }, + ], id: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a16", metadata: {}, parser: "unstructured", @@ -238,13 +245,22 @@ describe("createDocumentCompilationWorker lease integration", () => { }, profileImageExtractionEnabled, reindexer: { - reindex: async (input) => ({ - artifact: input.parseArtifact, - nodesCreated: 0, - projectionIds: [], - projectionsCreated: 0, - status: "rebuilt", - }), + reindex: async (input) => { + const assetRef = input.parseArtifact.elements[0]?.metadata.assetRef; + if (!profileImageExtractionEnabled) expect(input.skipVisual).toBe(true); + expect(assetRef).toMatchObject( + profileImageExtractionEnabled + ? { objectKey: expect.any(String) } + : { uri: "data:image/png;base64,AQIDBA==" }, + ); + return { + artifact: input.parseArtifact, + nodesCreated: 0, + projectionIds: [], + projectionsCreated: 0, + status: "rebuilt", + }; + }, }, }); diff --git a/knowledge-fs/packages/api/src/document-compilation-worker.ts b/knowledge-fs/packages/api/src/document-compilation-worker.ts index fb787be61f8..6d21db6f39a 100644 --- a/knowledge-fs/packages/api/src/document-compilation-worker.ts +++ b/knowledge-fs/packages/api/src/document-compilation-worker.ts @@ -53,6 +53,11 @@ import { buildDocumentOutlineKnowledgePath, buildDocumentSectionKnowledgePaths, } from "./document-knowledge-paths"; +import { + createDocumentMediaExecutionPlan, + documentParserHints, + withDocumentMediaExecutionPlan, +} from "./document-media-execution-plan"; import type { DocumentModelBudget } from "./document-model-budget"; import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact"; import { @@ -465,14 +470,14 @@ export function createDocumentCompilationWorker({ signal ? { signal } : undefined, ); } else { - const requiresImages = - profileImageExtractionEnabled ?? - Boolean( - visualEmbeddingMode === "profile" || - visualEmbeddingModel || - multimodalImageVariantGenerator || - pdfRasterizer, - ); + const mediaPlan = createDocumentMediaExecutionPlan({ + profileImageExtractionEnabled, + hasPdfRasterizer: Boolean(pdfRasterizer), + hasImageVariantGenerator: Boolean(multimodalImageVariantGenerator), + visualEmbeddingEnabled: + visualEmbeddingMode === "profile" || Boolean(visualEmbeddingModel), + }); + const requiresImages = mediaPlan.requestParserImages; const externalPdfImages = Boolean(pdfRasterizer) && isPdfDocument(activeAsset.mimeType); const primaryParserHints = documentParserHints({ assetMetadata: activeAsset.metadata, @@ -629,7 +634,10 @@ export function createDocumentCompilationWorker({ let multimodalArtifact: ParseArtifact; try { - if (activeParserOutput.route === "provider-fallback") { + if ( + activeParserOutput.route === "provider-fallback" || + !mediaPlan.materializeImages + ) { multimodalArtifact = activeParserOutput.artifact; } else { const rasterized = await rasterizeDocumentPdfMultimodalAssets({ @@ -672,31 +680,35 @@ export function createDocumentCompilationWorker({ multimodalArtifact = activeParserOutput.artifact; } await assertWritable(); - const { artifact } = await extractDocumentMultimodalAssets({ - ...(multimodalLocalAssetAllowlist - ? { allowLocalAssetPaths: multimodalLocalAssetAllowlist } - : {}), - artifact: multimodalArtifact, - knowledgeSpaceId: input.knowledgeSpaceId, - ...(multimodalMaxExtractedAssets - ? { maxExtractedAssets: multimodalMaxExtractedAssets } - : {}), - ...(multimodalMaxLocalAssetBytes - ? { maxLocalAssetBytes: multimodalMaxLocalAssetBytes } - : {}), - ...(multimodalImageVariantGenerator - ? { imageVariantGenerator: multimodalImageVariantGenerator } - : {}), - objectStorage: multimodalObjectStorage, - ...(multimodalRemoteAssetFetcher - ? { remoteAssetFetcher: multimodalRemoteAssetFetcher } - : {}), - ...(signal ? { signal } : {}), - tenantId: input.tenantId, - writeOwnerId: multimodalWriteOwnerId, - }); + const { artifact } = mediaPlan.materializeImages + ? await extractDocumentMultimodalAssets({ + ...(multimodalLocalAssetAllowlist + ? { allowLocalAssetPaths: multimodalLocalAssetAllowlist } + : {}), + artifact: multimodalArtifact, + knowledgeSpaceId: input.knowledgeSpaceId, + ...(multimodalMaxExtractedAssets + ? { maxExtractedAssets: multimodalMaxExtractedAssets } + : {}), + ...(multimodalMaxLocalAssetBytes + ? { maxLocalAssetBytes: multimodalMaxLocalAssetBytes } + : {}), + ...(multimodalImageVariantGenerator + ? { imageVariantGenerator: multimodalImageVariantGenerator } + : {}), + objectStorage: multimodalObjectStorage, + ...(multimodalRemoteAssetFetcher + ? { remoteAssetFetcher: multimodalRemoteAssetFetcher } + : {}), + ...(signal ? { signal } : {}), + tenantId: input.tenantId, + writeOwnerId: multimodalWriteOwnerId, + }) + : { artifact: multimodalArtifact }; await assertWritable(); - const finalizedArtifact = finalizeDocumentMultimodalArtifact(artifact); + const finalizedArtifact = finalizeDocumentMultimodalArtifact( + withDocumentMediaExecutionPlan(artifact, mediaPlan), + ); const checkpointPolicyFingerprint = activeParserOutput.rawCheckpointPolicyFingerprint ?? persistedCheckpoint?.policyFingerprint; @@ -948,12 +960,18 @@ export function createDocumentCompilationWorker({ : (resolvedEmbedding?.vectorSpaceId ?? denseEmbeddingModel); await assertWritable(); const resolvedVisualEmbeddingModel = - visualEmbeddingMode === "profile" - ? frozenEmbeddingProfile?.model - : visualEmbeddingMode === "disabled" - ? undefined - : visualEmbeddingModel; - if (visualEmbeddingMode === "profile" && !resolvedVisualEmbeddingModel) { + profileImageExtractionEnabled === false + ? undefined + : visualEmbeddingMode === "profile" + ? frozenEmbeddingProfile?.model + : visualEmbeddingMode === "disabled" + ? undefined + : visualEmbeddingModel; + if ( + profileImageExtractionEnabled !== false && + visualEmbeddingMode === "profile" && + !resolvedVisualEmbeddingModel + ) { throw new Error("Profile-driven visual embedding requires a frozen embedding profile"); } const reindexResult = await reindexer.reindex({ @@ -985,7 +1003,7 @@ export function createDocumentCompilationWorker({ tenantId: input.tenantId, ...(resolvedVisualEmbeddingModel ? { visualModel: resolvedVisualEmbeddingModel } - : visualEmbeddingMode === "disabled" + : visualEmbeddingMode === "disabled" || profileImageExtractionEnabled === false ? { skipVisual: true as const } : {}), }); @@ -1307,30 +1325,6 @@ function isPdfDocument(mimeType: string): boolean { return mimeType.split(";", 1)[0]?.trim().toLowerCase() === "application/pdf"; } -function documentParserHints(input: { - readonly assetMetadata: Readonly>; - readonly imagesHandledExternally: boolean; - readonly requiresImages: boolean; -}): ParserRouteHints { - const language = - typeof input.assetMetadata.language === "string" && input.assetMetadata.language.trim() - ? input.assetMetadata.language.trim() - : undefined; - const layoutComplexity = - input.assetMetadata.layoutComplexity === "complex" || - input.assetMetadata.layoutComplexity === "simple" - ? input.assetMetadata.layoutComplexity - : undefined; - return { - imagesHandledExternally: input.imagesHandledExternally, - ...(language ? { language } : {}), - ...(layoutComplexity ? { layoutComplexity } : {}), - requiresImages: input.requiresImages, - ...(input.assetMetadata.requiresOcr === true ? { requiresOcr: true } : {}), - ...(input.assetMetadata.requiresTables === true ? { requiresTables: true } : {}), - }; -} - type ParseCheckpointRoute = "primary" | "provider-fallback"; interface ParserCheckpointPolicy { diff --git a/knowledge-fs/packages/api/src/document-image-variant-generator.test.ts b/knowledge-fs/packages/api/src/document-image-variant-generator.test.ts index 18a5eba16b3..aedbf36596a 100644 --- a/knowledge-fs/packages/api/src/document-image-variant-generator.test.ts +++ b/knowledge-fs/packages/api/src/document-image-variant-generator.test.ts @@ -3,6 +3,48 @@ import { describe, expect, it } from "vitest"; import { createSharpImageThumbnailVariantGenerator } from "./document-image-variant-generator"; describe("createSharpImageThumbnailVariantGenerator", () => { + it.each([0, 319, 4097, Number.NaN])( + "rejects unsafe analysis dimension %s", + (analysisMaxDimension) => { + expect(() => createSharpImageThumbnailVariantGenerator({ analysisMaxDimension })).toThrow( + "Sharp image analysis dimension", + ); + }, + ); + it("requires distinct variant names and skips non-image/empty input", async () => { + expect(() => + createSharpImageThumbnailVariantGenerator({ + analysisMaxDimension: 640, + variantName: "analysis", + }), + ).toThrow("distinct variant names"); + const generator = createSharpImageThumbnailVariantGenerator({ analysisMaxDimension: 640 }); + await expect( + generator.generate({ + body: new Uint8Array([1]), + contentType: "text/plain", + elementId: "text", + }), + ).resolves.toEqual([]); + await expect( + generator.generate({ body: new Uint8Array(), contentType: "image/png", elementId: "empty" }), + ).resolves.toEqual([]); + }); + it("keeps a separate analysis image with enough detail instead of feeding the preview thumbnail to vision", async () => { + const sharp = (await import("sharp")).default; + const body = await sharp({ + create: { background: "white", channels: 3, width: 800, height: 400 }, + }) + .png() + .toBuffer(); + const variants = await createSharpImageThumbnailVariantGenerator({ + analysisMaxDimension: 640, + }).generate({ body, contentType: "image/png", elementId: "figure" }); + expect(variants.map(({ name, width, height }) => ({ name, width, height }))).toEqual([ + { name: "thumbnail", width: 320, height: 160 }, + { name: "analysis", width: 640, height: 320 }, + ]); + }); it("generates bounded PNG thumbnail variants from image bytes", async () => { const sharp = (await import("sharp")).default; const generator = createSharpImageThumbnailVariantGenerator({ diff --git a/knowledge-fs/packages/api/src/document-image-variant-generator.ts b/knowledge-fs/packages/api/src/document-image-variant-generator.ts index 0f02050516d..6fc1962e2c0 100644 --- a/knowledge-fs/packages/api/src/document-image-variant-generator.ts +++ b/knowledge-fs/packages/api/src/document-image-variant-generator.ts @@ -2,9 +2,19 @@ export interface GenerateDocumentImageVariantsInput { readonly body: Uint8Array; readonly contentType: string; readonly elementId: string; + readonly signal?: AbortSignal | undefined; } export interface GeneratedDocumentImageVariant { + readonly execution?: + | { + readonly isolation: "child-process"; + readonly inputBytes: number; + readonly outputBytes: number; + readonly wallMs: number; + readonly peakRssKiB: number; + } + | undefined; readonly body: Uint8Array; readonly contentType: string; readonly height?: number | undefined; @@ -19,6 +29,8 @@ export interface DocumentImageVariantGenerator { } export interface SharpImageThumbnailVariantGeneratorOptions { + /** Separate vision input; preview dimensions must never become model input resolution. */ + readonly analysisMaxDimension?: number | undefined; readonly maxDimension?: number | undefined; readonly maxInputPixels?: number | undefined; readonly maxOutputBytes?: number | undefined; @@ -31,6 +43,7 @@ const defaultThumbnailMaxOutputBytes = 8 * 1024 * 1024; const defaultThumbnailVariantName = "thumbnail"; export function createSharpImageThumbnailVariantGenerator({ + analysisMaxDimension, maxDimension = defaultThumbnailMaxDimension, maxInputPixels = defaultThumbnailMaxInputPixels, maxOutputBytes = defaultThumbnailMaxOutputBytes, @@ -52,8 +65,50 @@ export function createSharpImageThumbnailVariantGenerator({ throw new Error("Sharp image thumbnail variantName must be non-empty"); } + if ( + analysisMaxDimension !== undefined && + (!Number.isSafeInteger(analysisMaxDimension) || + analysisMaxDimension < maxDimension || + analysisMaxDimension > 4096 || + variantName === "analysis") + ) { + throw new Error( + "Sharp image analysis dimension must be between thumbnail dimension and 4096, with distinct variant names", + ); + } + + if (analysisMaxDimension !== undefined) { + const analysisGenerator = createSharpImageThumbnailVariantGenerator({ + maxDimension: analysisMaxDimension, + maxInputPixels, + maxOutputBytes, + variantName: "analysis", + }); + const previewGenerator = createSharpImageThumbnailVariantGenerator({ + maxDimension, + maxInputPixels, + maxOutputBytes, + variantName, + }); + return { + generate: async (input) => { + input.signal?.throwIfAborted(); + const analysis = await analysisGenerator.generate(input); + const main = analysis[0]; + if (!main) return []; + const preview = await previewGenerator.generate({ + ...input, + body: main.body, + contentType: main.contentType, + }); + return [...preview, ...analysis]; + }, + }; + } + return { - generate: async ({ body, contentType }) => { + generate: async ({ body, contentType, signal }) => { + signal?.throwIfAborted(); if (!contentType.toLowerCase().startsWith("image/") || body.byteLength === 0) { return []; } @@ -69,6 +124,7 @@ export function createSharpImageThumbnailVariantGenerator({ }) .png() .toBuffer({ resolveWithObject: true }); + signal?.throwIfAborted(); if (data.byteLength > maxOutputBytes) { throw new Error( diff --git a/knowledge-fs/packages/api/src/document-knowledge-paths.test.ts b/knowledge-fs/packages/api/src/document-knowledge-paths.test.ts index a56948dcb84..eaed44a4ab8 100644 --- a/knowledge-fs/packages/api/src/document-knowledge-paths.test.ts +++ b/knowledge-fs/packages/api/src/document-knowledge-paths.test.ts @@ -107,6 +107,7 @@ describe("document KnowledgeFS paths", () => { ).toEqual([ expect.objectContaining({ metadata: expect.objectContaining({ + analysisUnavailable: { reason: "variant-pixel-budget" }, assetContentType: "image/png", contentKind: "document-multimodal-asset", filename: "image-架构图--018f0d60.json", @@ -400,6 +401,7 @@ function documentMultimodalManifest( items: [ { assetRef: { + analysisUnavailable: { reason: "variant-pixel-budget" }, contentType: "image/png", objectKey: "tenant-dev/spaces/ab96d0c6-5853-4979-ba1a-98128e54fa7d/documents/6c07b8ca-dd64-4ccd-a6e9-91f816795412/assets/figure-1.png", diff --git a/knowledge-fs/packages/api/src/document-knowledge-paths.ts b/knowledge-fs/packages/api/src/document-knowledge-paths.ts index 07865f6daaa..2b55d15c1db 100644 --- a/knowledge-fs/packages/api/src/document-knowledge-paths.ts +++ b/knowledge-fs/packages/api/src/document-knowledge-paths.ts @@ -144,6 +144,9 @@ export function buildDocumentMultimodalAssetKnowledgePaths({ }), knowledgeSpaceId: asset.knowledgeSpaceId, metadata: { + ...(item.assetRef?.analysisUnavailable + ? { analysisUnavailable: { ...item.assetRef.analysisUnavailable } } + : {}), ...(item.assetRef?.contentType ? { assetContentType: item.assetRef.contentType } : {}), ...(item.assetRef?.objectKey ? { objectKey: item.assetRef.objectKey } : {}), ...(item.assetRef?.sha256 ? { sha256: item.assetRef.sha256 } : {}), @@ -398,6 +401,9 @@ function buildDocumentMultimodalItemResourceKnowledgePath({ }), knowledgeSpaceId: asset.knowledgeSpaceId, metadata: { + ...(item.assetRef?.analysisUnavailable + ? { analysisUnavailable: { ...item.assetRef.analysisUnavailable } } + : {}), ...(item.assetRef?.contentType ? { assetContentType: item.assetRef.contentType } : {}), ...(item.assetRef?.objectKey ? { objectKey: item.assetRef.objectKey } : {}), ...(item.assetRef?.sha256 ? { sha256: item.assetRef.sha256 } : {}), diff --git a/knowledge-fs/packages/api/src/document-media-analysis-safety.test.ts b/knowledge-fs/packages/api/src/document-media-analysis-safety.test.ts new file mode 100644 index 00000000000..d5ad7af4eec --- /dev/null +++ b/knowledge-fs/packages/api/src/document-media-analysis-safety.test.ts @@ -0,0 +1,259 @@ +import { createNodePlatformAdapter } from "@knowledge/adapters/node"; +import { createTypeScriptComputeRuntime } from "@knowledge/compute"; +import { DocumentMultimodalAssetRefSchema, type ParseArtifact } from "@knowledge/core"; +import { describe, expect, it, vi } from "vitest"; + +import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor"; +import { createUnderstandingDocumentMultimodalEnrichmentProvider } from "./document-multimodal-enrichment-providers"; +import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder"; +import { createDocumentMultimodalManifestEnhancer } from "./document-multimodal-manifest-enhancer"; +import { createObjectStorageVisualEmbeddingProvider } from "./index-projection-builders"; +import { + createContentBlockMultimodalAnswerProvider, + createObjectStorageContentBlockMultimodalAnswerProvider, +} from "./llm-multimodal-answer-provider"; + +const knowledgeSpaceId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42"; +const rejected = { reason: "variant-pixel-budget" as const }; +const assetRef = { + analysisUnavailable: rejected, + contentType: "image/png", + objectKey: "tenant/assets/image.png", +}; + +describe("rejected document image analysis", () => { + it.each([ + ["count", ""], + ["bytes", ""], + ["count", " \n\t"], + ["bytes", " \n\t"], + ])( + "keeps overlong inline sources recoverable but bounded in manifests after %s limits with prefix %j", + async (limit, prefix) => { + const uri = `${prefix}data:image/png;base64,${Buffer.alloc(4096).toString("base64")}`; + const artifact: ParseArtifact = { + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + elements: ["data:image/png;base64,AQID", uri].map((source, index) => ({ + id: `image-${index}`, + type: "image", + sectionPath: [], + metadata: { assetRef: { uri: source } }, + })), + metadata: {}, + parser: "native-markdown", + version: 1, + }; + const extracted = await extractDocumentMultimodalAssets({ + artifact, + knowledgeSpaceId, + tenantId: "tenant", + objectStorage: createNodePlatformAdapter({ env: {} }).objectStorage, + ...(limit === "count" ? { maxExtractedAssets: 1 } : { maxTotalAssetBytes: 3 }), + }); + expect(extracted.artifact.elements[1]?.metadata.assetRef).toMatchObject({ uri }); + const manifest = createDocumentMultimodalManifestBuilder().build({ + artifact: extracted.artifact, + knowledgeSpaceId, + }); + const item = manifest.items[1]; + expect(item?.assetRef).toEqual({ + analysisUnavailable: { + reason: limit === "count" ? "asset-count-budget" : "materialized-byte-budget", + }, + }); + expect(item?.enrichment.asset).toBe("missing"); + expect(item?.sourceMetadata.assetRef).toMatchObject({ + sourceUriSha256: expect.stringMatching(/^[a-f0-9]{64}$/u), + }); + expect(JSON.stringify(manifest)).not.toContain(uri); + const enriched = await createDocumentMultimodalManifestEnhancer({ + maxItems: 2, + maxSourceTextChars: 100, + model: "vision", + promptVersion: "v1", + provider: { + enrich: async () => ({ + caption: "Text-only metadata", + visualEmbeddingStatus: "provided", + }), + }, + }).enhance({ manifest, parseArtifact: extracted.artifact }); + expect(enriched.items[1]?.enrichment).toMatchObject({ + asset: "missing", + visualEmbedding: "unsupported", + }); + }, + ); + it("bounds the reason contract and preserves the marker through extraction, manifest, and nodes", async () => { + expect(DocumentMultimodalAssetRefSchema.parse(assetRef)).toEqual(assetRef); + expect( + DocumentMultimodalAssetRefSchema.safeParse({ + ...assetRef, + analysisUnavailable: { reason: "x".repeat(5000) }, + }).success, + ).toBe(false); + const objectStorage = createNodePlatformAdapter({ env: {} }).objectStorage; + const parseArtifact: ParseArtifact = { + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + elements: [ + { + id: "image-1", + type: "image", + sectionPath: [], + text: "Readable caption", + metadata: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } }, + }, + ], + metadata: {}, + parser: "native-markdown", + version: 1, + }; + const { artifact } = await extractDocumentMultimodalAssets({ + artifact: parseArtifact, + objectStorage, + knowledgeSpaceId, + tenantId: "tenant", + maxTotalVariantPixels: 1, + imageVariantGenerator: { generate: vi.fn() }, + }); + const manifest = createDocumentMultimodalManifestBuilder().build({ + artifact, + knowledgeSpaceId, + }); + const item = manifest.items[0]; + expect(item?.assetRef).toMatchObject({ + analysisUnavailable: rejected, + objectKey: expect.any(String), + }); + expect(item?.enrichment.visualEmbedding).toBe("unsupported"); + const nodes = createTypeScriptComputeRuntime().chunkParseArtifact({ + parseArtifact: artifact, + knowledgeSpaceId, + }); + expect(nodes[0]?.metadata.assetRef).toMatchObject({ analysisUnavailable: rejected }); + expect(nodes[0]?.text).toBe("Readable caption"); + const originalKey = item?.assetRef?.objectKey; + const node = nodes[0]; + if (!originalKey || !node) throw new Error("Expected retained original and text node"); + expect(await objectStorage.getObject(originalKey)).not.toBeNull(); + const embedImages = vi.fn(); + const embedding = createObjectStorageVisualEmbeddingProvider({ + objectStorage, + provider: { embedImages }, + }); + const result = await embedding.embedAssets({ + model: "vision", + assets: [ + { + assetRef: node.metadata.assetRef as Record, + documentAssetId: artifact.documentAssetId, + metadata: {}, + modality: "image", + nodeId: node.id, + sourceText: "Readable caption", + }, + ], + }); + expect(result.dense).toEqual([]); + expect(embedImages).not.toHaveBeenCalled(); + }); + + it.each(["object", "url"])( + "does not load rejected originals in %s answer consumers", + async (kind) => { + const objectStorage = createNodePlatformAdapter({ env: {} }).objectStorage; + await objectStorage.putObject({ + key: assetRef.objectKey, + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }); + const get = vi.spyOn(objectStorage, "getObjectStream"); + const generate = vi.fn(async () => ({ text: "Answer from caption" })); + const resolve = vi.fn(() => "https://example.com/original.png"); + const provider = + kind === "object" + ? createObjectStorageContentBlockMultimodalAnswerProvider({ + model: "vision", + objectStorage, + provider: { generate }, + }) + : createContentBlockMultimodalAnswerProvider({ + model: "vision", + assetUrlResolver: resolve, + provider: { generate }, + }); + await expect( + provider.generate({ + query: "What does it show?", + evidence: [], + multimodalEvidence: [ + { + assetRef, + documentAssetId: "doc-1", + modality: "image", + parseElementId: "image-1", + sectionPath: [], + caption: "Readable caption", + }, + ], + }), + ).resolves.toMatchObject({ metadata: { imageBlockCount: 0 } }); + expect(get).not.toHaveBeenCalled(); + expect(resolve).not.toHaveBeenCalled(); + expect(JSON.stringify(generate.mock.calls)).toContain("parseElement=image-1"); + }, + ); + + it("does not delegate flagged images to understanding providers or let enrichment clear the marker", async () => { + const artifact: ParseArtifact = { + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + elements: [{ id: "image-1", type: "image", sectionPath: [], metadata: { assetRef } }], + metadata: {}, + parser: "native-markdown", + version: 1, + }; + const manifest = createDocumentMultimodalManifestBuilder().build({ + artifact, + knowledgeSpaceId, + }); + const understand = vi.fn(async () => ({ caption: "Should not execute" })); + const enhancer = createDocumentMultimodalManifestEnhancer({ + maxItems: 1, + maxSourceTextChars: 100, + model: "vision", + promptVersion: "v1", + provider: createUnderstandingDocumentMultimodalEnrichmentProvider({ + provider: { understand }, + }), + }); + await enhancer.enhance({ manifest, parseArtifact: artifact }); + expect(understand).not.toHaveBeenCalled(); + const rewriting = createDocumentMultimodalManifestEnhancer({ + maxItems: 1, + maxSourceTextChars: 100, + model: "vision", + promptVersion: "v1", + provider: { + enrich: async () => ({ + assetRef: { objectKey: assetRef.objectKey, contentType: "image/png" }, + caption: "Text-only caption", + }), + }, + }); + expect( + (await rewriting.enhance({ manifest, parseArtifact: artifact })).items[0]?.assetRef, + ).toMatchObject({ analysisUnavailable: rejected }); + }); +}); diff --git a/knowledge-fs/packages/api/src/document-media-execution-plan.test.ts b/knowledge-fs/packages/api/src/document-media-execution-plan.test.ts new file mode 100644 index 00000000000..7e3ddc882da --- /dev/null +++ b/knowledge-fs/packages/api/src/document-media-execution-plan.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; +import { + createDocumentMediaExecutionPlan, + createProfileDocumentMediaCapabilityResolver, + documentParserHints, + withDocumentMediaExecutionPlan, +} from "./document-media-execution-plan"; +import type { KnowledgeSpaceProfileHead } from "./knowledge-space-profile-repository"; + +describe("immutable document media execution plan", () => { + it("persists the immutable plan without replacing other completeness diagnostics", () => { + const original = { + metadata: { parseCoverage: { text: { status: "complete" }, media: { status: "unknown" } } }, + } as unknown as import("@knowledge/core").ParseArtifact; + const plan = createDocumentMediaExecutionPlan({ + hasPdfRasterizer: false, + hasImageVariantGenerator: false, + visualEmbeddingEnabled: true, + }); + expect(withDocumentMediaExecutionPlan(original, plan).metadata).toMatchObject({ + mediaExecutionPlan: plan, + parseCoverage: original.metadata.parseCoverage, + }); + expect( + withDocumentMediaExecutionPlan(original, { ...plan, materializeImages: false }).metadata, + ).toMatchObject({ + parseCoverage: { text: { status: "complete" }, media: { status: "not-requested" } }, + }); + expect(withDocumentMediaExecutionPlan({ ...original, metadata: {} }, plan).metadata).toEqual({ + mediaExecutionPlan: plan, + }); + }); + it("keeps source references but disables rasterization and decoding for a text-only profile", () => { + const plan = createDocumentMediaExecutionPlan({ + profileImageExtractionEnabled: false, + hasPdfRasterizer: true, + hasImageVariantGenerator: true, + visualEmbeddingEnabled: false, + }); + expect(plan).toMatchObject({ + preserveSourceReferences: true, + materializeImages: false, + rasterizePdf: false, + generateVariants: false, + visualEmbedding: false, + }); + expect(Object.isFrozen(plan)).toBe(true); + }); + it("enables vision materialization independently of available embedding support", () => { + expect( + createDocumentMediaExecutionPlan({ + profileImageExtractionEnabled: true, + hasPdfRasterizer: true, + hasImageVariantGenerator: true, + visualEmbeddingEnabled: false, + }), + ).toMatchObject({ + materializeImages: true, + rasterizePdf: true, + generateVariants: true, + visualEmbedding: false, + }); + }); + it("preserves legacy inline-image materialization without explicitly configured profiles", () => { + expect( + createDocumentMediaExecutionPlan({ + hasPdfRasterizer: false, + hasImageVariantGenerator: false, + visualEmbeddingEnabled: false, + }), + ).toMatchObject({ materializeImages: true, rasterizePdf: false, generateVariants: false }); + }); + it("captures identical bounded parser hints for synchronous and durable compilation", () => { + expect( + documentParserHints({ + assetMetadata: { + language: " zh ", + layoutComplexity: "complex", + requiresOcr: true, + requiresTables: true, + }, + imagesHandledExternally: false, + requiresImages: true, + }), + ).toEqual({ + language: "zh", + layoutComplexity: "complex", + requiresOcr: true, + requiresTables: true, + imagesHandledExternally: false, + requiresImages: true, + }); + expect( + documentParserHints({ + assetMetadata: { language: 1, layoutComplexity: "invalid" }, + imagesHandledExternally: false, + requiresImages: false, + }), + ).toEqual({ imagesHandledExternally: false, requiresImages: false }); + }); + it("resolves synchronous capabilities once from the space's immutable profile snapshots", async () => { + const resolver = createProfileDocumentMediaCapabilityResolver({ + profiles: { + getHead: async ({ kind }) => + ({ + profile: { + capabilitySnapshot: + kind === "embedding" ? { image: false } : { reasoning: { image: true } }, + }, + }) as unknown as KnowledgeSpaceProfileHead, + }, + modalities: { + resolve: async ({ snapshot }) => + (snapshot as { image: boolean }).image ? ["text", "image"] : ["text"], + }, + }); + await expect(resolver({ knowledgeSpaceId: "space", tenantId: "tenant" })).resolves.toEqual({ + imageExtractionEnabled: true, + visualEmbeddingEnabled: false, + }); + }); + it("fails closed when neither profile contains a vision capability", async () => { + const resolver = createProfileDocumentMediaCapabilityResolver({ + profiles: { getHead: async () => null }, + modalities: { + resolve: async () => { + throw new Error("No snapshot should resolve"); + }, + }, + }); + await expect(resolver({ knowledgeSpaceId: "space", tenantId: "tenant" })).resolves.toEqual({ + imageExtractionEnabled: false, + visualEmbeddingEnabled: false, + }); + }); +}); diff --git a/knowledge-fs/packages/api/src/document-media-execution-plan.ts b/knowledge-fs/packages/api/src/document-media-execution-plan.ts new file mode 100644 index 00000000000..0ef4ee847f1 --- /dev/null +++ b/knowledge-fs/packages/api/src/document-media-execution-plan.ts @@ -0,0 +1,130 @@ +import type { ParseArtifact } from "@knowledge/core"; +import type { ParserRouteHints } from "@knowledge/parsers"; +import { isPlainObject } from "./json-utils"; +import type { KnowledgeSpaceProfileRepository } from "./knowledge-space-profile-repository"; +import type { ModelInputModality } from "./model-capability-preflight"; +import type { ModelInputModalityResolver } from "./model-input-modality-resolver"; + +export type DocumentMediaCapabilityResolver = (input: { + readonly knowledgeSpaceId: string; + readonly tenantId: string; + readonly signal?: AbortSignal | undefined; +}) => Promise<{ + readonly imageExtractionEnabled: boolean; + readonly visualEmbeddingEnabled: boolean; +}>; + +/** Legacy synchronous ingestion has no publication attempt: freeze its two heads before parsing. */ +export function createProfileDocumentMediaCapabilityResolver(input: { + readonly profiles: Pick; + readonly modalities: ModelInputModalityResolver; +}): DocumentMediaCapabilityResolver { + return async ({ knowledgeSpaceId, tenantId, signal }) => { + signal?.throwIfAborted(); + const [embedding, retrieval] = await Promise.all([ + input.profiles.getHead({ knowledgeSpaceId, tenantId, kind: "embedding" }), + input.profiles.getHead({ knowledgeSpaceId, tenantId, kind: "retrieval" }), + ]); + const [embeddingModalities, reasoningModalities] = await Promise.all([ + embedding + ? input.modalities.resolve({ + snapshot: embedding.profile.capabilitySnapshot, + tenantId, + signal, + }) + : ([] as readonly ModelInputModality[]), + retrieval?.profile.capabilitySnapshot.reasoning + ? input.modalities.resolve({ + snapshot: retrieval.profile.capabilitySnapshot.reasoning, + tenantId, + signal, + }) + : ([] as readonly ModelInputModality[]), + ]); + signal?.throwIfAborted(); + const visualEmbeddingEnabled = embeddingModalities.includes("image"); + return Object.freeze({ + imageExtractionEnabled: visualEmbeddingEnabled || reasoningModalities.includes("image"), + visualEmbeddingEnabled, + }); + }; +} + +export interface DocumentMediaExecutionPlan { + readonly version: "document-media-v1"; + readonly preserveSourceReferences: true; + readonly materializeImages: boolean; + readonly requestParserImages: boolean; + readonly rasterizePdf: boolean; + readonly generateVariants: boolean; + readonly visualEmbedding: boolean; +} + +/** Capabilities are captured once; installed tools must not override an explicit text-only profile. */ +export function createDocumentMediaExecutionPlan(input: { + readonly profileImageExtractionEnabled?: boolean | undefined; + readonly hasPdfRasterizer: boolean; + readonly hasImageVariantGenerator: boolean; + readonly visualEmbeddingEnabled: boolean; +}): DocumentMediaExecutionPlan { + const materializeImages = input.profileImageExtractionEnabled !== false; + return Object.freeze({ + version: "document-media-v1", + preserveSourceReferences: true, + materializeImages, + requestParserImages: + input.profileImageExtractionEnabled ?? + Boolean( + input.hasPdfRasterizer || input.hasImageVariantGenerator || input.visualEmbeddingEnabled, + ), + rasterizePdf: materializeImages && input.hasPdfRasterizer, + generateVariants: materializeImages && input.hasImageVariantGenerator, + visualEmbedding: materializeImages && input.visualEmbeddingEnabled, + }); +} + +export function documentParserHints(input: { + readonly assetMetadata: Readonly>; + readonly imagesHandledExternally: boolean; + readonly requiresImages: boolean; +}): ParserRouteHints { + const language = + typeof input.assetMetadata.language === "string" ? input.assetMetadata.language.trim() : ""; + const layoutComplexity = + input.assetMetadata.layoutComplexity === "complex" || + input.assetMetadata.layoutComplexity === "simple" + ? input.assetMetadata.layoutComplexity + : undefined; + return Object.freeze({ + imagesHandledExternally: input.imagesHandledExternally, + ...(language ? { language } : {}), + ...(layoutComplexity ? { layoutComplexity } : {}), + requiresImages: input.requiresImages, + ...(input.assetMetadata.requiresOcr === true ? { requiresOcr: true } : {}), + ...(input.assetMetadata.requiresTables === true ? { requiresTables: true } : {}), + }); +} + +export function withDocumentMediaExecutionPlan( + artifact: ParseArtifact, + plan: DocumentMediaExecutionPlan, +): ParseArtifact { + const coverage = isPlainObject(artifact.metadata.parseCoverage) + ? artifact.metadata.parseCoverage + : {}; + return { + ...artifact, + metadata: { + ...artifact.metadata, + mediaExecutionPlan: { ...plan }, + ...(!plan.materializeImages + ? { + parseCoverage: { + ...coverage, + media: { status: "not-requested", reasons: ["profile-text-only"] }, + }, + } + : {}), + }, + }; +} diff --git a/knowledge-fs/packages/api/src/document-multimodal-asset-budget.test.ts b/knowledge-fs/packages/api/src/document-multimodal-asset-budget.test.ts new file mode 100644 index 00000000000..ede5f9cc3e4 --- /dev/null +++ b/knowledge-fs/packages/api/src/document-multimodal-asset-budget.test.ts @@ -0,0 +1,397 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createNodePlatformAdapter } from "@knowledge/adapters/node"; +import type { ParseArtifact } from "@knowledge/core"; +import { describe, expect, it, vi } from "vitest"; +import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor"; +import { createDocumentRemoteMediaBudget } from "./document-remote-media-budget"; + +const options = () => ({ + knowledgeSpaceId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42", + objectStorage: createNodePlatformAdapter({ env: {} }).objectStorage, + tenantId: "tenant-1", +}); +function artifact(uris: string[]): ParseArtifact { + return { + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43", + id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + elements: uris.map((uri, index) => ({ + id: `image-${index}`, + type: "image", + sectionPath: [], + metadata: { assetRef: { uri }, caption: `Source ${index}` }, + })), + metadata: {}, + parser: "native-markdown", + version: 1, + }; +} + +describe("document-wide media budgets", () => { + it("validates the remote budget's per-image cap independently of its caller", () => { + expect(() => + createDocumentRemoteMediaBudget({ + maxAttempts: 1, + maxBytes: 0, + maxTotalBytes: 1, + timeoutMs: 1, + }), + ).toThrow("maxRemoteAssetBytes"); + }); + it("charges derived image bytes to the same document materialization budget", async () => { + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA=="]), + maxTotalAssetBytes: 8, + imageVariantGenerator: { + generate: async () => [ + { name: "thumbnail", contentType: "image/png", body: new Uint8Array([1, 2, 3]) }, + { name: "analysis", contentType: "image/png", body: new Uint8Array([1, 2, 3]) }, + ], + }, + }); + expect(result.artifact.elements[0]?.metadata.assetRef).not.toHaveProperty("variants"); + expect(result.artifact.elements[0]?.metadata.assetRef).toHaveProperty("analysisUnavailable", { + reason: "materialized-byte-budget", + }); + expect(result.artifact.metadata).toMatchObject({ + multimodalAssets: { materializedBytes: 4 }, + parseCoverage: { media: { reasons: ["materialized-byte-budget"] } }, + }); + }); + it.each([ + new Error("decoder transport unavailable"), + Object.assign(new Error("retry decoder"), { code: "provider_input", retryable: true }), + Object.assign(new Error("service unavailable"), { + code: "provider_request_failed", + retryable: false, + }), + ])("propagates transient or unrelated variant failures", async (error) => { + await expect( + extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA=="]), + imageVariantGenerator: { + generate: async () => { + throw error; + }, + }, + }), + ).rejects.toBe(error); + }); + it("does not accept a variant returned after its deadline", async () => { + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA=="]), + maxVariantDurationMs: 5, + imageVariantGenerator: { + generate: async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + return []; + }, + }, + }); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { reasons: ["variant-deadline"] } }, + }); + }); + it("propagates lease loss during variant generation", async () => { + const controller = new AbortController(); + await expect( + extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA=="]), + signal: controller.signal, + imageVariantGenerator: { + generate: async () => { + controller.abort(new Error("lease lost during decode")); + throw controller.signal.reason; + }, + }, + }), + ).rejects.toThrow("lease lost during decode"); + }); + it("does not materialize a single image larger than the remaining byte budget", async () => { + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA=="]), + maxTotalAssetBytes: 3, + }); + expect(result.extractedCount).toBe(0); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { reasons: ["materialized-byte-budget"] } }, + }); + }); + it("ignores directory-shaped image paths and unavailable allowlisted roots", async () => { + const root = await mkdtemp(join(tmpdir(), "kfs-media-directory-")); + try { + const directory = join(root, "folder.png"); + await mkdir(directory); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact([directory]), + allowLocalAssetPaths: [root, join(root, "missing")], + }); + expect(result.extractedCount).toBe(0); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("cancels variant work on its document deadline and does not start more decoders", async () => { + const generate = vi.fn( + async ({ signal }: { signal?: AbortSignal | undefined }) => + new Promise((_resolve, reject) => { + if (signal?.aborted) reject(signal.reason); + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }), + ); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA==", "data:image/png;base64,AQIDBA=="]), + maxVariantDurationMs: 5, + imageVariantGenerator: { generate }, + }); + expect(generate).toHaveBeenCalledTimes(1); + expect(result.extractedCount).toBe(2); + expect(result.artifact.elements[0]?.metadata.assetRef).toHaveProperty("analysisUnavailable", { + reason: "variant-deadline", + }); + expect(result.artifact.elements[1]?.metadata.assetRef).toHaveProperty("analysisUnavailable", { + reason: "variant-deadline", + }); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { status: "partial", reasons: ["variant-deadline"] } }, + }); + }); + it("propagates cancellation of in-flight downloads rather than treating it as missing media", async () => { + const controller = new AbortController(); + const fetch = vi.fn(async () => { + controller.abort(new Error("lease lost during download")); + return null; + }); + await expect( + extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["https://a.test/1.png"]), + signal: controller.signal, + remoteAssetFetcher: { fetch }, + }), + ).rejects.toThrow("lease lost during download"); + }); + it.each([ + "maxRemoteAssetAttempts", + "maxTotalRemoteAssetBytes", + "remoteAssetTimeoutMs", + "maxTotalAssetBytes", + "maxVariantPixels", + "maxTotalVariantPixels", + "maxVariantDurationMs", + ])("rejects invalid %s", async (field) => { + await expect( + extractDocumentMultimodalAssets({ ...options(), artifact: artifact([]), [field]: 0 }), + ).rejects.toThrow("must be at least 1"); + }); + it("records unavailable remote capability without losing references or earlier coverage", async () => { + const source = artifact(["https://a.test/image.png"]); + source.metadata = { + parseCoverage: { + text: { status: "complete", reasons: [] }, + media: { status: "partial", reasons: ["unsupported-media"] }, + }, + }; + const result = await extractDocumentMultimodalAssets({ ...options(), artifact: source }); + expect(result.artifact.elements).toEqual(source.elements); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { + text: { status: "complete" }, + media: { status: "partial", reasons: ["unsupported-media", "remote-fetcher-unavailable"] }, + }, + }); + }); + it("preserves originals without decoding variants beyond the aggregate pixel budget", async () => { + const png = new Uint8Array([ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 2, 0, 0, 0, 3, + ]); + const generate = vi.fn(async () => [ + { body: new Uint8Array([1]), contentType: "image/png", name: "thumbnail" }, + ]); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact([ + `data:image/png;base64,${Buffer.from(png).toString("base64")}`, + `data:image/png;base64,${Buffer.from(png).toString("base64")}`, + ]), + maxTotalVariantPixels: 6, + imageVariantGenerator: { generate }, + }); + expect(generate).toHaveBeenCalledTimes(1); + expect(result.extractedCount).toBe(2); + expect(result.artifact.elements[0]?.metadata.assetRef).not.toHaveProperty( + "analysisUnavailable", + ); + expect(result.artifact.elements[1]?.metadata.assetRef).toHaveProperty("analysisUnavailable", { + reason: "variant-pixel-budget", + }); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { status: "partial", reasons: ["variant-pixel-budget"] } }, + }); + }); + it("leaves images inline before exceeding the document's materialized byte cap", async () => { + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA==", "data:image/png;base64,AQIDBA=="]), + maxTotalAssetBytes: 4, + }); + expect(result.extractedCount).toBe(1); + expect(result.artifact.elements[1]?.metadata.assetRef).toEqual({ + analysisUnavailable: { reason: "materialized-byte-budget" }, + uri: "data:image/png;base64,AQIDBA==", + }); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { status: "partial", reasons: ["materialized-byte-budget"] } }, + }); + }); + it.each(["empty", "invalid"])("preserves the source when variants are %s", async (failure) => { + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["data:image/png;base64,AQIDBA=="]), + imageVariantGenerator: { + generate: async () => { + if (failure === "invalid") + throw Object.assign(new Error("Image decoder rejected input"), { + code: "provider_input", + retryable: false, + }); + return []; + }, + }, + }); + expect(result.extractedCount).toBe(1); + expect(result.artifact.elements[0]?.metadata.assetRef).toMatchObject({ + analysisUnavailable: { + reason: failure === "empty" ? "variant-unavailable" : "variant-input-rejected", + }, + objectKey: expect.any(String), + }); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { + media: { + status: "partial", + reasons: [failure === "empty" ? "variant-unavailable" : "variant-input-rejected"], + }, + }, + }); + }); + it("counts failed unique URL attempts and preserves unresolved references", async () => { + const fetch = vi.fn(async () => null); + const source = artifact([ + "https://a.test/1.png", + "https://a.test/2.png", + "https://a.test/3.png", + ]); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: source, + maxRemoteAssetAttempts: 2, + remoteAssetFetcher: { fetch }, + }); + expect(fetch).toHaveBeenCalledTimes(2); + expect(result.artifact.elements).toEqual(source.elements); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { + media: { + status: "partial", + reasons: expect.arrayContaining(["remote-attempt-budget", "remote-unavailable"]), + }, + }, + multimodalAssets: { remoteAttempts: 2, remoteDownloadedBytes: 0 }, + }); + }); + + it("fetches repeated URLs once while preserving every element and caption", async () => { + const fetch = vi.fn(async () => ({ body: new Uint8Array([1, 2]), contentType: "image/png" })); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["https://a.test/1.png", "https://a.test/1.png"]), + remoteAssetFetcher: { fetch }, + }); + expect(fetch).toHaveBeenCalledTimes(1); + expect(result.extractedCount).toBe(2); + expect(result.artifact.elements.map((element) => element.metadata.caption)).toEqual([ + "Source 0", + "Source 1", + ]); + expect(result.artifact.metadata).toMatchObject({ + multimodalAssets: { remoteAttempts: 1, remoteDownloadedBytes: 2 }, + }); + }); + + it("bounds aggregate remote bytes before requesting the next image", async () => { + const fetch = vi.fn(async () => ({ body: new Uint8Array([1, 2]), contentType: "image/png" })); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["https://a.test/1.png", "https://a.test/2.png"]), + maxTotalRemoteAssetBytes: 2, + remoteAssetFetcher: { fetch }, + }); + expect(fetch).toHaveBeenCalledTimes(1); + expect(result.extractedCount).toBe(1); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { status: "partial", reasons: ["remote-byte-budget"] } }, + }); + }); + + it("aborts a stalled fetch on the document-wide deadline and reports partial media", async () => { + const fetch = vi.fn(async () => new Promise(() => {})); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["https://a.test/1.png", "https://a.test/2.png"]), + remoteAssetTimeoutMs: 10, + remoteAssetFetcher: { fetch }, + }); + expect(fetch).toHaveBeenCalledTimes(1); + expect(result.artifact.metadata).toMatchObject({ + parseCoverage: { media: { status: "partial", reasons: ["remote-deadline"] } }, + }); + }); + + it("does not start fetching after cancellation", async () => { + const fetch = vi.fn(async () => null); + const controller = new AbortController(); + controller.abort(new Error("lease lost")); + await expect( + extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact(["https://a.test/1.png"]), + signal: controller.signal, + remoteAssetFetcher: { fetch }, + }), + ).rejects.toThrow("lease lost"); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("denies symlinks escaping an allowed local root", async () => { + const allowed = await mkdtemp(join(tmpdir(), "kfs-media-allowed-")); + const outside = await mkdtemp(join(tmpdir(), "kfs-media-outside-")); + try { + const privatePath = join(outside, "secret.png"); + await writeFile(privatePath, new Uint8Array([1, 2])); + const link = join(allowed, "image.png"); + await symlink(privatePath, link); + const result = await extractDocumentMultimodalAssets({ + ...options(), + artifact: artifact([link]), + allowLocalAssetPaths: [allowed], + }); + expect(result.extractedCount).toBe(0); + expect(result.artifact.elements[0]?.metadata.assetRef).toEqual({ uri: link }); + } finally { + await rm(allowed, { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts index 0219db21a2c..c435d7b936b 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts @@ -58,8 +58,16 @@ describe("extractDocumentMultimodalAssets", () => { expect(result.extractedCount).toBe(2); expect(fetchCalls).toEqual([ - { maxBytes: 1024, url: "https://cdn.example.test/markdown.png" }, - { maxBytes: 1024, url: "https://cdn.example.test/office.png" }, + { + maxBytes: 1024, + signal: expect.any(AbortSignal), + url: "https://cdn.example.test/markdown.png", + }, + { + maxBytes: 1024, + signal: expect.any(AbortSignal), + url: "https://cdn.example.test/office.png", + }, ]); for (const element of result.artifact.elements) { expect(element.metadata).toMatchObject({ @@ -127,7 +135,7 @@ describe("extractDocumentMultimodalAssets", () => { expect(remoteFetches).toEqual([ { maxBytes: 10 * 1024 * 1024, - signal: controller.signal, + signal: expect.any(AbortSignal), url: "https://cdn.example.test/missing.png", }, ]); @@ -227,7 +235,10 @@ describe("extractDocumentMultimodalAssets", () => { expect(result).toMatchObject({ extractedCount: 1, skippedForCapCount: 1 }); expect(fetchCalls).toEqual([]); expect(result.artifact.elements[1]?.metadata).toEqual({ - assetRef: { uri: "https://cdn.example.test/second.png" }, + assetRef: { + uri: "https://cdn.example.test/second.png", + analysisUnavailable: { reason: "asset-count-budget" }, + }, }); expect(result.artifact.elements[2]?.metadata).toEqual({ assetRef: { uri: "https://user:secret@cdn.example.test/private.png" }, @@ -546,7 +557,10 @@ describe("extractDocumentMultimodalAssets", () => { }); // The over-cap image is left inline (its data URI preserved), not extracted. const figure2 = result.artifact.elements.find((element) => element.id === "figure-2"); - expect(figure2?.metadata.assetRef).toEqual({ uri: "data:image/png;base64,BQYHCA==" }); + expect(figure2?.metadata.assetRef).toEqual({ + uri: "data:image/png;base64,BQYHCA==", + analysisUnavailable: { reason: "asset-count-budget" }, + }); // Exactly one object was written (the extracted figure-1), so no orphan from figure-2. const figure1 = result.artifact.elements.find((element) => element.id === "figure-1"); const objectKey = (figure1?.metadata.assetRef as { objectKey?: string } | undefined)?.objectKey; diff --git a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts index 4a503839fae..a336bbf2b65 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts @@ -1,14 +1,22 @@ import { createHash } from "node:crypto"; -import { readFile, stat } from "node:fs/promises"; +import { constants } from "node:fs"; +import { open, realpath } from "node:fs/promises"; import { extname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { type ParseArtifact, ParseArtifactSchema, type PlatformAdapter } from "@knowledge/core"; +import { + type DocumentMultimodalAssetRef, + type ParseArtifact, + ParseArtifactSchema, + type ParseElement, + type PlatformAdapter, +} from "@knowledge/core"; import type { DocumentImageVariantGenerator, GeneratedDocumentImageVariant, } from "./document-image-variant-generator"; +import { createDocumentRemoteMediaBudget } from "./document-remote-media-budget"; import { cloneJsonObject, isPlainObject } from "./json-utils"; import { createDocumentMultimodalAssetObjectKey, @@ -23,6 +31,13 @@ export interface ExtractDocumentMultimodalAssetsInput { readonly maxExtractedAssets?: number | undefined; readonly maxLocalAssetBytes?: number | undefined; readonly maxRemoteAssetBytes?: number | undefined; + readonly maxRemoteAssetAttempts?: number | undefined; + readonly maxTotalRemoteAssetBytes?: number | undefined; + readonly remoteAssetTimeoutMs?: number | undefined; + readonly maxTotalAssetBytes?: number | undefined; + readonly maxVariantPixels?: number | undefined; + readonly maxTotalVariantPixels?: number | undefined; + readonly maxVariantDurationMs?: number | undefined; readonly imageVariantGenerator?: DocumentImageVariantGenerator | undefined; readonly objectStorage: PlatformAdapter["objectStorage"]; readonly remoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined; @@ -58,6 +73,10 @@ interface ImageDimensions { readonly width: number; } +type AnalysisUnavailableReason = NonNullable< + DocumentMultimodalAssetRef["analysisUnavailable"] +>["reason"]; + const dataUriPattern = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)$/iu; const defaultMaxEmbeddedAssetBytes = 10 * 1024 * 1024; const defaultMaxExtractedAssets = 1_000; @@ -78,6 +97,13 @@ export async function extractDocumentMultimodalAssets({ maxExtractedAssets = defaultMaxExtractedAssets, maxLocalAssetBytes = defaultMaxLocalAssetBytes, maxRemoteAssetBytes = defaultMaxRemoteAssetBytes, + maxRemoteAssetAttempts = 100, + maxTotalRemoteAssetBytes = 32 * 1024 * 1024, + remoteAssetTimeoutMs = 60_000, + maxTotalAssetBytes = 64 * 1024 * 1024, + maxVariantPixels = 20_000_000, + maxTotalVariantPixels = 100_000_000, + maxVariantDurationMs = 60_000, imageVariantGenerator, objectStorage, remoteAssetFetcher, @@ -100,14 +126,38 @@ export async function extractDocumentMultimodalAssets({ if (!Number.isSafeInteger(maxRemoteAssetBytes) || maxRemoteAssetBytes < 1) { throw new Error("Document multimodal remote asset max bytes must be at least 1"); } + for (const [name, value] of Object.entries({ + maxTotalAssetBytes, + maxVariantPixels, + maxTotalVariantPixels, + maxVariantDurationMs, + })) { + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be at least 1`); + } let extractedCount = 0; let skippedForCapCount = 0; + let materializedBytes = 0; + let variantPixels = 0; + const variantDeadline = performance.now() + maxVariantDurationMs; + const mediaReasons = new Set(); + const remoteBudget = createDocumentRemoteMediaBudget({ + fetcher: remoteAssetFetcher, + maxAttempts: maxRemoteAssetAttempts, + maxBytes: maxRemoteAssetBytes, + maxTotalBytes: maxTotalRemoteAssetBytes, + signal, + timeoutMs: remoteAssetTimeoutMs, + }); const extractionSources = new Set(); const elements = []; const allowedLocalRoots = normalizeAllowedLocalAssetPaths(allowLocalAssetPaths); + const canonicalAllowedLocalRoots = ( + await Promise.all(allowedLocalRoots.map((root) => realpath(root).catch(() => null))) + ).filter((root): root is string => root !== null); for (const element of artifact.elements) { + signal?.throwIfAborted(); if (element.type !== "image" && element.type !== "table") { elements.push(element); continue; @@ -115,32 +165,34 @@ export async function extractDocumentMultimodalAssets({ const assetRef = isPlainObject(element.metadata.assetRef) ? element.metadata.assetRef : null; const uri = typeof assetRef?.uri === "string" ? assetRef.uri.trim() : ""; + if ( + extractedCount >= maxExtractedAssets && + (dataUriPattern.test(uri) || + isRemoteHttpUri(uri) || + (allowedLocalRoots.length > 0 && localPathFromUri(uri))) + ) { + skippedForCapCount += 1; + elements.push(withAnalysisUnavailable(element, "asset-count-budget")); + continue; + } + if (materializedBytes >= maxTotalAssetBytes && uri) { + mediaReasons.add("materialized-byte-budget"); + elements.push(withAnalysisUnavailable(element, "materialized-byte-budget")); + continue; + } let image = parseDataUriImage(uri, maxEmbeddedAssetBytes) ?? (await readLocalImageAsset({ allowedRoots: allowedLocalRoots, + canonicalAllowedRoots: canonicalAllowedLocalRoots, assetRef, maxLocalAssetBytes, uri, })); - if (!image && remoteAssetFetcher && isRemoteHttpUri(uri)) { - if (extractedCount >= maxExtractedAssets) { - skippedForCapCount += 1; - elements.push(element); - continue; - } - const fetched = await remoteAssetFetcher.fetch({ - maxBytes: maxRemoteAssetBytes, - ...(signal ? { signal } : {}), - url: uri, - }); + if (!image && isRemoteHttpUri(uri)) { + const fetched = await remoteBudget.fetch(uri); if (fetched) { - if (fetched.body.byteLength > maxRemoteAssetBytes) { - throw new Error( - `Document multimodal remote asset exceeds maxRemoteAssetBytes=${maxRemoteAssetBytes}`, - ); - } const contentType = normalizeRemoteImageContentType(fetched.contentType); if (!contentType) { throw new Error("Document multimodal remote asset content type is unsupported"); @@ -163,13 +215,12 @@ export async function extractDocumentMultimodalAssets({ continue; } - if (extractedCount >= maxExtractedAssets) { - // Soft cap: leave the remaining extractable images inline instead of throwing (which would - // abort ingestion and orphan the assets already written to object storage this run). - skippedForCapCount += 1; - elements.push(element); + if (image.body.byteLength > maxTotalAssetBytes - materializedBytes) { + mediaReasons.add("materialized-byte-budget"); + elements.push(withAnalysisUnavailable(element, "materialized-byte-budget")); continue; } + materializedBytes += image.body.byteLength; const sha256 = sha256Hex(image.body); const objectKey = createDocumentMultimodalAssetObjectKey({ @@ -195,18 +246,47 @@ export async function extractDocumentMultimodalAssets({ ...(writeOwnerId ? { writeOwnerId } : {}), }, }); - const variants = imageVariantGenerator - ? await storeGeneratedImageVariants({ - assetId: artifact.documentAssetId, - elementId: element.id, - generator: imageVariantGenerator, - image, - knowledgeSpaceId, - objectStorage, - tenantId, - ...(writeOwnerId ? { writeOwnerId } : {}), - }) - : {}; + signal?.throwIfAborted(); + let analysisUnavailableReason: AnalysisUnavailableReason | undefined; + const markAnalysisUnavailable = (reason: AnalysisUnavailableReason) => { + analysisUnavailableReason = reason; + mediaReasons.add(reason); + }; + const imagePixels = image.dimensions + ? image.dimensions.width * image.dimensions.height + : maxVariantPixels; + const canGenerateVariants = + imageVariantGenerator && + imagePixels > 0 && + Number.isSafeInteger(imagePixels) && + imagePixels <= maxVariantPixels && + imagePixels <= maxTotalVariantPixels - variantPixels; + if (imageVariantGenerator && !canGenerateVariants) + markAnalysisUnavailable("variant-pixel-budget"); + if (canGenerateVariants) variantPixels += imagePixels; + const variantTimeRemaining = Math.ceil(variantDeadline - performance.now()); + if (canGenerateVariants && variantTimeRemaining <= 0) + markAnalysisUnavailable("variant-deadline"); + const variants = + canGenerateVariants && variantTimeRemaining > 0 + ? await storeGeneratedImageVariants({ + assetId: artifact.documentAssetId, + elementId: element.id, + generator: imageVariantGenerator, + image, + knowledgeSpaceId, + objectStorage, + onUnavailable: markAnalysisUnavailable, + onMaterializedBytes: (bytes) => { + materializedBytes += bytes; + }, + remainingBytes: maxTotalAssetBytes - materializedBytes, + timeoutMs: variantTimeRemaining, + tenantId, + ...(signal ? { signal } : {}), + ...(writeOwnerId ? { writeOwnerId } : {}), + }) + : {}; extractedCount += 1; extractionSources.add(image.source); @@ -220,6 +300,9 @@ export async function extractDocumentMultimodalAssets({ ...cloneJsonObject(element.metadata), assetRef: { ...remainingAssetRef, + ...(analysisUnavailableReason + ? { analysisUnavailable: { reason: analysisUnavailableReason } } + : {}), contentType: image.contentType, ...(image.dimensions ? image.dimensions : {}), objectKey, @@ -241,7 +324,12 @@ export async function extractDocumentMultimodalAssets({ }); } - if (extractedCount === 0) { + if ( + extractedCount === 0 && + remoteBudget.reasons.size === 0 && + mediaReasons.size === 0 && + skippedForCapCount === 0 + ) { return { artifact, extractedCount, skippedForCapCount }; } @@ -251,8 +339,17 @@ export async function extractDocumentMultimodalAssets({ elements, metadata: { ...artifact.metadata, + ...mediaCoverageMetadata(artifact, [ + ...remoteBudget.reasons, + ...mediaReasons, + ...(skippedForCapCount > 0 ? ["asset-count-budget"] : []), + ]), multimodalAssets: { extractedCount, + remoteAttempts: remoteBudget.attempts, + remoteDownloadedBytes: remoteBudget.downloadedBytes, + materializedBytes, + variantPixels, ...(skippedForCapCount > 0 ? { skippedForCapCount } : {}), sources: [...extractionSources].sort(), }, @@ -263,6 +360,43 @@ export async function extractDocumentMultimodalAssets({ }; } +function withAnalysisUnavailable( + element: ParseElement, + reason: AnalysisUnavailableReason, +): ParseElement { + return { + ...element, + metadata: { + ...element.metadata, + assetRef: { + ...(isPlainObject(element.metadata.assetRef) ? element.metadata.assetRef : {}), + analysisUnavailable: { reason }, + }, + }, + }; +} + +function mediaCoverageMetadata(artifact: ParseArtifact, reasons: readonly string[]) { + if (reasons.length === 0) return {}; + const previous = isPlainObject(artifact.metadata.parseCoverage) + ? artifact.metadata.parseCoverage + : {}; + const media = isPlainObject(previous.media) ? previous.media : {}; + const existingReasons = Array.isArray(media.reasons) + ? media.reasons.filter((item): item is string => typeof item === "string").slice(0, 32) + : []; + return { + parseCoverage: { + ...previous, + media: { + ...media, + status: "partial", + reasons: [...new Set([...existingReasons, ...reasons])], + }, + }, + }; +} + function isRemoteHttpUri(uri: string): boolean { try { const parsed = new URL(uri); @@ -288,7 +422,12 @@ async function storeGeneratedImageVariants({ image, knowledgeSpaceId, objectStorage, + onUnavailable, + onMaterializedBytes, + remainingBytes, + timeoutMs, tenantId, + signal, writeOwnerId, }: { readonly assetId: string; @@ -297,15 +436,61 @@ async function storeGeneratedImageVariants({ readonly image: DataUriImage; readonly knowledgeSpaceId: string; readonly objectStorage: PlatformAdapter["objectStorage"]; + readonly onUnavailable: (reason: AnalysisUnavailableReason) => void; + readonly onMaterializedBytes: (bytes: number) => void; + readonly remainingBytes: number; + readonly timeoutMs: number; readonly tenantId: string; + readonly signal?: AbortSignal | undefined; readonly writeOwnerId?: string | undefined; }): Promise>> { const variants: Record> = {}; - const generated = await generator.generate({ - body: image.body, - contentType: image.contentType, - elementId, - }); + const controller = new AbortController(); + const cancel = () => controller.abort(signal?.reason); + signal?.addEventListener("abort", cancel, { once: true }); + const timer = setTimeout(() => controller.abort(new Error("variant-deadline")), timeoutMs); + let generated: readonly GeneratedDocumentImageVariant[]; + try { + signal?.throwIfAborted(); + generated = await generator.generate({ + body: image.body, + contentType: image.contentType, + elementId, + signal: controller.signal, + }); + signal?.throwIfAborted(); + if (controller.signal.aborted) { + onUnavailable("variant-deadline"); + return {}; + } + } catch (error) { + signal?.throwIfAborted(); + if (controller.signal.aborted) { + onUnavailable("variant-deadline"); + return {}; + } + if ( + error instanceof Error && + "code" in error && + (error.code === "provider_input" || error.code === "provider_timeout") && + "retryable" in error && + error.retryable === false + ) { + onUnavailable("variant-input-rejected"); + return {}; + } + throw error; + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", cancel); + } + if (generated.length === 0) onUnavailable("variant-unavailable"); + const generatedBytes = generated.reduce((sum, variant) => sum + variant.body.byteLength, 0); + if (!Number.isSafeInteger(generatedBytes) || generatedBytes > remainingBytes) { + onUnavailable("materialized-byte-budget"); + return {}; + } + onMaterializedBytes(generatedBytes); for (const variant of generated) { const stored = await storeGeneratedImageVariant({ @@ -369,6 +554,7 @@ async function storeGeneratedImageVariant({ return { contentType: variant.contentType, + ...(variant.execution ? { execution: { ...variant.execution } } : {}), ...(variant.height !== undefined ? { height: variant.height } : {}), objectKey, sha256, @@ -407,11 +593,13 @@ function parseDataUriImage(uri: string, maxEmbeddedAssetBytes: number): DataUriI async function readLocalImageAsset({ allowedRoots, + canonicalAllowedRoots, assetRef, maxLocalAssetBytes, uri, }: { readonly allowedRoots: readonly string[]; + readonly canonicalAllowedRoots: readonly string[]; readonly assetRef: Readonly> | null; readonly maxLocalAssetBytes: number; readonly uri: string; @@ -432,27 +620,49 @@ async function readLocalImageAsset({ return null; } - const metadata = await stat(localPath); + // Resolve both roots and the target before opening; O_NOFOLLOW also rejects replacement of + // the final component with a symlink between realpath and open. + const canonicalPath = await realpath(localPath); + if (!pathIsWithinAllowedRoots(canonicalPath, canonicalAllowedRoots)) return null; + const handle = await open(canonicalPath, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); - if (!metadata.isFile()) { - return null; + if (!metadata.isFile()) { + return null; + } + + if (metadata.size > maxLocalAssetBytes) { + throw new Error( + `Document multimodal local asset exceeds maxLocalAssetBytes=${maxLocalAssetBytes}`, + ); + } + + const buffer = Buffer.alloc(Math.min(metadata.size, maxLocalAssetBytes) + 1); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const read = await handle.read(buffer, bytesRead, buffer.length - bytesRead, null); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + } + if (bytesRead > maxLocalAssetBytes) + throw new Error( + `Document multimodal local asset exceeds maxLocalAssetBytes=${maxLocalAssetBytes}`, + ); + if (bytesRead !== metadata.size) + throw new Error("Document multimodal local asset changed during read"); + const body = new Uint8Array(buffer.subarray(0, bytesRead)); + const dimensions = readImageDimensions(body, contentType); + + return { + body, + contentType, + ...(dimensions ? { dimensions } : {}), + source: "local-file", + }; + } finally { + await handle.close(); } - - if (metadata.size > maxLocalAssetBytes) { - throw new Error( - `Document multimodal local asset exceeds maxLocalAssetBytes=${maxLocalAssetBytes}`, - ); - } - - const body = new Uint8Array(await readFile(localPath)); - const dimensions = readImageDimensions(body, contentType); - - return { - body, - contentType, - ...(dimensions ? { dimensions } : {}), - source: "local-file", - }; } function readImageDimensions(body: Uint8Array, contentType: string): ImageDimensions | undefined { diff --git a/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.test.ts b/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.test.ts index 7f09f79ede7..90c349a75ad 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.test.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.test.ts @@ -38,11 +38,18 @@ describe("document multimodal candidate resolver", () => { id: "figure-1", metadata: { assetRef: { + analysisUnavailable: { reason: "variant-pixel-budget" }, contentType: "image/png", objectKey: "tenant-dev/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/018f0d60-7a49-7cc2-9c1b-5b36f18f2c43/assets/figure-1.png", sha256: "c".repeat(64), uri: "data:image/png;base64,AAAA", + variants: { + analysis: { + contentType: "image/png", + objectKey: "tenant-dev/assets/analysis.png", + }, + }, }, boundingBox: { height: 120, width: 240, x: 10, y: 20 }, caption: "Revenue bridge", @@ -95,10 +102,14 @@ describe("document multimodal candidate resolver", () => { assetDescriptorPath: "/knowledge/docs/Quarterly-Report.pdf--018f0d60/assets/image-Revenue-bridge--018f0d60.json", assetRef: { + analysisUnavailable: { reason: "variant-pixel-budget" }, contentType: "image/png", objectKey: "tenant-dev/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/018f0d60-7a49-7cc2-9c1b-5b36f18f2c43/assets/figure-1.png", sha256: "c".repeat(64), + variants: { + analysis: { contentType: "image/png", objectKey: "tenant-dev/assets/analysis.png" }, + }, }, assetRoute: "/knowledge-spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/018f0d60-7a49-7cc2-9c1b-5b36f18f2c43/multimodal/018f0d60-7a49-7cc2-9c1b-5b36f18f2c44%3A0%3Afigure-1/asset", diff --git a/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.ts b/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.ts index 0d2f9b006bb..f34fefd9870 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.ts @@ -134,6 +134,10 @@ function metadataInteger( function cloneAssetRef(assetRef: DocumentMultimodalAssetRef): DocumentMultimodalAssetRef { return { + ...(assetRef.analysisUnavailable + ? { analysisUnavailable: { ...assetRef.analysisUnavailable } } + : {}), + ...(assetRef.variants ? { variants: structuredClone(assetRef.variants) } : {}), ...(assetRef.contentType ? { contentType: assetRef.contentType } : {}), ...(assetRef.objectKey ? { objectKey: assetRef.objectKey } : {}), ...(assetRef.sha256 ? { sha256: assetRef.sha256 } : {}), diff --git a/knowledge-fs/packages/api/src/document-multimodal-enrichment-providers.ts b/knowledge-fs/packages/api/src/document-multimodal-enrichment-providers.ts index c4b57c435d0..060cc8718fc 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-enrichment-providers.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-enrichment-providers.ts @@ -76,6 +76,8 @@ export function createUnderstandingDocumentMultimodalEnrichmentProvider({ return { enrich: async (input) => { + // Metadata/OCR already present remains usable, but never forward a rejected original to a VLM. + if (input.item.assetRef?.analysisUnavailable !== undefined) return {}; const task = understandingTaskForItem(input.item); if (!task) { diff --git a/knowledge-fs/packages/api/src/document-multimodal-manifest-builder.ts b/knowledge-fs/packages/api/src/document-multimodal-manifest-builder.ts index 2214af1cf62..de390cd6de1 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-manifest-builder.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-manifest-builder.ts @@ -127,7 +127,8 @@ function documentMultimodalItemFromElement({ const caption = metadataString(element.metadata, "caption"); const ocrText = metadataString(element.metadata, "ocrText"); const title = metadataString(element.metadata, "title") ?? caption; - const assetRef = parseAssetRef(element.metadata); + const sourceMetadata = boundedMultimodalSourceMetadata(element.metadata); + const assetRef = parseAssetRef(sourceMetadata); const boundingBox = parseBoundingBox(element.metadata.boundingBox); const textPreview = textPreviewForElement(element, ocrText, maxTextPreviewChars); const positionUnknown = element.metadata.positionUnknown === true; @@ -148,13 +149,35 @@ function documentMultimodalItemFromElement({ ...(element.pageNumber ? { pageNumber: element.pageNumber } : {}), parseElementId: element.id, sectionPath: [...element.sectionPath], - sourceMetadata: cloneJsonObject(element.metadata), + sourceMetadata, ...(startOffset !== undefined ? { startOffset } : {}), ...(textPreview ? { textPreview } : {}), ...(title ? { title } : {}), }; } +function boundedMultimodalSourceMetadata( + metadata: Readonly>, +): Record { + const sourceMetadata = cloneJsonObject(metadata); + const ref = isPlainObject(sourceMetadata.assetRef) ? sourceMetadata.assetRef : undefined; + if ( + ref?.analysisUnavailable !== undefined && + typeof ref.uri === "string" && + ref.uri.length > 2048 && + ref.uri.trimStart().slice(0, 5).toLowerCase() === "data:" + ) { + // The original parse element remains the recovery source. Manifests are descriptors, not a + // second copy of rejected image bytes; parseArtifactId + parseElementId preserve provenance. + const { uri, ...boundedRef } = ref; + sourceMetadata.assetRef = { + ...boundedRef, + sourceUriSha256: createHash("sha256").update(uri).digest("hex"), + }; + } + return sourceMetadata; +} + function multimodalModality(element: ParseElement): DocumentMultimodalItem["modality"] | null { switch (element.type) { case "code": @@ -182,17 +205,20 @@ function enrichmentForElement({ readonly ocrText: string | undefined; }): DocumentMultimodalItem["enrichment"] { return { - asset: assetRef - ? "provided" - : element.type === "image" || element.type === "table" - ? "missing" - : "unsupported", + asset: + assetRef && (assetRef.objectKey || assetRef.uri || assetRef.sha256) + ? "provided" + : element.type === "image" || element.type === "table" + ? "missing" + : "unsupported", caption: caption ? "provided" : element.type === "image" ? "missing" : "unsupported", ocr: ocrText || element.text ? "provided" : element.type === "image" ? "missing" : "unsupported", tableStructure: element.type === "table" ? tableStructureStatus(element) : "unsupported", visualEmbedding: - assetRef && isVisualEmbeddingEligibleElement(element) ? "missing" : "unsupported", + assetRef && !assetRef.analysisUnavailable && isVisualEmbeddingEligibleElement(element) + ? "missing" + : "unsupported", }; } @@ -246,11 +272,18 @@ function parseAssetRef( const contentType = metadataString(candidate, "contentType") ?? metadataString(candidate, "mimeType"); - if (!objectKey && !uri && !sha256) { + if (!objectKey && !uri && !sha256 && candidate.analysisUnavailable === undefined) { return undefined; } return { + ...(candidate.analysisUnavailable !== undefined + ? { + analysisUnavailable: DocumentMultimodalAssetRefSchema.shape.analysisUnavailable.parse( + candidate.analysisUnavailable, + ), + } + : {}), ...(contentType ? { contentType } : {}), ...(objectKey ? { objectKey } : {}), ...(sha256 ? { sha256 } : {}), diff --git a/knowledge-fs/packages/api/src/document-multimodal-manifest-enhancer.ts b/knowledge-fs/packages/api/src/document-multimodal-manifest-enhancer.ts index c783cd74e50..c11ead7b01a 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-manifest-enhancer.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-manifest-enhancer.ts @@ -357,7 +357,10 @@ function mergeMultimodalItemEnrichment( ...(caption ? { caption } : {}), enrichment: { ...item.enrichment, - asset: assetRef ? "provided" : item.enrichment.asset, + asset: + assetRef && (assetRef.objectKey || assetRef.uri || assetRef.sha256) + ? "provided" + : item.enrichment.asset, caption: caption ? "provided" : item.enrichment.caption, ocr: ocrText ? "provided" : item.enrichment.ocr, // Never downgrade a "provided" status to a weaker provider result. @@ -365,10 +368,9 @@ function mergeMultimodalItemEnrichment( item.enrichment.tableStructure, result.tableStructureStatus, ), - visualEmbedding: preferProvidedStatus( - item.enrichment.visualEmbedding, - result.visualEmbeddingStatus, - ), + visualEmbedding: assetRef?.analysisUnavailable + ? "unsupported" + : preferProvidedStatus(item.enrichment.visualEmbedding, result.visualEmbeddingStatus), }, ...(ocrText ? { ocrText } : {}), sourceMetadata: { @@ -390,6 +392,8 @@ function pickBetterAssetRef( current: DocumentMultimodalAssetRef | undefined, incoming: DocumentMultimodalAssetRef | undefined, ): DocumentMultimodalAssetRef | undefined { + // Enrichment does not validate image resource bounds and cannot clear a decoder rejection. + if (current?.analysisUnavailable) return current; if (!incoming) { return current; } diff --git a/knowledge-fs/packages/api/src/document-pdf-rasterizer-geometry.test.ts b/knowledge-fs/packages/api/src/document-pdf-rasterizer-geometry.test.ts new file mode 100644 index 00000000000..ae01a69bf97 --- /dev/null +++ b/knowledge-fs/packages/api/src/document-pdf-rasterizer-geometry.test.ts @@ -0,0 +1,184 @@ +import sharp from "sharp"; +import { describe, expect, it } from "vitest"; + +import { createPopplerPdfRasterizer } from "./document-pdf-rasterizer"; + +function geometryPdf({ + cropBox, + mediaBox, + rotation = 0, +}: { + readonly cropBox: readonly [number, number, number, number]; + readonly mediaBox: readonly [number, number, number, number]; + readonly rotation?: number; +}): Uint8Array { + const [x, y, right, top] = mediaBox; + const halfWidth = (right - x) / 2; + const halfHeight = (top - y) / 2; + const content = [ + `1 0 0 rg ${x} ${y + halfHeight} ${halfWidth} ${halfHeight} re f`, + `0 1 0 rg ${x + halfWidth} ${y + halfHeight} ${halfWidth} ${halfHeight} re f`, + `0 0 1 rg ${x} ${y} ${halfWidth} ${halfHeight} re f`, + `1 1 0 rg ${x + halfWidth} ${y} ${halfWidth} ${halfHeight} re f`, + ].join("\n"); + const objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + `<< /Type /Page /Parent 2 0 R /MediaBox [${mediaBox.join(" ")}] /CropBox [${cropBox.join(" ")}] /Rotate ${rotation} /Resources << >> /Contents 4 0 R >>`, + `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`, + ]; + let pdf = "%PDF-1.7\n"; + const offsets: number[] = []; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.byteLength(pdf)); + pdf += `${index + 1} 0 obj\n${object}\nendobj\n`; + } + const xref = Buffer.byteLength(pdf); + pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + pdf += offsets.map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`).join(""); + pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return new TextEncoder().encode(pdf); +} + +describe("real Poppler MediaBox geometry admission", () => { + it("caps the rendered MediaBox before allocation even when the CropBox is smaller", async () => { + const rasterizer = createPopplerPdfRasterizer({ + dpi: 72, + maxPageDimension: 1_000, + maxPagePixels: 10_000, + thumbnailDpi: 72, + }); + + const image = await rasterizer.render({ + documentBody: geometryPdf({ cropBox: [36, 36, 108, 108], mediaBox: [0, 0, 144, 144] }), + elementId: "small-crop-box", + pageNumber: 1, + }); + + expect(image).not.toBeNull(); + await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 100, width: 100 }); + }); + + it("accounts for rounded pdfinfo box coordinates before a boundary-sized allocation", async () => { + const rasterizer = createPopplerPdfRasterizer({ + dpi: 72, + maxPageDimension: 1_000, + maxPagePixels: 10_000, + thumbnailDpi: 72, + }); + + const image = await rasterizer.render({ + documentBody: geometryPdf({ + cropBox: [0, 0, 100.004, 100], + mediaBox: [0, 0, 100.004, 100], + }), + elementId: "rounded-media-box", + pageNumber: 1, + }); + + await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 100, width: 100 }); + }); + + it("bounds the shorter scaled edge when rounding conceals a larger aspect ratio", async () => { + const rasterizer = createPopplerPdfRasterizer({ + dpi: 72, + maxPageDimension: 1_000, + maxPagePixels: 20_000, + thumbnailDpi: 72, + }); + + const image = await rasterizer.render({ + documentBody: geometryPdf({ + cropBox: [0, 0, 100.004, 199.996], + mediaBox: [0, 0, 100.004, 199.996], + }), + elementId: "rounded-aspect-ratio", + pageNumber: 1, + }); + + await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 199, width: 100 }); + }); + + it("keeps the original rendering of an ordinary uncapped page", async () => { + const image = await createPopplerPdfRasterizer({ dpi: 72, thumbnailDpi: 72 }).render({ + documentBody: geometryPdf({ cropBox: [0, 0, 144, 72], mediaBox: [0, 0, 144, 72] }), + elementId: "ordinary-page", + pageNumber: 1, + }); + + await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 72, width: 144 }); + }); + + it.each([ + { color: [255, 0, 0], rotation: 0 }, + { color: [0, 0, 255], rotation: 90 }, + { color: [255, 255, 0], rotation: 180 }, + { color: [0, 255, 0], rotation: 270 }, + ])( + "preserves displayed crop coordinates with nonzero origin and rotation=$rotation", + async ({ color, rotation }) => { + const rasterizer = createPopplerPdfRasterizer({ + dpi: 72, + maxPageDimension: 100, + maxPagePixels: 20_000, + thumbnailDpi: 72, + }); + const rotated = rotation === 90 || rotation === 270; + const pageWidth = rotated ? 72 : 144; + const pageHeight = rotated ? 144 : 72; + + const images = await rasterizer.renderBatch?.({ + documentBody: geometryPdf({ + cropBox: [72, 90, 144, 126], + mediaBox: [36, 72, 180, 144], + rotation, + }), + requests: [ + { elementId: "page", pageNumber: 1 }, + { + boundingBox: { height: 0.2, width: 0.2, x: 0.1, y: 0.1 }, + boundingBoxGeometry: { coordinateSystem: "relative" }, + elementId: "relative-crop", + pageNumber: 1, + }, + { + boundingBox: { + height: pageHeight * 0.2, + width: pageWidth * 0.2, + x: pageWidth * 0.1, + y: pageHeight * 0.1, + }, + boundingBoxGeometry: { coordinateSystem: "pixel", pageHeight, pageWidth }, + elementId: "pixel-crop", + pageNumber: 1, + }, + { + boundingBox: { + height: pageHeight * 0.2, + width: pageWidth * 0.2, + x: pageWidth * 0.1, + y: pageHeight * 0.1, + }, + boundingBoxGeometry: { coordinateSystem: "pdf-point", pageHeight, pageWidth }, + elementId: "displayed-point-crop", + pageNumber: 1, + }, + ], + }); + + await expect(sharp(images?.[0]?.body).metadata()).resolves.toMatchObject({ + height: rotated ? 100 : 50, + width: rotated ? 50 : 100, + }); + for (const image of images?.slice(1) ?? []) { + const { data, info } = await sharp(image?.body) + .removeAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + expect(info).toMatchObject({ height: rotated ? 20 : 10, width: rotated ? 10 : 20 }); + expect([...data.subarray(0, 3)]).toEqual(color); + expect([...data.subarray(data.length - 3)]).toEqual(color); + } + }, + ); +}); diff --git a/knowledge-fs/packages/api/src/document-pdf-rasterizer.test.ts b/knowledge-fs/packages/api/src/document-pdf-rasterizer.test.ts index fb453cf6c0e..be534209145 100644 --- a/knowledge-fs/packages/api/src/document-pdf-rasterizer.test.ts +++ b/knowledge-fs/packages/api/src/document-pdf-rasterizer.test.ts @@ -32,6 +32,11 @@ interface FakePopplerCommand { async function createFakePopplerCommand( mode: "failure" | "success" | "timeout" = "success", + options: { + readonly pdfInfoOutput?: string; + readonly pageSizePoints?: { readonly height: number; readonly width: number }; + readonly renderedSize?: { readonly height: number; readonly width: number }; + } = {}, ): Promise { const root = await mkdtemp(join(tmpdir(), "knowledge-fs-fake-poppler-")); const command = join(root, "pdftoppm.cjs"); @@ -43,8 +48,8 @@ async function createFakePopplerCommand( create: { background: { alpha: 1, b: 255, g: 127, r: 63 }, channels: 4, - height: 80, - width: 100, + height: options.renderedSize?.height ?? 80, + width: options.renderedSize?.width ?? 100, }, }) .png() @@ -57,7 +62,12 @@ if (args.includes("-box")) { fs.appendFileSync(${JSON.stringify(pdfInfoInvocationLog)}, JSON.stringify(args) + "\\n"); fs.appendFileSync(${JSON.stringify(workDirLog)}, path.dirname(args.at(-1)) + "\\n"); const pageNumber = args[args.indexOf("-f") + 1]; - process.stdout.write("Page " + pageNumber + " size: 612 x 792 pts\\n"); + if (${options.pdfInfoOutput !== undefined}) { + process.stdout.write(${JSON.stringify(options.pdfInfoOutput ?? "")}); + process.exit(0); + } + process.stdout.write("Page " + pageNumber + " size: ${options.pageSizePoints?.width ?? 612} x ${options.pageSizePoints?.height ?? 792} pts\\n"); + process.stdout.write("Page " + pageNumber + " MediaBox: 0 0 ${options.pageSizePoints?.width ?? 612} ${options.pageSizePoints?.height ?? 792}\\n"); process.exit(0); } const inputPath = args.at(-2); @@ -1198,6 +1208,38 @@ describe("createPopplerPdfRasterizer batch rendering", () => { } }); + it.each([ + "Page 1 size: 72 x 72 pts\n", + "Page 2 MediaBox: 0 0 72 72\n", + "Page 1 MediaBox: 0 0 72\n", + "Page 1 MediaBox: 0 0 Infinity 72\n", + "Page 1 MediaBox: 0 0 NaN 72\n", + "Page 1 MediaBox: 72 0 0 72\n", + "Page 1 MediaBox: 0 72 72 0\n", + "Page 1 MediaBox: 0 0 1e200 1e200\n", + "Page 1 MediaBox: 0 0 72 72\nPage 1 MediaBox: 0 0 144 144\n", + ])( + "rejects unsafe or ambiguous MediaBox metadata before rendering (%j)", + async (pdfInfoOutput) => { + const fakePoppler = await createFakePopplerCommand("success", { pdfInfoOutput }); + + try { + const rasterizer = createPopplerPdfRasterizer({ + command: fakePoppler.command, + pdfInfoCommand: fakePoppler.pdfInfoCommand, + }); + + await expect( + rasterizer.render({ documentBody, elementId: "unsafe-page", pageNumber: 1 }), + ).rejects.toThrow("could not determine MediaBox size"); + expect(await fakePoppler.readInvocations()).toHaveLength(0); + await expectTemporaryDirectoriesRemoved(await fakePoppler.readWorkDirs()); + } finally { + await fakePoppler.cleanup(); + } + }, + ); + it("caches page sizes across batches in one document session and removes its work directory", async () => { const fakePoppler = await createFakePopplerCommand(); @@ -1570,6 +1612,104 @@ describe("createPopplerPdfRasterizer batch rendering", () => { } }); + it.each([ + { + height: 5_102.36, + maxPageDimension: 4_096, + maxPagePixels: 1_000_000, + scaleTo: 1_499, + width: 2_267.72, + }, + { height: 792, maxPageDimension: 4_096, maxPagePixels: 100_000, scaleTo: 359, width: 612 }, + { height: 100, maxPageDimension: 4_096, maxPagePixels: 10_000, scaleTo: 100, width: 100.1 }, + { height: 100_000, maxPageDimension: 20_000, maxPagePixels: 10_000, scaleTo: 10_000, width: 1 }, + ])( + "caps Poppler allocation before rendering a $width x $height point page to $maxPagePixels pixels", + async ({ height, maxPageDimension, maxPagePixels, scaleTo, width }) => { + const fakePoppler = await createFakePopplerCommand("success", { + pageSizePoints: { height, width }, + }); + + try { + const rasterizer = createPopplerPdfRasterizer({ + command: fakePoppler.command, + maxPageDimension, + maxPagePixels, + pdfInfoCommand: fakePoppler.pdfInfoCommand, + thumbnailDpi: 144, + }); + + await rasterizer.render({ documentBody, elementId: "page-1", pageNumber: 1 }); + const [args] = await fakePoppler.readInvocations(); + expect(args).toContain("-scale-to"); + expect(Number(args?.[args.indexOf("-scale-to") + 1])).toBe(scaleTo); + const shortEdge = Math.max( + 1, + Math.ceil((scaleTo * Math.min(width, height)) / Math.max(width, height)), + ); + expect(scaleTo * shortEdge).toBeLessThanOrEqual(maxPagePixels); + expect(scaleTo).toBeLessThanOrEqual(maxPageDimension); + } finally { + await fakePoppler.cleanup(); + } + }, + ); + + it("preserves PDF, pixel, and relative crop positions after pixel-budget scaling", async () => { + const fakePoppler = await createFakePopplerCommand("success", { + pageSizePoints: { height: 800, width: 600 }, + renderedSize: { height: 240, width: 180 }, + }); + + try { + const rasterizer = createPopplerPdfRasterizer({ + command: fakePoppler.command, + maxPagePixels: 43_200, + pdfInfoCommand: fakePoppler.pdfInfoCommand, + thumbnailDpi: 144, + }); + const images = await rasterizer.renderBatch?.({ + documentBody, + requests: [ + { + boundingBox: { height: 400, width: 300, x: 60, y: 160 }, + boundingBoxGeometry: { coordinateSystem: "pdf-point", pageHeight: 800, pageWidth: 600 }, + elementId: "pdf-points", + pageNumber: 1, + }, + { + boundingBox: { height: 800, width: 600, x: 120, y: 320 }, + boundingBoxGeometry: { coordinateSystem: "pixel", pageHeight: 1_600, pageWidth: 1_200 }, + elementId: "pixels", + pageNumber: 1, + }, + { + boundingBox: { height: 0.5, width: 0.5, x: 0.1, y: 0.2 }, + boundingBoxGeometry: { coordinateSystem: "relative" }, + elementId: "relative", + pageNumber: 1, + }, + ], + }); + + const [args] = await fakePoppler.readInvocations(); + expect(args?.[args.indexOf("-scale-to") + 1]).toBe("239"); + expect(images).toHaveLength(3); + const sharp = (await import("sharp")).default; + for (const image of images ?? []) { + expect(image?.metadata).toMatchObject({ + crop: { normalizedBoundingBox: { height: 120, width: 90, x: 18, y: 48 } }, + }); + await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ + height: 120, + width: 90, + }); + } + } finally { + await fakePoppler.cleanup(); + } + }); + it("rejects encoded Poppler pages before reading oversized output files", async () => { const fakePoppler = await createFakePopplerCommand(); diff --git a/knowledge-fs/packages/api/src/document-pdf-rasterizer.ts b/knowledge-fs/packages/api/src/document-pdf-rasterizer.ts index 26a16995c36..fe827c2e37c 100644 --- a/knowledge-fs/packages/api/src/document-pdf-rasterizer.ts +++ b/knowledge-fs/packages/api/src/document-pdf-rasterizer.ts @@ -1346,7 +1346,7 @@ async function renderPopplerPage({ }): Promise { const outputName = `page-${pageNumber}-dpi-${dpi}`; const outputPrefix = join(workDir, outputName); - const scaleTo = popplerScaleToForPage({ dpi, maxPageDimension, pageSize }); + const scaleTo = popplerScaleToForPage({ dpi, maxPageDimension, maxPagePixels, pageSize }); await renderPopplerPng({ command, dpi, @@ -1456,15 +1456,39 @@ async function readPopplerPdfPageSize({ const { stdout } = await execFileAsync( command, ["-f", String(pageNumber), "-l", String(pageNumber), "-box", inputPath], - { signal, timeout: timeoutMs, windowsHide: true }, + { + env: { ...process.env, LANG: "C", LC_ALL: "C" }, + signal, + timeout: timeoutMs, + windowsHide: true, + }, ); - const match = /Page(?:\s+\d+)?\s+size:\s*([\d.]+)\s+x\s+([\d.]+)\s+pts/iu.exec(String(stdout)); - const widthPoints = Number(match?.[1]); - const heightPoints = Number(match?.[2]); + // pdftoppm deliberately retains its default MediaBox rendering so provider coordinates keep + // their established displayed-page frame. pdfinfo's "Page size" reports the CropBox instead: + // budgeting from it can admit a small visible crop while allocating a much larger MediaBox. + // Subtract endpoints rather than assuming a zero origin. Rotation only swaps the axes, which + // cannot change the area, longest edge or aspect ratio used by the pre-render pixel budget. + const boxes = [ + ...String(stdout).matchAll( + new RegExp(`^Page(?:\\s+${pageNumber})?\\s+MediaBox:\\s*(.*?)\\s*$`, "gmu"), + ), + ]; + const coordinates = boxes[0]?.[1]?.trim().split(/\s+/u).map(Number) ?? []; + const [left = Number.NaN, bottom = Number.NaN, right = Number.NaN, top = Number.NaN] = + coordinates; + const widthPoints = right - left; + const heightPoints = top - bottom; - if (!(widthPoints > 0) || !(heightPoints > 0)) { + if ( + boxes.length !== 1 || + coordinates.length !== 4 || + !coordinates.every(Number.isFinite) || + !(widthPoints > 0) || + !(heightPoints > 0) || + !Number.isFinite(widthPoints * heightPoints) + ) { throw new DocumentPdfRenderError( - `Poppler PDF rasterizer could not determine page size for pageNumber=${pageNumber}`, + `Poppler PDF rasterizer could not determine MediaBox size for pageNumber=${pageNumber}`, ); } @@ -1474,15 +1498,50 @@ async function readPopplerPdfPageSize({ function popplerScaleToForPage({ dpi, maxPageDimension, + maxPagePixels, pageSize, }: { readonly dpi: number; readonly maxPageDimension: number; + readonly maxPagePixels: number; readonly pageSize: PopplerPdfPageSize; }): number | undefined { - const naturalMaxDimension = (Math.max(pageSize.widthPoints, pageSize.heightPoints) * dpi) / 72; + // pdfinfo prints box endpoints to two decimal places. Each difference can hide up to 0.01 pt. + // Use upper dimensions for allocation and the worst-case aspect ratio for scale-to; otherwise + // a rounded 100.00 x 200.00 box can render a 101 x 200 bitmap just past the pixel ceiling. + const roundingAllowance = 0.01; + const naturalWidth = ((pageSize.widthPoints + roundingAllowance) * dpi) / 72; + const naturalHeight = ((pageSize.heightPoints + roundingAllowance) * dpi) / 72; - return naturalMaxDimension > maxPageDimension ? maxPageDimension : undefined; + if ( + Math.max(naturalWidth, naturalHeight) <= maxPageDimension && + Math.ceil(naturalWidth) * Math.ceil(naturalHeight) <= maxPagePixels + ) { + return undefined; + } + + const shortEdgeUpper = Math.min(pageSize.widthPoints, pageSize.heightPoints) + roundingAllowance; + const longEdgeLower = Math.max(pageSize.widthPoints, pageSize.heightPoints) - roundingAllowance; + const aspectRatio = longEdgeLower > 0 ? Math.min(1, shortEdgeUpper / longEdgeLower) : 1; + let lower = 1; + let upper = Math.floor( + Math.min(maxPageDimension, maxPagePixels, Math.sqrt(maxPagePixels / aspectRatio)), + ); + + // Poppler rounds the shorter edge up. A square-root area cap alone can therefore exceed + // the pixel budget; find the largest integer edge whose rounded bitmap is still bounded. + while (lower < upper) { + const candidate = lower + Math.ceil((upper - lower) / 2); + const shortEdge = Math.max(1, Math.ceil(candidate * aspectRatio)); + + if (candidate * shortEdge <= maxPagePixels) { + lower = candidate; + } else { + upper = candidate - 1; + } + } + + return lower; } async function renderPopplerPng({ diff --git a/knowledge-fs/packages/api/src/document-remote-media-budget.ts b/knowledge-fs/packages/api/src/document-remote-media-budget.ts new file mode 100644 index 00000000000..4b04f3d977a --- /dev/null +++ b/knowledge-fs/packages/api/src/document-remote-media-budget.ts @@ -0,0 +1,108 @@ +import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor"; + +type RemoteImage = Awaited>; + +/** One document, one bounded cache and one absolute deadline, including unsuccessful URLs. */ +export function createDocumentRemoteMediaBudget(input: { + readonly fetcher?: DocumentRemoteAssetFetcher | undefined; + readonly maxAttempts: number; + readonly maxBytes: number; + readonly maxTotalBytes: number; + readonly signal?: AbortSignal | undefined; + readonly timeoutMs: number; +}) { + for (const [name, value] of Object.entries({ + maxRemoteAssetAttempts: input.maxAttempts, + maxRemoteAssetBytes: input.maxBytes, + maxTotalRemoteAssetBytes: input.maxTotalBytes, + remoteAssetTimeoutMs: input.timeoutMs, + })) { + if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be at least 1`); + } + const deadline = performance.now() + input.timeoutMs; + const cache = new Map(); + const reasons = new Set(); + let attempts = 0; + let downloadedBytes = 0; + let reservedBytes = 0; + let deadlineReached = false; + return { + reasons, + get attempts() { + return attempts; + }, + get downloadedBytes() { + return downloadedBytes; + }, + async fetch(url: string): Promise { + input.signal?.throwIfAborted(); + if (cache.has(url)) return cache.get(url) ?? null; + if (!input.fetcher) { + reasons.add("remote-fetcher-unavailable"); + return null; + } + if (deadlineReached || performance.now() >= deadline) { + reasons.add("remote-deadline"); + return null; + } + if (attempts >= input.maxAttempts) { + reasons.add("remote-attempt-budget"); + return null; + } + if (reservedBytes >= input.maxTotalBytes) { + reasons.add("remote-byte-budget"); + return null; + } + attempts += 1; + const maxBytes = Math.min(input.maxBytes, input.maxTotalBytes - reservedBytes); + // A failed fetch can have consumed its entire body limit without returning a body. Keep + // that reservation charged so failures cannot bypass the aggregate network budget. + reservedBytes += maxBytes; + const controller = new AbortController(); + const abortFromParent = () => controller.abort(input.signal?.reason); + input.signal?.addEventListener("abort", abortFromParent, { once: true }); + const timer = setTimeout( + () => controller.abort(new Error("remote-deadline")), + Math.max(1, deadline - performance.now()), + ); + let rejectAbort: (() => void) | undefined; + try { + const aborted = new Promise((_resolve, reject) => { + rejectAbort = () => reject(controller.signal.reason); + controller.signal.addEventListener("abort", rejectAbort, { once: true }); + if (controller.signal.aborted) rejectAbort(); + }); + const image = await Promise.race([ + input.fetcher.fetch({ maxBytes, signal: controller.signal, url }), + aborted, + ]); + input.signal?.throwIfAborted(); + if (image) { + if (image.body.byteLength > maxBytes) + throw new RemoteMediaContractError( + `Document multimodal remote asset exceeds maxRemoteAssetBytes=${maxBytes}`, + ); + downloadedBytes += image.body.byteLength; + reservedBytes -= maxBytes - image.body.byteLength; + } else { + reasons.add("remote-unavailable"); + } + cache.set(url, image); + return image; + } catch (error) { + input.signal?.throwIfAborted(); + if (error instanceof RemoteMediaContractError || !controller.signal.aborted) throw error; + deadlineReached = true; + reasons.add("remote-deadline"); + cache.set(url, null); + return null; + } finally { + clearTimeout(timer); + input.signal?.removeEventListener("abort", abortFromParent); + if (rejectAbort) controller.signal.removeEventListener("abort", rejectAbort); + } + }, + }; +} + +class RemoteMediaContractError extends Error {} diff --git a/knowledge-fs/packages/api/src/document-upload-utils.test.ts b/knowledge-fs/packages/api/src/document-upload-utils.test.ts index 50b7de7ac98..36e49cd9078 100644 --- a/knowledge-fs/packages/api/src/document-upload-utils.test.ts +++ b/knowledge-fs/packages/api/src/document-upload-utils.test.ts @@ -20,6 +20,21 @@ import { } from "./document-upload-utils"; describe("document upload utilities", () => { + it.each(["constructor", "__proto__"])( + "rejects inherited-key extension %s as unsupported instead of crashing", + async (extension) => { + await expect( + readDocumentUpload( + { + parseBody: async () => ({ + file: new File(["x"], `x.${extension}`, { type: "text/plain" }), + }), + }, + 10, + ), + ).rejects.toBeInstanceOf(DocumentUploadValidationError); + }, + ); it("reads a single multipart file with bounded bytes and optional source id", async () => { const file = new File(["hello"], "note.md", { type: "text/markdown" }); diff --git a/knowledge-fs/packages/api/src/document-upload-utils.ts b/knowledge-fs/packages/api/src/document-upload-utils.ts index 2171d5b749e..08d82700c86 100644 --- a/knowledge-fs/packages/api/src/document-upload-utils.ts +++ b/knowledge-fs/packages/api/src/document-upload-utils.ts @@ -1,3 +1,8 @@ +import { + DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION, + documentMimeTypesForFilename, +} from "@knowledge/parsers"; + export interface ParsedDocumentUpload { readonly body: Uint8Array; readonly documentId?: string; @@ -20,39 +25,6 @@ export interface BulkDocumentRevisionTarget { readonly expectedDocumentRowVersion: number; } -const DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION = { - csv: ["text/csv", "application/vnd.ms-excel"], - doc: ["application/msword"], - docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"], - eml: ["message/rfc822"], - epub: ["application/epub+zip"], - htm: ["text/html"], - html: ["text/html"], - json: ["application/json"], - jsonl: ["application/x-ndjson", "application/jsonl", "application/ndjson", "application/json"], - markdown: ["text/markdown", "text/x-markdown", "text/plain"], - md: ["text/markdown", "text/x-markdown", "text/plain"], - mdx: ["text/mdx", "text/markdown", "text/plain"], - msg: ["application/vnd.ms-outlook", "application/x-msg"], - odt: ["application/vnd.oasis.opendocument.text"], - pdf: ["application/pdf"], - ppt: ["application/vnd.ms-powerpoint", "application/mspowerpoint", "application/x-mspowerpoint"], - pptx: ["application/vnd.openxmlformats-officedocument.presentationml.presentation"], - properties: ["text/x-java-properties", "text/plain"], - rtf: ["application/rtf", "text/rtf", "application/x-rtf"], - text: ["text/plain"], - txt: ["text/plain"], - vtt: ["text/vtt", "text/plain"], - xls: [ - "application/vnd.ms-excel", - "application/excel", - "application/x-excel", - "application/x-msexcel", - ], - xlsx: ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"], - xml: ["application/xml", "text/xml"], -} as const satisfies Readonly>; - export const SUPPORTED_DOCUMENT_UPLOAD_MIME_TYPES = new Set( Object.values(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION).flat(), ); @@ -539,32 +511,17 @@ function isJsonObject(value: unknown): value is Record { export function normalizeDocumentMimeType(file: File): string { const declared = file.type.trim().toLocaleLowerCase(); - const extension = documentExtension(file.name); - const inferred = extension - ? (DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION as Readonly>)[ - extension - ]?.[0] - : undefined; + const inferred = documentMimeTypesForFilename(file.name)?.[0]; return !declared || declared === "application/octet-stream" ? (inferred ?? "application/octet-stream") : declared; } function isSupportedDocumentUpload(file: File, mimeType: string): boolean { - const extension = documentExtension(file.name); - if (extension === undefined) return false; - const allowedMimeTypes = ( - DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION as Readonly> - )[extension]; + const allowedMimeTypes = documentMimeTypesForFilename(file.name); return allowedMimeTypes?.includes(mimeType) === true; } -function documentExtension(filename: string): string | undefined { - const normalized = filename.trim().toLocaleLowerCase(); - const dot = normalized.lastIndexOf("."); - return dot >= 0 && dot < normalized.length - 1 ? normalized.slice(dot + 1) : undefined; -} - export async function sha256Hex(bytes: Uint8Array): Promise { const buffer = bytes.buffer.slice( bytes.byteOffset, diff --git a/knowledge-fs/packages/api/src/document-write-handlers.ts b/knowledge-fs/packages/api/src/document-write-handlers.ts index 721ac9b3f9c..cecc4a9a042 100644 --- a/knowledge-fs/packages/api/src/document-write-handlers.ts +++ b/knowledge-fs/packages/api/src/document-write-handlers.ts @@ -39,6 +39,7 @@ import type { DocumentCompilationJobStateMachine } from "./document-compilation- import { compileDocumentArtifact } from "./document-compilation-pipeline"; import type { DocumentImageVariantGenerator } from "./document-image-variant-generator"; import { buildDocumentKnowledgePath } from "./document-knowledge-paths"; +import type { DocumentMediaCapabilityResolver } from "./document-media-execution-plan"; import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor"; import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository"; import type { DocumentOutlineBuilder } from "./document-outline-builder"; @@ -194,6 +195,7 @@ export interface RegisterDocumentWriteHandlersOptions { readonly synchronousUploadDenseModel?: string | undefined; readonly traces: TraceRecorder; readonly visualEmbeddingModel?: string | undefined; + readonly resolveDocumentMediaCapabilities?: DocumentMediaCapabilityResolver | undefined; } interface BulkUploadAcceptedItem { @@ -269,6 +271,7 @@ export function registerDocumentWriteHandlers({ synchronousUploadDenseModel, traces, visualEmbeddingModel, + resolveDocumentMediaCapabilities, }: RegisterDocumentWriteHandlersOptions): void { const effectiveBufferedDocumentUploadAdmission = bufferedDocumentUploadAdmission ?? @@ -1480,6 +1483,7 @@ export function registerDocumentWriteHandlers({ body: upload.body, knowledgeSpaceId, permissionScope: [], + signal: context.req.raw.signal, tenantId: subject.tenantId, traceId, }, @@ -1510,6 +1514,7 @@ export function registerDocumentWriteHandlers({ synchronousUploadReindexer, traces, visualEmbeddingModel, + resolveDocumentMediaCapabilities, }, ); await assertWritable(); diff --git a/knowledge-fs/packages/api/src/index-projection-builders.test.ts b/knowledge-fs/packages/api/src/index-projection-builders.test.ts index db9bcb885b3..ff0224d2f24 100644 --- a/knowledge-fs/packages/api/src/index-projection-builders.test.ts +++ b/knowledge-fs/packages/api/src/index-projection-builders.test.ts @@ -74,6 +74,45 @@ function createRecordingProjectionRepository() { } describe("index projection builders", () => { + it("excludes rejected image nodes before invoking any visual provider", async () => { + const { repository } = createRecordingProjectionRepository(); + const calls: EmbedVisualAssetsInput[] = []; + const builder = createVisualEmbeddingProjectionBuilder({ + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f9006", + maxBatchSize: 2, + projections: repository, + provider: { + embedAssets: async (input) => { + calls.push(input); + return { + dense: [[0.2, 0.8]], + metadata: { provider: "test", model: "vision" }, + model: "vision", + }; + }, + }, + }); + await expect( + builder.build({ + model: "vision", + projectionVersion: 1, + nodes: [ + knowledgeNode({ + kind: "image", + metadata: { + assetRef: { + contentType: "image/png", + objectKey: "tenant/assets/image.png", + analysisUnavailable: { reason: "variant-pixel-budget" }, + }, + elementTypes: ["image"], + }, + }), + ], + }), + ).resolves.toEqual([]); + expect(calls).toEqual([]); + }); it("propagates cancellation to embedding calls and never persists an aborted batch", async () => { const controller = new AbortController(); const { created, repository } = createRecordingProjectionRepository(); diff --git a/knowledge-fs/packages/api/src/index-projection-builders.ts b/knowledge-fs/packages/api/src/index-projection-builders.ts index 6b3337a0cbb..024589cf0b7 100644 --- a/knowledge-fs/packages/api/src/index-projection-builders.ts +++ b/knowledge-fs/packages/api/src/index-projection-builders.ts @@ -1047,6 +1047,7 @@ export function createObjectStorageVisualEmbeddingProvider({ for (const asset of assets) { signal?.throwIfAborted(); + if (asset.assetRef.analysisUnavailable !== undefined) continue; // We cannot know the next object's exact byte length without an extra HEAD request. Flush // whenever the remaining raw-byte budget cannot admit the configured per-asset maximum; // the following bounded GET therefore cannot create a currentBatchBytes + maxAssetBytes @@ -1313,7 +1314,7 @@ function visualEmbeddingAssetCandidateFromNode( const modality = metadataString(multimodal ?? {}, "modality") ?? multimodalProjectionModality(node); - if (!assetRef || !modality) { + if (!assetRef || assetRef.analysisUnavailable !== undefined || !modality) { return null; } diff --git a/knowledge-fs/packages/api/src/index.ts b/knowledge-fs/packages/api/src/index.ts index cc55d6a7262..0e3b839d567 100644 --- a/knowledge-fs/packages/api/src/index.ts +++ b/knowledge-fs/packages/api/src/index.ts @@ -456,6 +456,7 @@ import { createInMemoryDocumentAssetRepository, } from "./document-asset-repository"; import { registerDocumentCompilationHandlers } from "./document-compilation-handlers"; +import { createProfileDocumentMediaCapabilityResolver } from "./document-media-execution-plan"; import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder"; import { createCachedDocumentMultimodalManifestEnhancer } from "./document-multimodal-manifest-enhancer"; import { createInMemoryDocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository"; @@ -2036,6 +2037,13 @@ export function createKnowledgeGateway({ }) : undefined; + const resolveDocumentMediaCapabilities = + knowledgeSpaceProfiles && modelInputModalityResolver + ? createProfileDocumentMediaCapabilityResolver({ + profiles: knowledgeSpaceProfiles, + modalities: modelInputModalityResolver, + }) + : undefined; registerDocumentWriteHandlers({ access: accessService, adapter, @@ -2069,6 +2077,7 @@ export function createKnowledgeGateway({ : {}), documentMultimodalManifests: multimodalManifestRepository, documentParser, + resolveDocumentMediaCapabilities, ...(documentPdfRasterizer ? { documentPdfRasterizer } : {}), effectiveMaxBulkUploadBytes, generateArtifactSegmentId, @@ -2109,6 +2118,7 @@ export function createKnowledgeGateway({ }); const sourceDocumentMaterializer = createSourceDocumentMaterializer({ + resolveDocumentMediaCapabilities, artifacts, artifactSegments: segments, assets, diff --git a/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.test.ts b/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.test.ts index 3d7571124c5..c170a04ab77 100644 --- a/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.test.ts +++ b/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.test.ts @@ -272,6 +272,11 @@ describe("createObjectStorageContentBlockMultimodalAnswerProvider", () => { await adapter.objectStorage.putObject({ body: new Uint8Array([1, 2, 3]), contentType: "image/png", + key: "tenant/spaces/space/documents/doc/assets/chart.png", + }); + await adapter.objectStorage.putObject({ + body: new Uint8Array([9]), + contentType: "image/png", key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png", }); const calls: GenerateMultimodalAnswerContentInput[] = []; diff --git a/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.ts b/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.ts index 4a37d087b16..7697438ad1f 100644 --- a/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.ts +++ b/knowledge-fs/packages/api/src/llm-multimodal-answer-provider.ts @@ -155,7 +155,7 @@ export function createObjectStorageContentBlockMultimodalAnswerProvider({ maxImageBytes = 10 * 1024 * 1024, maxTotalImageBytes = 32 * 1024 * 1024, objectStorage, - preferredVariant = "thumbnail", + preferredVariant = "analysis", ...options }: ObjectStorageContentBlockMultimodalAnswerProviderOptions): MultimodalAnswerProvider { if (!Number.isSafeInteger(maxImageBytes) || maxImageBytes < 1) { @@ -385,7 +385,10 @@ function objectBackedImageAssetRef({ readonly attachment: MultimodalAnswerProviderInput["multimodalEvidence"][number]; readonly preferredVariant: string; }): { readonly contentType: string; readonly objectKey: string } | undefined { - if (!isPlainObject(attachment.assetRef)) { + if ( + !isPlainObject(attachment.assetRef) || + attachment.assetRef.analysisUnavailable !== undefined + ) { return undefined; } @@ -472,7 +475,10 @@ function multimodalContentBlockMessages({ for (const [index, attachment] of input.multimodalEvidence.entries()) { const label = `M${index + 1}`; - const url = isVisualAttachment(attachment) ? assetUrlResolver(attachment) : undefined; + const url = + isVisualAttachment(attachment) && attachment.assetRef?.analysisUnavailable === undefined + ? assetUrlResolver(attachment) + : undefined; attachmentTextBlocks.push({ text: multimodalAttachmentLine(label, attachment, url), diff --git a/knowledge-fs/packages/api/src/source-handlers-coverage.test.ts b/knowledge-fs/packages/api/src/source-handlers-coverage.test.ts index 519f4ef91ec..cd8b8676eb1 100644 --- a/knowledge-fs/packages/api/src/source-handlers-coverage.test.ts +++ b/knowledge-fs/packages/api/src/source-handlers-coverage.test.ts @@ -1141,7 +1141,7 @@ describe("online document import edge branches", () => { }); expect(response.status).toBe(200); const body = await response.json(); - expect(body.documents).toHaveLength(1); + expect(body.documents, JSON.stringify(body)).toHaveLength(1); expect(body.failed).toEqual([ { code: SOURCE_OPERATION_FAILURES.onlineDocumentPageFetch.code, @@ -1376,7 +1376,7 @@ describe("online drive import edge branches", () => { ); expect(response.status).toBe(200); const body = await response.json(); - expect(body.documents).toEqual([ + expect(body.documents, JSON.stringify(body)).toEqual([ { documentAssetId: expect.any(String), filename: "plain.txt" }, ]); expect(body.failed).toEqual([ diff --git a/knowledge-fs/packages/api/src/source-handlers.ts b/knowledge-fs/packages/api/src/source-handlers.ts index f95f17d19b7..97dbee5bf3a 100644 --- a/knowledge-fs/packages/api/src/source-handlers.ts +++ b/knowledge-fs/packages/api/src/source-handlers.ts @@ -1,5 +1,6 @@ import { isDeepStrictEqual } from "node:util"; import type { OpenAPIHono } from "@hono/zod-openapi"; +import { documentMimeTypesForFilename } from "@knowledge/parsers"; import { CANDIDATE_VISIBILITY_SCAN_BUDGET_EXCEEDED_MESSAGE, @@ -1436,25 +1437,8 @@ function slugPart(value: string): string { return value.replace(/[^a-zA-Z0-9._-]+/gu, "-").replace(/^-+|-+$/gu, ""); } -const MIME_TYPES_BY_EXTENSION: Readonly> = { - csv: "text/csv", - docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - htm: "text/html", - html: "text/html", - json: "application/json", - markdown: "text/markdown", - md: "text/markdown", - pdf: "application/pdf", - pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation", - txt: "text/plain", - xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - xml: "application/xml", -}; - export function mimeTypeForFilename(filename: string): string { - const extension = /\.([a-zA-Z0-9]+)$/u.exec(filename)?.[1]?.toLowerCase(); - - return (extension && MIME_TYPES_BY_EXTENSION[extension]) || "application/octet-stream"; + return documentMimeTypesForFilename(filename)?.[0] ?? "application/octet-stream"; } export interface ImportedPageState { diff --git a/knowledge-fs/packages/api/src/source-mime-registry.test.ts b/knowledge-fs/packages/api/src/source-mime-registry.test.ts new file mode 100644 index 00000000000..1f24f9c97a5 --- /dev/null +++ b/knowledge-fs/packages/api/src/source-mime-registry.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { mimeTypeForFilename } from "./source-handlers"; + +describe("source file MIME inference shares the upload registry", () => { + it.each([ + ["source.JSONL", "application/x-ndjson"], + [" mail.eml ", "message/rfc822"], + ["legacy.doc", "application/msword"], + ["deck.ppt", "application/vnd.ms-powerpoint"], + ["caption.vtt", "text/vtt"], + ["settings.properties", "text/x-java-properties"], + ["data.unknown", "application/octet-stream"], + ["README", "application/octet-stream"], + ])("infers %s without widening upload types", (filename, expected) => { + expect(mimeTypeForFilename(filename)).toBe(expected); + }); +}); diff --git a/knowledge-fs/packages/core/src/models.ts b/knowledge-fs/packages/core/src/models.ts index dca0dfff470..ca07aeba42a 100644 --- a/knowledge-fs/packages/core/src/models.ts +++ b/knowledge-fs/packages/core/src/models.ts @@ -929,6 +929,21 @@ export const DocumentMultimodalAssetVariantSchema = z.object({ export type DocumentMultimodalAssetVariant = z.infer; export const DocumentMultimodalAssetRefSchema = z.object({ + // A retained download/preview reference is not necessarily safe to decode for model analysis. + // Absent on legacy artifacts; only explicit rejection disables image analysis. + analysisUnavailable: z + .object({ + reason: z.enum([ + "asset-count-budget", + "materialized-byte-budget", + "variant-pixel-budget", + "variant-deadline", + "variant-input-rejected", + "variant-unavailable", + ]), + }) + .strict() + .optional(), contentType: z.string().min(1).optional(), objectKey: ObjectStorageKeySchema.optional(), sha256: Sha256Schema.optional(), diff --git a/knowledge-fs/packages/parsers/src/archive-media-completeness.test.ts b/knowledge-fs/packages/parsers/src/archive-media-completeness.test.ts new file mode 100644 index 00000000000..0262228fbc7 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/archive-media-completeness.test.ts @@ -0,0 +1,72 @@ +import { zipSync } from "fflate"; +import { describe, expect, it } from "vitest"; +import { createUnstructuredParserClient } from "./index"; + +const parser = () => + createUnstructuredParserClient({ + endpoint: "https://parser.test", + fetch: async () => + Response.json([{ type: "NarrativeText", text: "Readable body", metadata: {} }]), + }); +const input = (entries: Record, requiresImages?: boolean) => ({ + body: zipSync(entries), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "x.docx", + mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + parserHints: requiresImages === undefined ? {} : { requiresImages }, + version: 1, +}); + +describe("archive media completeness", () => { + it("does not coalesce or reuse legacy-auto image output for explicitly disabled media", async () => { + const remote = parser(); + const entries = { "word/media/photo.png": new Uint8Array([1, 2, 3]) }; + const automatic = input(entries); + const disabled = { ...automatic, parserHints: { requiresImages: false } }; + expect(remote.policyFingerprint?.(automatic)).not.toBe(remote.policyFingerprint?.(disabled)); + const [withImages, withoutImages] = await Promise.all([ + remote.parse(automatic), + remote.parse(disabled), + ]); + expect(withImages.elements.some((element) => element.type === "image")).toBe(true); + expect(withoutImages.elements.some((element) => element.type === "image")).toBe(false); + expect(withImages.artifactHash).not.toBe(withoutImages.artifactHash); + }); + it("reports unsupported media by source path without silently claiming complete", async () => { + const artifact = await parser().parse( + input({ "word/media/vector.svg": new TextEncoder().encode("") }, true), + ); + expect(artifact.elements.map((element) => element.text)).toEqual(["Readable body"]); + expect(artifact.metadata.parseCoverage).toMatchObject({ + media: { status: "partial", reasons: ["unsupported-media-format"] }, + }); + expect(artifact.metadata.archiveMediaReport).toMatchObject({ + skippedResources: [ + { archivePath: "word/media/vector.svg", reason: "unsupported-media-format" }, + ], + }); + }); + it("honors disabled image extraction without materializing archive images", async () => { + const artifact = await parser().parse( + input({ "word/media/photo.png": new Uint8Array([1, 2, 3]) }, false), + ); + expect(artifact.elements.some((element) => element.type === "image")).toBe(false); + expect(artifact.metadata.parseCoverage).toMatchObject({ media: { status: "not-requested" } }); + }); + it("records Office charts and drawing definitions whose visuals cannot be extracted", async () => { + const artifact = await parser().parse( + input({ "word/charts/chart1.xml": new TextEncoder().encode("") }), + ); + expect(artifact.metadata.parseCoverage).toMatchObject({ + media: { status: "partial", reasons: ["office-visual-structure-not-rendered"] }, + }); + }); + it("reports oversized image omission and retains its archive reference", async () => { + const artifact = await parser().parse( + input({ "word/media/photo.png": new Uint8Array(10 * 1024 * 1024 + 1) }), + ); + expect(artifact.metadata.archiveMediaReport).toMatchObject({ + skippedResources: [{ archivePath: "word/media/photo.png", reason: "media-byte-budget" }], + }); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/document-format-registry.test.ts b/knowledge-fs/packages/parsers/src/document-format-registry.test.ts new file mode 100644 index 00000000000..bbe6f18082b --- /dev/null +++ b/knowledge-fs/packages/parsers/src/document-format-registry.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { + DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION, + documentMimeTypesForFilename, + resolveDocumentFormat, +} from "./document-format-registry"; +import { + createNativeHtmlParser, + createNativeMarkdownParser, + createParserRouter, + createUnstructuredParserClient, +} from "./index"; + +describe("shared document format registry", () => { + it("only returns MIME aliases for actual registered filename extensions", () => { + expect(documentMimeTypesForFilename("file.docx")).toEqual([ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ]); + for (const filename of [ + "file.constructor", + "file.__proto__", + "file.unknown", + "file.", + "README", + ]) + expect(documentMimeTypesForFilename(filename)).toBeUndefined(); + }); + it("preserves the existing upload allowlist exactly", () => { + expect(Object.keys(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION).sort()).toEqual( + "csv doc docx eml epub htm html json jsonl markdown md mdx msg odt pdf ppt pptx properties rtf text txt vtt xls xlsx xml".split( + " ", + ), + ); + }); + it.each([ + [" Report.JSONL ", "application/json", "jsonl"], + ["x.csv", "application/vnd.ms-excel", "csv"], + ["x.unknown", "text/html; charset=utf-8", "html"], + ["x.yaml", "text/plain", "yaml"], + ["README", "text/plain", "markdown"], + ["x.pdf", "text/plain", "unstructured"], + ["x.xyz", "", null], + ["x.constructor", "", null], + ["x.__proto__", "", null], + ])("resolves %s deterministically", (filename, mimeType, expected) => { + expect(resolveDocumentFormat({ filename, mimeType })).toBe(expected); + }); + it("never bypasses native resource limits by sending oversized markup remotely", async () => { + let requests = 0; + const parser = createParserRouter({ + html: createNativeHtmlParser(), + markdown: createNativeMarkdownParser(), + maxNativeInputBytes: 3, + unstructured: createUnstructuredParserClient({ + endpoint: "https://parser.test", + fetch: async () => { + requests += 1; + return Response.json([]); + }, + }), + }); + await expect( + parser.parse({ + body: new TextEncoder().encode("too long"), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "x.md", + mimeType: "text/markdown", + version: 1, + }), + ).rejects.toMatchObject({ code: "provider_input" }); + expect(requests).toBe(0); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/document-format-registry.ts b/knowledge-fs/packages/parsers/src/document-format-registry.ts new file mode 100644 index 00000000000..b6244ac6d8b --- /dev/null +++ b/knowledge-fs/packages/parsers/src/document-format-registry.ts @@ -0,0 +1,119 @@ +/** One source of truth for upload MIME aliases and parser family. Internal source formats do not widen uploads. */ +export const DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION = { + csv: ["text/csv", "application/vnd.ms-excel"], + doc: ["application/msword"], + docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"], + eml: ["message/rfc822"], + epub: ["application/epub+zip"], + htm: ["text/html"], + html: ["text/html"], + json: ["application/json"], + jsonl: ["application/x-ndjson", "application/jsonl", "application/ndjson", "application/json"], + markdown: ["text/markdown", "text/x-markdown", "text/plain"], + md: ["text/markdown", "text/x-markdown", "text/plain"], + mdx: ["text/mdx", "text/markdown", "text/plain"], + msg: ["application/vnd.ms-outlook", "application/x-msg"], + odt: ["application/vnd.oasis.opendocument.text"], + pdf: ["application/pdf"], + ppt: ["application/vnd.ms-powerpoint", "application/mspowerpoint", "application/x-mspowerpoint"], + pptx: ["application/vnd.openxmlformats-officedocument.presentationml.presentation"], + properties: ["text/x-java-properties", "text/plain"], + rtf: ["application/rtf", "text/rtf", "application/x-rtf"], + text: ["text/plain"], + txt: ["text/plain"], + vtt: ["text/vtt", "text/plain"], + xls: [ + "application/vnd.ms-excel", + "application/excel", + "application/x-excel", + "application/x-msexcel", + ], + xlsx: ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"], + xml: ["application/xml", "text/xml"], +} as const satisfies Readonly>; + +export type DocumentFormat = + | "csv" + | "html" + | "json" + | "jsonl" + | "markdown" + | "properties" + | "unstructured" + | "vtt" + | "xml" + | "yaml"; +const formatsByExtension: Readonly> = { + csv: "csv", + doc: "unstructured", + docx: "unstructured", + eml: "unstructured", + epub: "unstructured", + htm: "html", + html: "html", + json: "json", + jsonl: "jsonl", + markdown: "markdown", + md: "markdown", + mdx: "markdown", + msg: "unstructured", + odt: "unstructured", + pdf: "unstructured", + ppt: "unstructured", + pptx: "unstructured", + properties: "properties", + rtf: "unstructured", + text: "markdown", + txt: "markdown", + vtt: "vtt", + xls: "unstructured", + xlsx: "unstructured", + xml: "xml", + // Source connectors historically support these; upload admission deliberately remains unchanged. + ndjson: "jsonl", + yaml: "yaml", + yml: "yaml", +}; + +const formatsByMime = new Map(); +for (const [extension, mimes] of Object.entries(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION)) { + const format = formatsByExtension[extension]; + if (format) for (const mime of mimes) formatsByMime.set(mime, format); +} +// Ambiguous upload aliases must not change generic MIME routing (CSV may be declared Excel). +formatsByMime.set("text/plain", "markdown"); +formatsByMime.set("text/markdown", "markdown"); +formatsByMime.set("application/json", "json"); +formatsByMime.set("application/xhtml+xml", "html"); +formatsByMime.set("text/json", "json"); +for (const mime of ["application/yaml", "text/yaml", "application/x-yaml"]) + formatsByMime.set(mime, "yaml"); + +export function documentExtension(filename: string): string | undefined { + const normalized = filename.trim().toLowerCase(); + const dot = normalized.lastIndexOf("."); + return dot >= 0 && dot < normalized.length - 1 ? normalized.slice(dot + 1) : undefined; +} + +export function resolveDocumentFormat(input: { + readonly filename: string; + readonly mimeType: string; +}): DocumentFormat | null { + const extension = documentExtension(input.filename); + const byExtension = + extension && Object.hasOwn(formatsByExtension, extension) + ? formatsByExtension[extension] + : undefined; + return ( + byExtension ?? + formatsByMime.get(input.mimeType.split(";", 1)[0]?.trim().toLowerCase() ?? "") ?? + null + ); +} + +export function documentMimeTypesForFilename(filename: string): readonly string[] | undefined { + const extension = documentExtension(filename); + const registry: Readonly> = + DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION; + return extension && Object.hasOwn(registry, extension) ? registry[extension] : undefined; +} diff --git a/knowledge-fs/packages/parsers/src/document-table-bytes.test.ts b/knowledge-fs/packages/parsers/src/document-table-bytes.test.ts new file mode 100644 index 00000000000..f2ed9dca3a7 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/document-table-bytes.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createNativeHtmlParser, + createNativeMarkdownParser, + createUnstructuredParserClient, +} from "./index"; + +vi.mock("./parser-resource-budget", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + parserResourceLimits: { ...actual.parserResourceLimits, maxOutputBytes: 64 }, + }; +}); + +function input(body: string, extension = "html") { + return { + body: new TextEncoder().encode(body), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: `tables.${extension}`, + mimeType: + extension === "html" + ? "text/html" + : extension === "pdf" + ? "application/pdf" + : "text/markdown", + version: 1, + }; +} + +const label = "中".repeat(12); +const htmlTable = `
${label}
`; +const markdownTable = `| ${label} |\n| --- |`; + +describe("document-local table projection bytes", () => { + it("charges header-only HTML tables against one byte budget", async () => { + await expect( + createNativeHtmlParser().parse(input(htmlTable + htmlTable)), + ).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + message: expect.stringContaining("table projection bytes"), + }); + }); + + it("charges headerless HTML data rows against one byte budget", async () => { + const table = `
${label}
`; + await expect(createNativeHtmlParser().parse(input(table + table))).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("table projection bytes"), + }); + }); + + it.each(["md", "mdx"])( + "shares bytes across ordinary tables and HTML tokens in %s", + async (extension) => { + for (const body of [ + `${markdownTable}\n\nBetween\n\n${markdownTable}`, + `${markdownTable}\n\n${htmlTable}`, + ]) { + await expect( + createNativeMarkdownParser().parse(input(body, extension)), + ).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("table projection bytes"), + }); + } + }, + ); + + it("charges every provider table before joining subsequent table output", async () => { + const table = { type: "Table", metadata: { text_as_html: htmlTable } }; + const parser = createUnstructuredParserClient({ + endpoint: "http://parser.invalid", + fetch: async () => Response.json([table, table]), + }); + await expect(parser.parse(input("%PDF-1.4", "pdf"))).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + cause: { message: expect.stringContaining("table projection bytes") }, + }); + }); + + it("admits the exact cumulative byte boundary without charging header labels twice", async () => { + const table = `
${"x".repeat(32)}
`; + const artifact = await createNativeHtmlParser().parse(input(table + table)); + expect(artifact.elements.map((element) => element.text)).toEqual([ + "x".repeat(32), + "x".repeat(32), + ]); + }); + + it.each(["html", "md", "mdx"])( + "keeps independent %s parses byte-budget isolated", + async (extension) => { + const parser = extension === "html" ? createNativeHtmlParser() : createNativeMarkdownParser(); + const source = input(extension === "html" ? htmlTable : markdownTable, extension); + expect((await parser.parse(source)).elements[0]?.text).toBe(label); + expect((await parser.parse(source)).elements[0]?.text).toBe(label); + }, + ); + + it("keeps independent provider transports byte-budget isolated", async () => { + const table = { type: "Table", metadata: { text_as_html: htmlTable } }; + const fetch = vi.fn(async () => Response.json([table])); + const parser = createUnstructuredParserClient({ endpoint: "http://parser.invalid", fetch }); + const source = input("%PDF-1.4", "pdf"); + expect((await parser.parse(source)).elements[0]?.text).toBe(label); + expect((await parser.parse(source)).elements[0]?.text).toBe(label); + expect(fetch).toHaveBeenCalledTimes(2); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/html-table-expansion.test.ts b/knowledge-fs/packages/parsers/src/html-table-expansion.test.ts new file mode 100644 index 00000000000..0ef7d329632 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/html-table-expansion.test.ts @@ -0,0 +1,218 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createNativeHtmlParser, + createNativeMarkdownParser, + createUnstructuredParserClient, +} from "./index"; + +vi.mock("./parser-resource-budget", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + parserResourceLimits: { + ...actual.parserResourceLimits, + maxOutputBytes: 256, + maxTableCells: 6, + maxTableColumns: 4, + }, + }; +}); + +function parseTable(table: string) { + return createNativeHtmlParser().parse({ + body: new TextEncoder().encode(table), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "table.html", + mimeType: "text/html", + version: 1, + }); +} + +describe("HTML table pre-expansion resource budget", () => { + it("shares the expansion budget across separate empty HTML tables", async () => { + const table = '
'; + await expect(parseTable(table + table)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it("charges ragged-row padding against the document budget before dense projection", async () => { + const table = "
A
BC
"; + await expect(parseTable(table + table)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it.each(["md", "mdx"])( + "shares the expansion budget across separate %s HTML tokens", + async (extension) => { + const table = '
'; + await expect( + createNativeMarkdownParser().parse({ + body: new TextEncoder().encode(`${table}\n\nBetween\n\n${table}`), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: `table.${extension}`, + mimeType: "text/markdown", + version: 1, + }), + ).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }, + ); + + it("shares the expansion budget across separate provider tables", async () => { + const table = { + type: "Table", + metadata: { text_as_html: '
' }, + }; + const parser = createUnstructuredParserClient({ + endpoint: "http://parser.invalid", + fetch: async () => new Response(JSON.stringify([table, table])), + }); + await expect( + parser.parse({ + body: new TextEncoder().encode("%PDF-1.4\n"), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "table.pdf", + mimeType: "application/pdf", + version: 1, + }), + ).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + cause: { message: expect.stringContaining("expanded HTML table cell count") }, + }); + }); + + it("does not retain provider table budget between independent transports", async () => { + const table = { + type: "Table", + metadata: { + text_as_html: '
', + }, + }; + const fetch = vi.fn(async () => new Response(JSON.stringify([table]))); + const parser = createUnstructuredParserClient({ endpoint: "http://parser.invalid", fetch }); + const input = { + body: new TextEncoder().encode("%PDF-1.4\n"), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "table.pdf", + mimeType: "application/pdf", + version: 1, + }; + await expect(parser.parse(input)).resolves.toMatchObject({ parser: "unstructured" }); + await expect(parser.parse(input)).resolves.toMatchObject({ parser: "unstructured" }); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it.each(["html", "md", "mdx"])( + "does not share the expansion budget across independent %s parses", + async (extension) => { + const parser = extension === "html" ? createNativeHtmlParser() : createNativeMarkdownParser(); + const input = { + body: new TextEncoder().encode( + '
', + ), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: `table.${extension}`, + mimeType: extension === "html" ? "text/html" : "text/markdown", + version: 1, + }; + await expect(parser.parse(input)).resolves.toMatchObject({ elements: [] }); + await expect(parser.parse(input)).resolves.toMatchObject({ elements: [] }); + }, + ); + + it("rejects an overwide empty colspan before filtered rows can hide its allocation", async () => { + await expect(parseTable('
')).rejects.toMatchObject( + { + code: "provider_input", + retryable: false, + message: expect.stringContaining("HTML table column count"), + }, + ); + }); + + it("charges cumulative empty cell spans even when no row produces searchable text", async () => { + const html = '
'; + await expect(parseTable(html)).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it("charges carried rowspans against the same cumulative cell budget", async () => { + const html = '
'; + await expect(parseTable(html)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it("charges the logical width of sparse carried rows, not just their nonempty cell count", async () => { + const html = + '
'; + await expect(parseTable(html)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it("bounds dense header traversal before flattening ragged rows", async () => { + const html = '
A
B
'; + await expect(parseTable(html)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it("bounds dense headerless traversal before inferring column types", async () => { + const html = '
A
B
'; + await expect(parseTable(html)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("expanded HTML table cell count"), + }); + }); + + it("bounds repeated header label bytes before constructing joined columns", async () => { + const html = `
${"x".repeat(65)}
`; + await expect(parseTable(html)).rejects.toMatchObject({ + code: "provider_input", + message: expect.stringContaining("HTML table header bytes"), + }); + }); + + it("preserves empty tables exactly at the expansion boundary", async () => { + const artifact = await parseTable( + '
', + ); + expect(artifact.elements).toEqual([]); + }); + + it("preserves admitted multirow headers and rowspan carry semantics", async () => { + const html = + '
NameQ1
Q2
A1
'; + const artifact = await parseTable(html); + expect(artifact.elements[0]?.text).toBe("Name: A | Q1 / Q2: 1"); + expect(artifact.elements[0]?.metadata.table).toMatchObject({ + columns: ["Name", "Q1 / Q2"], + headerRowCount: 2, + recordCount: 1, + sourceRowCount: 3, + }); + }); + + it("preserves colspan carry before a later source cell without double charging", async () => { + const html = '
X
Y
'; + const artifact = await parseTable(html); + expect(artifact.elements[0]?.text).toBe( + "column_1: X | column_2: X | column_3:\ncolumn_1: X | column_2: X | column_3: Y", + ); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/index.ts b/knowledge-fs/packages/parsers/src/index.ts index 8802e102856..e7905264156 100644 --- a/knowledge-fs/packages/parsers/src/index.ts +++ b/knowledge-fs/packages/parsers/src/index.ts @@ -16,7 +16,37 @@ import { ParseElementSchema, } from "@knowledge/core"; +import { resolveDocumentFormat } from "./document-format-registry"; +import { OfficeArchiveAdmissionError, assertOfficeArchiveSafe } from "./office-parser-preflight"; +import { createArchiveMediaReportCollector } from "./parse-coverage"; +export { + DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION, + documentExtension, + documentMimeTypesForFilename, + resolveDocumentFormat, +} from "./document-format-registry"; +import { + ParserResourceLimitError, + assertParserResourceBudget, + parserResourceLimits, +} from "./parser-resource-budget"; +import { + documentJsonRootType, + isDocumentRecord, + parseDocumentJson, + stringifyDocumentJson, +} from "./structured-json"; +import { assertXmlStructureBudget, iterateDocumentLines } from "./structured-stream-admission"; +import { decodeDocumentText, propertiesElements, vttElements } from "./text-document-contracts"; +import { createUnstructuredGlyphIndex } from "./unstructured-glyph-index"; +import { + maxUnstructuredSectionDepth, + maxUnstructuredSectionPathItems, + maxUnstructuredVerticalCandidateComparisons, +} from "./unstructured-normalization-policy"; import { createUnstructuredRequestCoordinator } from "./unstructured-request-coordinator"; +import { parseUnstructuredResponsePayload } from "./unstructured-response-budget"; +import { classifyUnstructuredResourceResponse } from "./unstructured-sandbox-response"; import { type UnstructuredWorkloadClassification, classifyUnstructuredWorkload, @@ -203,6 +233,8 @@ export interface NativeParserOptions { } export interface UnstructuredParserClientOptions extends NativeParserOptions { + /** Deployment-owned semantic identity (pinned image + extraction policy), not transport limits. */ + readonly backendRevision?: string; readonly apiKey?: string; readonly defaultLanguage?: string; readonly endpoint: string; @@ -220,10 +252,16 @@ export interface UnstructuredParserClientOptions extends NativeParserOptions { /** @deprecated Compatibility alias for `heavyRequestTimeoutMs`. */ readonly pdfRequestTimeoutMs?: number; readonly requestTimeoutMs?: number; + /** Platform-specific resource checks, inside admission and before any remote work starts. */ + readonly requestPreflight?: UnstructuredRequestPreflight; readonly retryDelayMs?: number; readonly sleep?: (ms: number) => Promise; } +export interface UnstructuredRequestPreflight { + check(input: ParseDocumentInput): Promise; +} + export interface StructuredDataParserOptions extends NativeParserOptions { readonly maxRows?: number; } @@ -280,20 +318,6 @@ const defaultMaxRows = 20_000; const defaultRetryDelayMs = 100; const defaultNow = () => new Date().toISOString(); const defaultGenerateId = () => crypto.randomUUID(); -const unstructuredDocumentExtensions = new Set([ - "doc", - "docx", - "eml", - "epub", - "msg", - "odt", - "pdf", - "ppt", - "pptx", - "rtf", - "xls", - "xlsx", -]); const UnstructuredElementSchema = z.object({ element_id: z.string().min(1).max(512).optional(), @@ -332,7 +356,7 @@ function parserPolicyFingerprintHash(context: string): string { export function createNativeMarkdownParser(options: NativeParserOptions = {}): ParserAdapter { const policyFingerprint = (input: ParseDocumentInput): string => { const parserVersion = - options.parserVersion ?? (isMdxInput(input) ? "native-mdx@2" : "native-markdown@2"); + options.parserVersion ?? (isMdxInput(input) ? "native-mdx@4" : "native-markdown@4"); return nativeParserPolicyFingerprint(input, "native-markdown", parserVersion, options); }; @@ -340,14 +364,22 @@ export function createNativeMarkdownParser(options: NativeParserOptions = {}): P kind: "native-markdown", policyFingerprint, parse: async (input) => { + input.signal?.throwIfAborted(); const isMdx = isMdxInput(input); - const parserVersion = options.parserVersion ?? (isMdx ? "native-mdx@2" : "native-markdown@2"); + const parserVersion = options.parserVersion ?? (isMdx ? "native-mdx@4" : "native-markdown@4"); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); - const text = decodeUtf8(input.body); - const tokens = marked.lexer(text, { gfm: true }); - const elements = markdownTokensToElements(tokens, { preserveHtmlText: isMdx }); + const { text, encoding } = decodeDocumentText(input.body); + const extension = input.filename.trim().toLowerCase().split(".").at(-1); + const mimeType = normalizedMimeType(input.mimeType); + const elements = + extension === "properties" || mimeType === "text/x-java-properties" + ? propertiesElements(text, options.maxElements ?? defaultMaxElements) + : extension === "vtt" || mimeType === "text/vtt" + ? vttElements(text, options.maxElements ?? defaultMaxElements) + : markdownTokensToElements(marked.lexer(text, { gfm: true })); return createParseArtifact({ + artifactMetadata: { textEncoding: encoding }, elements, input, kind: "native-markdown", @@ -359,15 +391,16 @@ export function createNativeMarkdownParser(options: NativeParserOptions = {}): P } export function createNativeHtmlParser(options: NativeParserOptions = {}): ParserAdapter { - const parserVersion = options.parserVersion ?? "native-html@3"; + const parserVersion = options.parserVersion ?? "native-html@5"; return { kind: "native-html", policyFingerprint: (input) => nativeParserPolicyFingerprint(input, "native-html", parserVersion, options), parse: async (input) => { + input.signal?.throwIfAborted(); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); - const text = decodeUtf8(input.body); + const { text, encoding } = decodeDocumentText(input.body); const document = parseDocument(text, { lowerCaseAttributeNames: true, lowerCaseTags: true, @@ -377,7 +410,7 @@ export function createNativeHtmlParser(options: NativeParserOptions = {}): Parse const documentTitle = htmlDocumentTitle(nodes); return createParseArtifact({ - ...(documentTitle ? { artifactMetadata: { documentTitle } } : {}), + artifactMetadata: { ...(documentTitle ? { documentTitle } : {}), textEncoding: encoding }, elements, input, kind: "native-html", @@ -391,7 +424,7 @@ export function createNativeHtmlParser(options: NativeParserOptions = {}): Parse export function createNativeStructuredDataParser( options: StructuredDataParserOptions = {}, ): ParserAdapter { - const parserVersion = options.parserVersion ?? "native-structured@2"; + const parserVersion = options.parserVersion ?? "native-structured@4"; return { kind: "native-structured", @@ -403,17 +436,24 @@ export function createNativeStructuredDataParser( }), ), parse: async (input) => { + input.signal?.throwIfAborted(); assertInputBounds(input.body, options.maxInputBytes ?? defaultMaxInputBytes); - const text = decodeUtf8(input.body); + const { text, encoding } = decodeDocumentText(input.body); const format = structuredDataFormat(input); if (!format) { throw new ProviderUnsupportedFileTypeError("Structured parser unsupported file type"); } - const elements = structuredDataElements(format, text, options.maxRows ?? defaultMaxRows); + const elements = structuredDataElements( + format, + text, + options.maxRows ?? defaultMaxRows, + input.signal, + ); return createParseArtifact({ + artifactMetadata: { textEncoding: encoding }, elements, input, kind: "native-structured", @@ -426,6 +466,7 @@ export function createNativeStructuredDataParser( export function createUnstructuredParserClient({ apiKey, + backendRevision = "external-unversioned", defaultLanguage, endpoint, fetch: fetchImpl = fetch, @@ -436,6 +477,7 @@ export function createUnstructuredParserClient({ maxRetries = defaultMaxRetries, pdfMaxConcurrency, requestTimeoutMs = defaultRequestTimeoutMs, + requestPreflight, pdfRequestTimeoutMs, retryDelayMs = defaultRetryDelayMs, sleep = sleepMs, @@ -466,7 +508,12 @@ export function createUnstructuredParserClient({ const requestGate = createAbortAwareConcurrencyGate(maxConcurrency); const heavyRequestGate = createAbortAwareConcurrencyGate(effectiveHeavyMaxConcurrency); const requestCoordinator = createUnstructuredRequestCoordinator(); - const parserVersion = options.parserVersion ?? "unstructured@10"; + const parserVersion = options.parserVersion ?? "unstructured@12"; + if (!backendRevision.trim() || backendRevision.length > 256) { + throw new ProviderInputError( + "Unstructured backendRevision must be a non-empty bounded identity", + ); + } const maxInputBytes = options.maxInputBytes ?? defaultMaxInputBytes; const workloadCache = new WeakMap< Uint8Array, @@ -488,6 +535,7 @@ export function createUnstructuredParserClient({ return classification; }; const resolveRequestPolicy = (input: ParseDocumentInput): UnstructuredRequestPolicy => ({ + backendRevision, partitionStrategy: unstructuredPartitionStrategy(input), providerImageBlockTypes: unstructuredProviderImageBlockTypes(input), providerLanguage: unstructuredLanguage(input.parserHints?.language ?? defaultLanguage), @@ -520,6 +568,7 @@ export function createUnstructuredParserClient({ ); } const requestPolicy = { + backendRevision, partitionStrategy, providerImageBlockTypes, providerLanguage, @@ -541,6 +590,19 @@ export function createUnstructuredParserClient({ // durable retry overlap a provider request that survived the client disconnect. request: async ({ markTransportStarted, signal: admissionSignal }) => { const runTransport = async (): Promise => { + // File inspection shares the remote admission slot and single-flight lifetime. Never + // mark an operation as remotely started until local checks succeed: a rejected file + // has no ambiguous provider outcome and cancellation can still stop inspection. + try { + await assertOfficeArchiveSafe({ ...input, signal: admissionSignal }); + } catch (error) { + if (error instanceof OfficeArchiveAdmissionError) { + throw new ProviderInputError(error.message); + } + throw error; + } + await requestPreflight?.check({ ...input, signal: admissionSignal }); + admissionSignal.throwIfAborted(); markTransportStarted(); // This deadline belongs to the transport, not to any one caller. The coordinator waits // for it to settle before it reports a caller abort. @@ -592,10 +654,15 @@ export function createUnstructuredParserClient({ let payload: unknown; try { - payload = JSON.parse(responseText); + payload = parseUnstructuredResponsePayload(responseText, { + maxResponseBytes, + signal: deadline.signal, + }); } catch (error) { throw new ProviderResponseError( - "Unstructured parser returned an invalid response", + error instanceof ParserResourceLimitError + ? "Unstructured parser response exceeds structural resource limits" + : "Unstructured parser returned an invalid response", { cause: error }, ); } @@ -609,12 +676,21 @@ export function createUnstructuredParserClient({ const providerElements = unstructuredElementsToElements( normalizeUnstructuredLayout(parsed.data), ); - const elements = appendArchiveMediaFallbackElements(input, providerElements); + const mediaReport = createArchiveMediaReportCollector( + input.parserHints?.requiresImages, + ); + const elements = appendArchiveMediaFallbackElements( + input, + providerElements, + mediaReport, + ); const artifact = await createParseArtifact({ artifactHashContext: unstructuredArtifactHashContext(input, requestPolicy), + artifactMetadata: { backendRevision, ...mediaReport.metadata() }, elements, - input, + // The shared operation can outlive the first caller's cancellation. + input: { ...input, signal: admissionSignal }, kind: "unstructured", options, parserVersion, @@ -631,6 +707,12 @@ export function createUnstructuredParserClient({ { cause: error }, ); } + if (error instanceof ParserResourceLimitError) { + throw new ProviderResponseError( + "Unstructured normalized output exceeds structural resource limits", + { cause: error }, + ); + } throw error; } finally { deadline.dispose(); @@ -727,6 +809,7 @@ function normalizedMimeType(value: string): string { } interface UnstructuredRequestPolicy { + readonly backendRevision: string; readonly partitionStrategy: "auto" | "fast" | "hi_res"; readonly providerImageBlockTypes: readonly ("Image" | "Table")[]; readonly providerLanguage?: string | undefined; @@ -761,13 +844,16 @@ function unstructuredArtifactHashContext( const hints = input.parserHints; return JSON.stringify({ + backendRevision: request.backendRevision, filename: input.filename, mimeType: input.mimeType.trim().toLowerCase(), parserHints: { imagesHandledExternally: hints?.imagesHandledExternally === true, language: hints?.language?.trim().toLowerCase() || null, layoutComplexity: hints?.layoutComplexity ?? null, - requiresImages: hints?.requiresImages === true, + // Legacy auto extraction (undefined) and explicit text-only (false) produce different + // archive media and must never share a checkpoint or in-flight request identity. + requiresImages: hints?.requiresImages ?? null, requiresOcr: hints?.requiresOcr === true, requiresTables: hints?.requiresTables === true, }, @@ -790,10 +876,15 @@ function unstructuredPartitionEndpoint(endpoint: string): string { function appendArchiveMediaFallbackElements( input: ParseDocumentInput, elements: readonly ParseElementInput[], + report: ReturnType, ): ParseElementInput[] { const roots = archiveMediaRoots(input); - if (!roots || !zipSignatureIsSupported(input.body)) { + if ( + !roots || + !zipSignatureIsSupported(input.body) || + input.parserHints?.requiresImages === false + ) { return [...elements]; } @@ -816,6 +907,10 @@ function appendArchiveMediaFallbackElements( try { const archive = unzipSync(input.body, { filter: (file) => { + if (/^(?:word|ppt|xl)\/(?:charts|diagrams)\/[^/]+\.xml$/iu.test(file.name)) { + report.observe(); + report.skip(file.name, "office-visual-structure-not-rendered"); + } if ( officeArchiveMetadataPath(input, file.name) && selectedMetadataCount < defaultMaxArchiveMetadataCount && @@ -829,15 +924,27 @@ function appendArchiveMediaFallbackElements( return true; } - if ( - selectedCount >= defaultMaxArchiveImageCount || - file.originalSize < 1 || - file.originalSize > defaultMaxArchiveImageBytes || - selectedBytes + file.originalSize > defaultMaxArchiveImageTotalBytes || - !archivePathIsSafe(file.name) || - !archivePathMatchesRoots(file.name, roots) || - !archiveImageContentType(file.name) - ) { + const mediaCandidate = + archivePathMatchesRoots(file.name, roots) && + !file.name.endsWith("/") && + (roots[0] !== "" || + /\.(?:png|jpe?g|gif|webp|svg|tiff?|bmp|emf|wmf|avif|heic|ico)$/iu.test(file.name)); + if (!mediaCandidate) return false; + report.observe(); + const reason = !archivePathIsSafe(file.name) + ? "unsafe-resource-reference" + : !archiveImageContentType(file.name) + ? "unsupported-media-format" + : file.originalSize < 1 + ? "empty-media-resource" + : selectedCount >= defaultMaxArchiveImageCount + ? "media-count-budget" + : file.originalSize > defaultMaxArchiveImageBytes || + selectedBytes + file.originalSize > defaultMaxArchiveImageTotalBytes + ? "media-byte-budget" + : undefined; + if (reason) { + report.skip(file.name, reason); return false; } @@ -925,6 +1032,7 @@ function appendArchiveMediaFallbackElements( } catch { // The authoritative parser response remains usable even when an optional archive-media // fallback cannot inspect a malformed or unsupported ZIP container. + report.skip("", "archive-media-inspection-failed"); return [...elements]; } } @@ -1978,8 +2086,7 @@ function selectParser( throw new Error("Parser router maxNativeInputBytes must be at least 1"); } - const mimeType = input.mimeType.toLowerCase(); - const filename = input.filename.toLowerCase(); + const format = resolveDocumentFormat(input); const language = input.parserHints?.language?.trim().toLowerCase(); if (input.parserHints?.requiresOcr) { @@ -1998,13 +2105,13 @@ function selectParser( return { parser: unstructured, reason: "unsupported-native-language" }; } - if (unstructuredDocumentExtensions.has(filename.split(".").at(-1) ?? "")) { + if (format === "unstructured") { return { parser: unstructured, reason: "complex-file-type" }; } const structuredFormat = structuredDataFormat(input); - if (structuredFormat && input.body.byteLength > maxNativeInputBytes) { + if (!structured && structuredFormat && input.body.byteLength > maxNativeInputBytes) { return { parser: unstructured, reason: "native-size-limit" }; } @@ -2013,21 +2120,9 @@ function selectParser( } const nativeParser = - mimeType === "text/markdown" || - mimeType === "text/mdx" || - mimeType === "text/plain" || - mimeType === "text/vtt" || - mimeType === "text/x-java-properties" || - filename.endsWith(".md") || - filename.endsWith(".markdown") || - filename.endsWith(".mdx") || - filename.endsWith(".properties") || - filename.endsWith(".vtt") + format === "markdown" || format === "properties" || format === "vtt" ? markdown - : mimeType === "text/html" || - mimeType === "application/xhtml+xml" || - filename.endsWith(".html") || - filename.endsWith(".htm") + : format === "html" ? html : null; @@ -2036,7 +2131,9 @@ function selectParser( } if (input.body.byteLength > maxNativeInputBytes) { - return { parser: unstructured, reason: "native-size-limit" }; + throw new ProviderInputError( + `Native parser input exceeds maxNativeInputBytes=${maxNativeInputBytes}`, + ); } return { parser: nativeParser, reason: "native-file-type" }; @@ -2061,8 +2158,20 @@ async function createParseArtifact({ }): Promise { const maxElements = options.maxElements ?? defaultMaxElements; + input.signal?.throwIfAborted(); + assertParserResourceBudget( + { elements, metadata: artifactMetadata }, + { + maxNodes: parserResourceLimits.maxArtifactNodes, + signal: input.signal, + }, + ); + if (elements.length > maxElements) { - throw new Error(`Parser output exceeds maxElements=${maxElements}`); + const message = `Parser output exceeds maxElements=${maxElements}`; + throw kind === "unstructured" + ? new ProviderResponseError(message) + : new ProviderInputError(message); } const id = (options.generateId ?? defaultGenerateId)(); @@ -2099,54 +2208,21 @@ function structuredDataFormat({ filename, mimeType, }: Pick): StructuredDataFormat | null { - const normalizedMime = mimeType.toLowerCase(); - const normalizedFilename = filename.toLowerCase(); - - if (normalizedMime === "text/csv" || normalizedFilename.endsWith(".csv")) { - return "csv"; - } - - if (normalizedFilename.endsWith(".jsonl") || normalizedFilename.endsWith(".ndjson")) { - return "jsonl"; - } - - if (normalizedFilename.endsWith(".json")) { - return "json"; - } - - if (normalizedMime === "application/x-ndjson" || normalizedMime === "application/jsonl") { - return "jsonl"; - } - - if (normalizedMime === "application/json" || normalizedMime === "text/json") { - return "json"; - } - - if ( - normalizedMime === "application/yaml" || - normalizedMime === "text/yaml" || - normalizedMime === "application/x-yaml" || - normalizedFilename.endsWith(".yaml") || - normalizedFilename.endsWith(".yml") - ) { - return "yaml"; - } - - if ( - normalizedMime === "application/xml" || - normalizedMime === "text/xml" || - normalizedFilename.endsWith(".xml") - ) { - return "xml"; - } - - return null; + const format = resolveDocumentFormat({ filename, mimeType }); + return format === "csv" || + format === "json" || + format === "jsonl" || + format === "xml" || + format === "yaml" + ? format + : null; } function structuredDataElements( format: StructuredDataFormat, text: string, maxRows: number, + signal?: AbortSignal, ): ParseElementInput[] { if (!Number.isInteger(maxRows) || maxRows < 1) { throw new Error("Structured parser maxRows must be at least 1"); @@ -2154,54 +2230,93 @@ function structuredDataElements( try { if (format === "csv") { - return rowsToTableElements(format, parseCsvRows(text, maxRows), maxRows); + const { columns, rows } = parseCsvRows(text, maxRows); + return rowsToTableElements(format, rows, maxRows, columns); } if (format === "jsonl") { - return rowsToTableElements(format, parseJsonLines(text, maxRows), maxRows); + const records = parseJsonLines(text, maxRows, signal); + if (records.every(isDocumentRecord)) return rowsToTableElements(format, records, maxRows); + const lines = records.map((record) => stringifyDocumentJson(record)); + assertParserResourceBudget(lines); + return [ + { + metadata: { format, rowCount: records.length }, + sectionPath: [], + text: lines.join("\n"), + type: "code", + }, + ]; } if (format === "json") { - return structuredValueElements(format, JSON.parse(text), maxRows); + return structuredValueElements(format, parseDocumentJson(text), maxRows); } if (format === "yaml") { return structuredValueElements(format, parseYaml(text), maxRows); } - return structuredValueElements(format, new XMLParser().parse(text), maxRows); + assertXmlStructureBudget(text, { signal }); + return structuredValueElements( + format, + new XMLParser({ + ignoreAttributes: false, + parseTagValue: false, + parseAttributeValue: false, + }).parse(text), + maxRows, + ); } catch (error) { + signal?.throwIfAborted(); + if (error instanceof ParserResourceLimitError) throw error; if (error instanceof Error && error.message.startsWith("Structured parser")) { - throw error; + throw new ProviderInputError(error.message, { cause: error }); } - throw new Error("Structured parser returned an invalid response"); + throw new ProviderInputError("Structured parser input is malformed", { cause: error }); } } -function parseCsvRows(text: string, maxRows: number): Record[] { +function parseCsvRows( + text: string, + maxRows: number, +): { columns: readonly string[]; rows: Record[] } { let rows = 0; + let columns: string[] | undefined; + const records: Record[] = []; - return parseCsv(text, { - columns: true, - on_record: (record) => { + parseCsv(text, { + // Decode arrays first: csv-parse's object projection assigns __proto__ instead of defining + // it as an own column. Object.fromEntries preserves every user-supplied header safely. + on_record: (record: string[]) => { + if (!columns) { + if (record.length > parserResourceLimits.maxTableColumns) + throw new ParserResourceLimitError("table column count"); + columns = uniqueColumnNames(record); + return null; + } rows += 1; if (rows > maxRows) { throw new Error(`Structured parser row count exceeds maxRows=${maxRows}`); } - return record as Record; + records.push( + Object.fromEntries(columns.map((column, index) => [column, record[index] ?? ""])), + ); + return null; }, skip_empty_lines: true, trim: true, - }) as Record[]; + }); + return { columns: columns ?? [], rows: records }; } -function parseJsonLines(text: string, maxRows: number): Record[] { - const rows: Record[] = []; +function parseJsonLines(text: string, maxRows: number, signal?: AbortSignal): unknown[] { + const rows: unknown[] = []; - for (const rawLine of text.split(/\r?\n/)) { + for (const rawLine of iterateDocumentLines(text, signal)) { const line = rawLine.trim(); if (!line) { @@ -2212,7 +2327,7 @@ function parseJsonLines(text: string, maxRows: number): Record[ throw new Error(`Structured parser row count exceeds maxRows=${maxRows}`); } - rows.push(JSON.parse(line) as Record); + rows.push(parseDocumentJson(line)); } return rows; @@ -2223,10 +2338,8 @@ function structuredValueElements( value: unknown, maxRows: number, ): ParseElementInput[] { - if ( - Array.isArray(value) && - value.every((item) => item && typeof item === "object" && !Array.isArray(item)) - ) { + assertParserResourceBudget(value); + if (Array.isArray(value) && value.every(isDocumentRecord)) { return rowsToTableElements(format, value as Record[], maxRows); } @@ -2234,10 +2347,10 @@ function structuredValueElements( { metadata: { format, - rootType: Array.isArray(value) ? "array" : typeof value, + rootType: documentJsonRootType(value), }, sectionPath: [], - text: JSON.stringify(value, null, 2), + text: stringifyDocumentJson(value, true), type: "code", }, ]; @@ -2247,17 +2360,63 @@ function rowsToTableElements( format: StructuredDataFormat, rows: readonly Record[], maxRows: number, + sourceColumns?: readonly string[], ): ParseElementInput[] { if (rows.length > maxRows) { throw new Error(`Structured parser row count exceeds maxRows=${maxRows}`); } - const columns = uniqueStrings(rows.flatMap((row) => Object.keys(row))); + assertParserResourceBudget(rows); + const columns = sourceColumns ?? uniqueStrings(rows.flatMap((row) => Object.keys(row))); + if (columns.length > parserResourceLimits.maxTableColumns) + throw new ParserResourceLimitError("table column count"); + const actualCellCount = rows.reduce((count, row) => count + Object.keys(row).length, 0); + if (actualCellCount > parserResourceLimits.maxTableCells) + throw new ParserResourceLimitError("table cell count"); + const denseCellCount = rows.length * columns.length; + if (denseCellCount > Math.max(actualCellCount * 8, 10_000)) { + const lines = rows.map((row) => + Object.entries(row) + .map( + ([key, value]) => + `${normalizeTableCell(key)}: ${normalizeTableCell(structuredCell(value))}`, + ) + .join(" | "), + ); + assertParserResourceBudget(lines); + return [ + { + metadata: { + columns, + format, + rowCount: rows.length, + table: { + columns, + headerRowCount: 0, + mode: "record-list", + recordCount: rows.length, + semanticVersion: 1, + sourceRowCount: rows.length, + sparse: true, + }, + }, + sectionPath: [], + text: lines.join("\n"), + type: "table", + }, + ]; + } + if (denseCellCount > parserResourceLimits.maxTableCells) + throw new ParserResourceLimitError("expanded table cell count"); const headerRowCount = format === "csv" ? 1 : 0; const projection = projectTableRecords({ columns, headerRowCount, - rows: rows.map((row) => columns.map((column) => structuredCell(row[column]))), + rows: rows.map((row) => + columns.map((column) => + Object.prototype.hasOwnProperty.call(row, column) ? structuredCell(row[column]) : "", + ), + ), }); return [ @@ -2295,23 +2454,31 @@ function projectTableRecords({ mode, rows, sourceRowCount, + tableBudget = { expandedCells: 0, projectedBytes: 0 }, }: { readonly columns: readonly string[]; readonly headerRowCount: number; readonly mode?: TableSemanticMode | undefined; readonly rows: readonly (readonly string[])[]; readonly sourceRowCount?: number | undefined; + readonly tableBudget?: HtmlTableExpansionBudget; }): TableProjection { let width = Math.max(rawColumns.length, 1); for (const row of rows) width = Math.max(width, row.length); - const columnCounts = new Map(); - const columns = Array.from({ length: width }, (_, index) => { - const value = normalizeTableCell(rawColumns[index] ?? ""); - const base = value || `column_${index + 1}`; - const count = (columnCounts.get(base) ?? 0) + 1; - columnCounts.set(base, count); - return count === 1 ? base : `${base}_${count}`; - }); + if (width > parserResourceLimits.maxTableColumns) + throw new ParserResourceLimitError("table column count"); + if (width * rows.length > parserResourceLimits.maxTableCells) + throw new ParserResourceLimitError("expanded table cell count"); + const columns = uniqueColumnNames( + Array.from({ length: width }, (_, index) => normalizeTableCell(rawColumns[index] ?? "")), + ); + const columnBytes = columns.map((column) => Buffer.byteLength(column)); + if (rows.length === 0) { + consumeTableProjectionBytes( + tableBudget, + columnBytes.reduce((total, bytes) => total + bytes, Math.max(0, columns.length - 1) * 3), + ); + } const lines: string[] = []; let matrixCellCount = 0; let numericCellCount = 0; @@ -2319,6 +2486,13 @@ function projectTableRecords({ const cells: string[] = []; for (let index = 0; index < columns.length; index += 1) { const value = normalizeTableCell(row[index] ?? ""); + consumeTableProjectionBytes( + tableBudget, + (columnBytes[index] ?? 0) + + 2 + + Buffer.byteLength(value) + + (index === 0 ? (lines.length > 0 ? 1 : 0) : 3), + ); cells.push(value); if (index === 0 || !value) continue; matrixCellCount += 1; @@ -2374,7 +2548,7 @@ function structuredCell(value: unknown): string { } if (typeof value === "object") { - return JSON.stringify(value); + return stringifyDocumentJson(value); } return String(value); @@ -2384,20 +2558,56 @@ function uniqueStrings(values: readonly string[]): string[] { return [...new Set(values)]; } -function markdownTokensToElements( - tokens: readonly Token[], - { preserveHtmlText }: { readonly preserveHtmlText: boolean }, -): ParseElementInput[] { +function uniqueColumnNames(names: readonly string[]): string[] { + const reserved = new Set(names); + const used = new Set(); + const nextSuffix = new Map(); + return names.map((name, index) => { + const base = name || `column_${index + 1}`; + let candidate = base; + let suffix = nextSuffix.get(base) ?? 2; + while (used.has(candidate) || (candidate !== base && reserved.has(candidate))) { + candidate = `${base}_${suffix++}`; + } + used.add(candidate); + nextSuffix.set(base, suffix); + return candidate; + }); +} + +function markdownTokensToElements(tokens: readonly Token[]): ParseElementInput[] { const elements: ParseElementInput[] = []; const sectionPath: string[] = []; + const tableBudget: HtmlTableExpansionBudget = { expandedCells: 0, projectedBytes: 0 }; + const pending = tokens.map((token) => ({ token, depth: 0 })).reverse(); + let visited = 0; - for (const token of tokens) { + while (pending.length > 0) { + const entry = pending.pop(); + if (!entry) break; + const { token, depth } = entry; + visited += 1; + if (visited > parserResourceLimits.maxNodes || depth > parserResourceLimits.maxDepth) { + throw new ParserResourceLimitError("Markdown token count or depth"); + } if (token.type === "space") { continue; } + if (token.type === "blockquote") { + for (const child of [...(token as Tokens.Blockquote).tokens].reverse()) { + pending.push({ token: child, depth: depth + 1 }); + } + continue; + } + if (token.type === "heading") { const heading = token as Tokens.Heading; + const images = markdownImagesFromToken(heading); + if (images.length > 0 || /<[!\/a-z]/iu.test(heading.text)) { + pushMarkdownHtmlElements(token, elements, sectionPath, images, tableBudget); + continue; + } const text = normalizeText(heading.text); if (!text) { @@ -2421,40 +2631,32 @@ function markdownTokensToElements( if (token.type === "paragraph") { const paragraph = token as Tokens.Paragraph; const images = markdownImagesFromToken(paragraph); - for (const image of images) { - pushImageElement(elements, sectionPath, { - assetRef: { - ...(image.contentType ? { contentType: image.contentType } : {}), - uri: image.uri, - }, - caption: image.alt, - source: "markdown-image", - ...(image.title ? { title: image.title } : {}), - }); + if (images.length > 0 || /<[!\/a-z]/iu.test(paragraph.text)) { + pushMarkdownHtmlElements(token, elements, sectionPath, images, tableBudget); + } else { + pushTextElement(elements, "paragraph", paragraph.text, sectionPath); } - - if (images.length > 0 && normalizeText(paragraph.text).startsWith("![")) { - continue; - } - - pushTextElement(elements, "paragraph", paragraph.text, sectionPath); continue; } - if (token.type === "html" && preserveHtmlText) { - const html = token as Tokens.HTML; - pushTextElement(elements, "paragraph", markdownHtmlBlockText(html.text), sectionPath); + if (token.type === "html") { + pushMarkdownHtmlElements(token, elements, sectionPath, [], tableBudget); continue; } if (token.type === "list") { const list = token as Tokens.List; - pushTextElement( - elements, - "list", - list.items.map((item) => item.text).join("\n"), - sectionPath, - ); + const images = markdownImagesFromToken(list); + if (images.length > 0 || /<[!\/a-z]/iu.test(list.raw)) { + pushMarkdownHtmlElements(token, elements, sectionPath, images, tableBudget); + } else { + pushTextElement( + elements, + "list", + list.items.map((item) => item.text).join("\n"), + sectionPath, + ); + } continue; } @@ -2468,7 +2670,12 @@ function markdownTokensToElements( if (token.type === "table") { const table = token as Tokens.Table; - const projection = markdownTableProjection(table); + const images = markdownImagesFromToken(table); + if (images.length > 0 || /<[!\/a-z]/iu.test(table.raw)) { + pushMarkdownHtmlElements(token, elements, sectionPath, images, tableBudget); + continue; + } + const projection = markdownTableProjection(table, tableBudget); pushTextElement(elements, "table", projection.text, sectionPath, { table: projection.metadata, }); @@ -2478,6 +2685,41 @@ function markdownTokensToElements( return elements; } +function pushMarkdownHtmlElements( + token: Token, + elements: ParseElementInput[], + sectionPath: string[], + images: readonly MarkdownImageRef[], + tableBudget: HtmlTableExpansionBudget, +): void { + // Render syntax to an inert DOM, never evaluate JSX, execute scripts or fetch referenced URLs. + // This keeps text/image order while the same HTML visitor excludes non-searchable subtrees. + const source = + token.type === "html" ? (token as Tokens.HTML).text : marked.parser([token], { async: false }); + const nodes = parseDocument(source).children as HtmlNode[]; + assertHtmlStructureBudget(nodes); + const start = elements.length; + visitHtmlNode({ children: nodes }, elements, sectionPath, tableBudget); + const references = new Set(images.map((image) => image.uri)); + for (let index = start; index < elements.length; index += 1) { + const element = elements[index]; + if (!element) continue; + const assetRef = element.metadata.assetRef as { uri?: string } | undefined; + if (element.type !== "image" || !assetRef?.uri || !references.has(assetRef.uri)) continue; + const alt = metadataString(element.metadata, "alt"); + const title = metadataString(element.metadata, "title"); + elements[index] = { + ...element, + metadata: { + assetRef: cloneMetadata(element.metadata.assetRef as Readonly>), + ...(alt ? { caption: alt } : {}), + source: "markdown-image", + ...(title ? { title } : {}), + }, + }; + } +} + function isMdxInput({ filename, mimeType, @@ -2487,53 +2729,136 @@ function isMdxInput({ ); } -function markdownHtmlBlockText(source: string): string { - const document = parseDocument(source, { - lowerCaseAttributeNames: true, - lowerCaseTags: true, - }); - const nodes = document.children as HtmlNode[]; - - return nodes.map(searchableMarkdownHtmlText).join("\n"); -} - -function searchableMarkdownHtmlText(node: HtmlNode): string { - const name = node.name?.toLowerCase(); - if (name && ["script", "style", "noscript"].includes(name)) { - return ""; - } - - if (!node.children?.length) { - return htmlText(node); - } - - return node.children.map(searchableMarkdownHtmlText).join("\n"); -} - function htmlNodesToElements(nodes: readonly HtmlNode[]): ParseElementInput[] { const elements: ParseElementInput[] = []; const sectionPath: string[] = []; - - for (const node of nodes) { - visitHtmlNode(node, elements, sectionPath); - } - + const tableBudget: HtmlTableExpansionBudget = { expandedCells: 0, projectedBytes: 0 }; + assertHtmlStructureBudget(nodes); + visitHtmlNode({ children: nodes }, elements, sectionPath, tableBudget); return elements; } function htmlDocumentTitle(nodes: readonly HtmlNode[]): string | undefined { - for (const node of nodes) { + const pending = [...nodes].reverse(); + while (pending.length > 0) { + const node = pending.pop(); + if (!node) break; if (node.name?.toLowerCase() === "title") { const title = normalizeText(htmlText(node)); if (title) return Array.from(title).slice(0, defaultMaxDocumentTitleChars).join(""); } - const childTitle = htmlDocumentTitle(node.children ?? []); - if (childTitle) return childTitle; + for (const child of [...(node.children ?? [])].reverse()) pending.push(child); } return undefined; } -function visitHtmlNode(node: HtmlNode, elements: ParseElementInput[], sectionPath: string[]): void { +function assertHtmlStructureBudget(nodes: readonly HtmlNode[]): void { + const pending = nodes.map((node) => ({ node, depth: 0 })); + let count = 0; + while (pending.length > 0) { + const entry = pending.pop(); + if (!entry) break; + const { node, depth } = entry; + count += 1; + if (count > parserResourceLimits.maxNodes || depth > parserResourceLimits.maxDepth) { + throw new ParserResourceLimitError("HTML node count or depth"); + } + for (const child of node.children ?? []) { + pending.push({ node: child, depth: depth + 1 }); + if (pending.length + count > parserResourceLimits.maxNodes) { + throw new ParserResourceLimitError("HTML node count"); + } + } + } +} + +const searchableHtmlBlockNames = new Set([ + "address", + "article", + "aside", + "blockquote", + "body", + "dd", + "div", + "dl", + "dt", + "figure", + "footer", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hr", + "html", + "li", + "main", + "nav", + "ol", + "p", + "pre", + "section", + "table", + "ul", +]); + +function pushHtmlInlineElements( + nodes: readonly HtmlNode[], + elements: ParseElementInput[], + sectionPath: readonly string[], + type: "paragraph" | "list" = "paragraph", + figureCaption?: string, +): void { + const pending: (HtmlNode | string)[] = [...nodes].reverse(); + let text = ""; + const flush = () => { + pushTextElement(elements, type, text, sectionPath); + text = ""; + }; + while (pending.length > 0) { + const node = pending.pop(); + if (node === undefined) break; + if (typeof node === "string") { + text += node; + continue; + } + const name = node.name?.toLowerCase(); + if (name && ["script", "style", "noscript", "title"].includes(name)) continue; + if (name === "figcaption" && figureCaption !== undefined) continue; + if (name === "img") { + flush(); + pushHtmlImageElement( + elements, + node, + sectionPath, + figureCaption || undefined, + figureCaption === undefined ? "html-img" : "html-figure", + ); + } else if (name === "br") { + text += "\n"; + } else if (node.children?.length) { + const block = name && searchableHtmlBlockNames.has(name); + if (block) { + text += "\n"; + pending.push("\n"); + } + for (const child of [...node.children].reverse()) pending.push(child); + } else { + text += htmlText(node); + } + } + flush(); +} + +function visitHtmlNode( + node: HtmlNode, + elements: ParseElementInput[], + sectionPath: string[], + tableBudget: HtmlTableExpansionBudget, +): void { const name = node.name?.toLowerCase(); if (name && ["script", "style", "noscript"].includes(name)) { @@ -2565,16 +2890,20 @@ function visitHtmlNode(node: HtmlNode, elements: ParseElementInput[], sectionPat }); } + for (const image of findHtmlElements(node, "img")) { + pushHtmlImageElement(elements, image, sectionPath, undefined, "html-img"); + } + return; } if (name === "p") { - pushTextElement(elements, "paragraph", htmlText(node), sectionPath); + pushHtmlInlineElements(node.children ?? [], elements, sectionPath); return; } if (name === "ul" || name === "ol") { - pushTextElement(elements, "list", htmlListText(node), sectionPath); + pushHtmlInlineElements(node.children ?? [], elements, sectionPath, "list"); return; } @@ -2584,22 +2913,24 @@ function visitHtmlNode(node: HtmlNode, elements: ParseElementInput[], sectionPat } if (name === "table") { - const projection = htmlTableProjection(node); + const projection = htmlTableProjection(node, tableBudget); pushTextElement(elements, "table", projection.text, sectionPath, { table: projection.metadata, }); + for (const image of findHtmlElements(node, "img")) { + pushHtmlImageElement(elements, image, sectionPath, undefined, "html-img"); + } return; } if (name === "figure") { - const image = firstHtmlImage(node); - if (image) { + if (findHtmlElements(node, "img").length > 0) { const caption = normalizeText( findHtmlElements(node, "figcaption") .map((captionNode) => htmlText(captionNode)) .join(" "), ); - pushHtmlImageElement(elements, image, sectionPath, caption || undefined, "html-figure"); + pushHtmlInlineElements(node.children ?? [], elements, sectionPath, "paragraph", caption); return; } } @@ -2609,9 +2940,21 @@ function visitHtmlNode(node: HtmlNode, elements: ParseElementInput[], sectionPat return; } - for (const child of node.children ?? []) { - visitHtmlNode(child, elements, sectionPath); + if (!node.children?.length) { + pushHtmlInlineElements([node], elements, sectionPath); + return; } + let inline: HtmlNode[] = []; + for (const child of node.children) { + if (child.name && searchableHtmlBlockNames.has(child.name.toLowerCase())) { + pushHtmlInlineElements(inline, elements, sectionPath); + inline = []; + visitHtmlNode(child, elements, sectionPath, tableBudget); + } else { + inline.push(child); + } + } + pushHtmlInlineElements(inline, elements, sectionPath); } function unstructuredElementsToElements( @@ -2620,11 +2963,15 @@ function unstructuredElementsToElements( const elements: ParseElementInput[] = []; const sectionPath: string[] = []; const headingPathsByElementId = new Map(); + const tableBudget: HtmlTableExpansionBudget = { expandedCells: 0, projectedBytes: 0 }; + let sectionPathItems = 0; for (const sourceElement of sourceElements) { const type = unstructuredType(sourceElement.type); const tableProjection = - type === "table" ? unstructuredTableProjection(sourceElement.metadata) : undefined; + type === "table" + ? unstructuredTableProjection(sourceElement.metadata, tableBudget) + : undefined; const providerText = tableProjection?.text ?? normalizeText(sourceElement.text ?? ""); const text = hasChineseOcrLanguage(sourceElement.metadata) ? normalizeChineseOcrText(providerText) @@ -2645,6 +2992,11 @@ function unstructuredElementsToElements( ? [...sectionPath.slice(0, categoryDepth), text] : undefined; const nextPath = parentPath ? [...parentPath, text] : (depthPath ?? [text]); + if (nextPath.length > maxUnstructuredSectionDepth) { + throw new ProviderResponseError( + `Unstructured parser output exceeds maxSectionDepth=${maxUnstructuredSectionDepth}`, + ); + } sectionPath.splice(0, sectionPath.length, ...nextPath); if (sourceElement.element_id) { @@ -2652,6 +3004,12 @@ function unstructuredElementsToElements( } } + sectionPathItems += sectionPath.length; + if (sectionPathItems > maxUnstructuredSectionPathItems) { + throw new ProviderResponseError( + `Unstructured parser output exceeds maxSectionPathItems=${maxUnstructuredSectionPathItems}`, + ); + } elements.push({ metadata: unstructuredParseElementMetadata({ metadata: sourceElement.metadata, @@ -2753,11 +3111,17 @@ function mergeUnstructuredVerticalText( return null; } + if (!Number.isFinite(box.width) || !Number.isFinite(box.height)) { + throw new ProviderResponseError("Unstructured parser returned invalid glyph geometry"); + } + return { box, element, index, pageNumber, text }; }) .filter((glyph): glyph is UnstructuredVerticalGlyph => glyph !== null) .sort(compareUnstructuredVerticalGlyphs); const availableIndexes = new Set(glyphs.map((glyph) => glyph.index)); + const candidateIndex = createUnstructuredGlyphIndex(glyphs); + let candidateComparisons = 0; const mergedByIndex = new Map(); const removedIndexes = new Set(); @@ -2765,27 +3129,39 @@ function mergeUnstructuredVerticalText( if (!availableIndexes.delete(first.index)) { continue; } + candidateIndex.remove(first); const group = [first]; let current = first; while (true) { - const next = glyphs - .filter( - (candidate) => - availableIndexes.has(candidate.index) && - unstructuredVerticalGlyphsAreAdjacent(current, candidate), - ) - .sort( - (left, right) => - verticalGlyphDistance(current, left) - verticalGlyphDistance(current, right), - )[0]; + let next: UnstructuredVerticalGlyph | undefined; + let nextDistance = Number.POSITIVE_INFINITY; + for (const candidate of candidateIndex.candidates(current)) { + candidateComparisons += 1; + if (candidateComparisons > maxUnstructuredVerticalCandidateComparisons) { + throw new ProviderResponseError( + `Unstructured parser layout exceeds maxVerticalCandidateComparisons=${maxUnstructuredVerticalCandidateComparisons}`, + ); + } + if (!unstructuredVerticalGlyphsAreAdjacent(current, candidate)) continue; + const distance = verticalGlyphDistance(current, candidate); + if ( + !next || + distance < nextDistance || + (distance === nextDistance && compareUnstructuredVerticalGlyphs(candidate, next) < 0) + ) { + next = candidate; + nextDistance = distance; + } + } if (!next) { break; } availableIndexes.delete(next.index); + candidateIndex.remove(next); group.push(next); current = next; } @@ -3097,6 +3473,7 @@ function unstructuredParseElementMetadata({ function unstructuredTableProjection( metadata: Readonly>, + tableBudget: HtmlTableExpansionBudget, ): TableProjection | undefined { const textAsHtml = metadataString(metadata, "text_as_html"); if (!textAsHtml) return undefined; @@ -3107,7 +3484,7 @@ function unstructuredTableProjection( const table = (document.children as HtmlNode[]).flatMap((node) => node.name?.toLowerCase() === "table" ? [node] : findHtmlElements(node, "table"), )[0]; - return table ? htmlTableProjection(table) : undefined; + return table ? htmlTableProjection(table, tableBudget) : undefined; } function unstructuredAssetRef( @@ -3278,24 +3655,56 @@ function compactSectionPath(sectionPath: readonly (string | undefined)[]): strin return sectionPath.filter((segment): segment is string => typeof segment === "string"); } -function markdownTableProjection(table: Tokens.Table): TableProjection { +function markdownTableProjection( + table: Tokens.Table, + tableBudget: HtmlTableExpansionBudget, +): TableProjection { return projectTableRecords({ columns: table.header.map((cell) => normalizeText(cell.text)), headerRowCount: 1, rows: table.rows.map((row) => row.map((cell) => normalizeText(cell.text))), + tableBudget, }); } -function htmlListText(node: HtmlNode): string { - return (node.children ?? []) - .filter((child) => child.name?.toLowerCase() === "li") - .map((child) => normalizeText(htmlText(child))) - .filter(Boolean) - .join("\n"); +interface HtmlTableExpansionBudget { + // Created once per document normalization, never stored on a reusable parser adapter. + expandedCells: number; + projectedBytes: number; } -function htmlTableProjection(node: HtmlNode): TableProjection { - const rows = htmlTableRows(node); +function consumeTableProjectionBytes( + tableBudget: HtmlTableExpansionBudget, + addedBytes: number, +): void { + if (tableBudget.projectedBytes + addedBytes > parserResourceLimits.maxOutputBytes) { + throw new ParserResourceLimitError("table projection bytes"); + } + tableBudget.projectedBytes += addedBytes; +} + +function consumeHtmlTableCells(tableBudget: HtmlTableExpansionBudget, addedCells: number): void { + if (tableBudget.expandedCells + addedCells > parserResourceLimits.maxTableCells) { + throw new ParserResourceLimitError("expanded HTML table cell count"); + } + tableBudget.expandedCells += addedCells; +} + +function htmlTableProjection( + node: HtmlNode, + tableBudget: HtmlTableExpansionBudget, +): TableProjection { + const rows = htmlTableRows(node, tableBudget); + let width = 0; + let populatedRowCells = 0; + for (const row of rows) { + width = Math.max(width, row.cells.length); + populatedRowCells += row.cells.length; + } + // Projection scans the rectangular logical table, including short-row padding. Charge only + // the added padding here; source/rowspan cells (including filtered empty rows) were charged + // before assignment, and must not be counted twice. + consumeHtmlTableCells(tableBudget, width * rows.length - populatedRowCells); if (rows.length === 0) { return { metadata: { @@ -3322,12 +3731,19 @@ function htmlTableProjection(node: HtmlNode): TableProjection { headerRowCount: resolvedHeaderRowCount, rows: rows.slice(resolvedHeaderRowCount).map((row) => row.cells), sourceRowCount: rows.length, + tableBudget, }); } - return projectHeaderlessTableRows(rows.map((row) => row.cells)); + return projectHeaderlessTableRows( + rows.map((row) => row.cells), + tableBudget, + ); } -function htmlTableRows(node: HtmlNode): Array<{ +function htmlTableRows( + node: HtmlNode, + tableBudget: HtmlTableExpansionBudget, +): Array<{ readonly cells: readonly string[]; readonly hasHeaderCell: boolean; readonly inHeaderGroup: boolean; @@ -3336,6 +3752,15 @@ function htmlTableRows(node: HtmlNode): Array<{ findHtmlElements(node, "thead").flatMap((header) => findHtmlElements(header, "tr")), ); let activeRowspans = new Map(); + const writeCell = (cells: string[], column: number, value: string) => { + if (column >= parserResourceLimits.maxTableColumns) { + throw new ParserResourceLimitError("HTML table column count"); + } + // Sparse carried rows still reserve every column up to their final cell. + const addedCells = Math.max(0, column + 1 - cells.length); + consumeHtmlTableCells(tableBudget, addedCells); + cells[column] = value; + }; return findHtmlElements(node, "tr") .map((row) => { const sourceCells = (row.children ?? []).filter((cell) => @@ -3350,7 +3775,7 @@ function htmlTableRows(node: HtmlNode): Array<{ const consumeRowspan = () => { const carried = activeRowspans.get(column); if (!carried) return false; - cells[column] = carried.value; + writeCell(cells, column, carried.value); if (carried.remaining > 1) { nextRowspans.set(column, { remaining: carried.remaining - 1, value: carried.value }); } @@ -3369,15 +3794,17 @@ function htmlTableRows(node: HtmlNode): Array<{ while (consumeRowspan()) { // A colspan only occupies columns not already reserved by a rowspan. } - cells[column] = value; + writeCell(cells, column, value); if (rowSpan > 1) { nextRowspans.set(column, { remaining: rowSpan - 1, value }); } column += 1; } } - while (activeRowspans.size > 0) { - if (!consumeRowspan()) column += 1; + // Remaining reservations are ordered by column; skip holes without scanning them. + for (const carriedColumn of activeRowspans.keys()) { + column = carriedColumn; + consumeRowspan(); } activeRowspans = nextRowspans; return { @@ -3390,12 +3817,23 @@ function htmlTableRows(node: HtmlNode): Array<{ } function flattenHtmlTableHeaders(rows: readonly { readonly cells: readonly string[] }[]): string[] { - const width = Math.max(...rows.map((row) => row.cells.length), 1); + let width = 1; + for (const row of rows) width = Math.max(width, row.cells.length); + if (width > parserResourceLimits.maxTableColumns) + throw new ParserResourceLimitError("HTML table column count"); + if (width * rows.length > parserResourceLimits.maxTableCells) + throw new ParserResourceLimitError("expanded HTML table cell count"); + let headerBytes = 0; return Array.from({ length: width }, (_, column) => { const labels: string[] = []; for (const row of rows) { const label = row.cells[column]?.trim(); - if (label && labels.at(-1) !== label) labels.push(label); + if (label && labels.at(-1) !== label) { + headerBytes += Buffer.byteLength(label, "utf8") + (labels.length > 0 ? 3 : 0); + if (headerBytes > parserResourceLimits.maxOutputBytes) + throw new ParserResourceLimitError("HTML table header bytes"); + labels.push(label); + } } return labels.join(" / "); }); @@ -3406,9 +3844,18 @@ function htmlTableCellSpan(cell: HtmlNode, attribute: "colspan" | "rowspan"): nu return Number.isSafeInteger(parsed) && parsed >= 1 && parsed <= 256 ? parsed : 1; } -function projectHeaderlessTableRows(rows: readonly (readonly string[])[]): TableProjection { +function projectHeaderlessTableRows( + rows: readonly (readonly string[])[], + tableBudget: HtmlTableExpansionBudget, +): TableProjection { + let width = 1; + for (const row of rows) width = Math.max(width, row.length); + if (width > parserResourceLimits.maxTableColumns) + throw new ParserResourceLimitError("HTML table column count"); + if (width * rows.length > parserResourceLimits.maxTableCells) + throw new ParserResourceLimitError("expanded HTML table cell count"); if (rows.length === 1) { - return projectTableRecords({ columns: [], headerRowCount: 0, rows }); + return projectTableRecords({ columns: [], headerRowCount: 0, rows, tableBudget }); } const firstRow = rows[0] ?? []; if (looksLikeTableHeader(firstRow, rows.slice(1))) { @@ -3416,6 +3863,7 @@ function projectHeaderlessTableRows(rows: readonly (readonly string[])[]): Table columns: firstRow, headerRowCount: 1, rows: rows.slice(1), + tableBudget, }); } if (looksLikeKeyValueTable(rows)) { @@ -3425,16 +3873,15 @@ function projectHeaderlessTableRows(rows: readonly (readonly string[])[]): Table mode: "single-record", rows: [rows.map((row) => row[1] ?? "")], sourceRowCount: rows.length, + tableBudget, }); } return projectTableRecords({ - columns: Array.from( - { length: Math.max(...rows.map((row) => row.length), 1) }, - (_, index) => `column_${index + 1}`, - ), + columns: Array.from({ length: width }, (_, index) => `column_${index + 1}`), headerRowCount: 0, mode: "record-list", rows, + tableBudget, }); } @@ -3489,49 +3936,47 @@ function tableCellValueKind(value: string): "boolean" | "date" | "number" | "tex } function markdownImagesFromToken(token: Token): MarkdownImageRef[] { - const candidate = token as Token & { - readonly href?: unknown; - readonly text?: unknown; - readonly title?: unknown; - readonly tokens?: readonly Token[]; - }; const images: MarkdownImageRef[] = []; - - if (candidate.type === "image" && typeof candidate.href === "string" && candidate.href.trim()) { - const uri = candidate.href.trim(); - const alt = typeof candidate.text === "string" ? normalizeText(candidate.text) : ""; - const title = typeof candidate.title === "string" ? normalizeText(candidate.title) : ""; - - images.push({ - ...(alt ? { alt } : {}), - ...(title ? { title } : {}), - ...(inferImageContentTypeFromUri(uri) - ? { contentType: inferImageContentTypeFromUri(uri) } - : {}), - uri, - }); - } - - for (const child of candidate.tokens ?? []) { - images.push(...markdownImagesFromToken(child)); - } - - return images; -} - -function firstHtmlImage(node: HtmlNode): HtmlNode | undefined { - if (node.name?.toLowerCase() === "img") { - return node; - } - - for (const child of node.children ?? []) { - const image = firstHtmlImage(child); - if (image) { - return image; + const pending = [{ token, depth: 0 }]; + let count = 0; + while (pending.length > 0) { + const entry = pending.pop(); + if (!entry) break; + const { token: current, depth } = entry; + if (++count > parserResourceLimits.maxNodes || depth > parserResourceLimits.maxDepth) { + throw new ParserResourceLimitError("Markdown inline node count or depth"); + } + const candidate = current as Token & { + readonly href?: unknown; + readonly text?: unknown; + readonly title?: unknown; + readonly tokens?: readonly Token[]; + readonly items?: readonly Token[]; + }; + if (candidate.type === "image" && typeof candidate.href === "string" && candidate.href.trim()) { + const uri = candidate.href.trim(); + const alt = typeof candidate.text === "string" ? normalizeText(candidate.text) : ""; + const title = typeof candidate.title === "string" ? normalizeText(candidate.title) : ""; + images.push({ + ...(alt ? { alt } : {}), + ...(title ? { title } : {}), + ...(inferImageContentTypeFromUri(uri) + ? { contentType: inferImageContentTypeFromUri(uri) } + : {}), + uri, + }); + } + const children = + candidate.type === "table" + ? [...(current as Tokens.Table).header, ...(current as Tokens.Table).rows.flat()].flatMap( + (cell) => cell.tokens, + ) + : (candidate.tokens ?? candidate.items ?? []); + for (const child of [...children].reverse()) { + pending.push({ token: child, depth: depth + 1 }); } } - - return undefined; + return images; } function pushHtmlImageElement( @@ -3607,20 +4052,34 @@ function inferImageContentTypeFromUri(uri: string): string | undefined { function findHtmlElements(node: HtmlNode, name: string): HtmlNode[] { const matches: HtmlNode[] = []; - - if (node.name?.toLowerCase() === name) { - matches.push(node); + const pending = [node]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + const currentName = current.name?.toLowerCase(); + if (currentName && ["script", "style", "noscript"].includes(currentName)) continue; + if (currentName === name) matches.push(current); + for (const child of [...(current.children ?? [])].reverse()) pending.push(child); } - - for (const child of node.children ?? []) { - matches.push(...findHtmlElements(child, name)); - } - return matches; } function htmlText(node: HtmlNode): string { - return DomUtils.textContent(node as never); + const text: string[] = []; + const pending = [node]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + const name = current.name?.toLowerCase(); + if (name && ["script", "style", "noscript"].includes(name)) continue; + if (name === "br") { + text.push("\n"); + continue; + } + if (!current.children?.length) text.push(DomUtils.textContent(current as never)); + else for (const child of [...current.children].reverse()) pending.push(child); + } + return text.join(""); } function htmlHeadingDepth(name: string | undefined): number | null { @@ -3645,7 +4104,7 @@ function inferContentType(elements: readonly ParseElement[]): ParseArtifact["con } function decodeUtf8(bytes: Uint8Array): string { - return new TextDecoder().decode(bytes); + return decodeDocumentText(bytes).text; } function normalizeText(text: string): string { @@ -3811,6 +4270,29 @@ async function fetchWithRetries({ }); } + // The pinned provider wraps its pre-allocation PDF safety rejection as HTTP 500. Classify + // that exact, bounded error before retry admission so neither inline nor durable retries + // repeatedly submit a page that cannot fit. Other failures keep their HTTP semantics. + const resourceRejection = await classifyUnstructuredResourceResponse(response, signal); + if (resourceRejection?.kind === "pdf") { + throw new ProviderInputError( + "PDF page exceeds safe raster limits. Reduce page dimensions before importing.", + ); + } + if (resourceRejection?.kind === "input") { + throw new ProviderInputError( + `Unstructured parser resource limit: ${resourceRejection.reason}`, + ); + } + if (resourceRejection?.kind === "timeout") { + throw new ProviderError("Unstructured isolated worker exceeded its execution deadline", { + code: "provider_timeout", + requestOutcomeAmbiguous: false, + retryable: false, + status: response.status, + }); + } + if (!isRetryableProviderStatus(response.status) || attempt >= maxRetries) { return response; } @@ -3990,6 +4472,7 @@ interface UnstructuredRequestDeadline { function createUnstructuredRequestDeadline(requestTimeoutMs: number): UnstructuredRequestDeadline { const controller = new AbortController(); const timeoutReason = new Error("Unstructured parser request deadline exceeded"); + const expiresAt = performance.now() + requestTimeoutMs; let expired = false; const timer = setTimeout(() => { if (!controller.signal.aborted) { @@ -4004,9 +4487,14 @@ function createUnstructuredRequestDeadline(requestTimeoutMs: number): Unstructur dispose: () => { clearTimeout(timer); }, - expired: () => expired, + expired: () => expired || performance.now() >= expiresAt, throwIfExpired: () => { - if (expired) { + // Synchronous normalization can delay timers. Measure elapsed monotonic time as well, so + // overruns cannot become successful results merely because the timeout callback was late. + // This detects overruns; a hard CPU cancellation boundary still requires a worker process. + if (expired || performance.now() >= expiresAt) { + expired = true; + controller.abort(timeoutReason); throw timeoutReason; } }, diff --git a/knowledge-fs/packages/parsers/src/native-markup-fidelity.test.ts b/knowledge-fs/packages/parsers/src/native-markup-fidelity.test.ts new file mode 100644 index 00000000000..7549d1c71d5 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/native-markup-fidelity.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, it } from "vitest"; + +import { createNativeHtmlParser, createNativeMarkdownParser } from "./index"; + +const input = (body: string, extension: "html" | "md" | "mdx" = "md") => ({ + body: new TextEncoder().encode(body), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: `fixture.${extension}`, + mimeType: extension === "html" ? "text/html" : "text/markdown", + version: 1, +}); + +describe("native markup content fidelity", () => { + it("retains nested blockquotes and their images in section order", async () => { + const artifact = await createNativeMarkdownParser().parse( + input( + "# Rules\n\n> Quoted policy\n>\n> > Nested policy\n>\n> ![Evidence](proof.png)\n\nFinal policy", + ), + ); + expect(artifact.elements.map((element) => element.text)).toEqual([ + "Rules", + "Quoted policy", + "Nested policy", + "Evidence", + "Final policy", + ]); + expect(artifact.elements.every((element) => element.sectionPath[0] === "Rules")).toBe(true); + }); + + it.each(["md", "mdx"] as const)( + "keeps static HTML text in %s without indexing script/style bodies", + async (extension) => { + const artifact = await createNativeMarkdownParser().parse( + input( + "Critical policy", + extension, + ), + ); + expect(artifact.elements.map((element) => element.text).join(" ")).toBe("Critical policy"); + }, + ); + + it("keeps text on both sides of Markdown images in source order without duplicating image syntax", async () => { + const artifact = await createNativeMarkdownParser().parse( + input( + "Before ![Diagram](a.png) after ![Photo](b.png) ending.\n\n![First](c.png) retained suffix.", + ), + ); + expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([ + ["paragraph", "Before"], + ["image", "Diagram"], + ["paragraph", "after"], + ["image", "Photo"], + ["paragraph", "ending."], + ["image", "First"], + ["paragraph", "retained suffix."], + ]); + }); + + it("retains inline HTML text and excludes embedded executable content in Markdown paragraphs", async () => { + const artifact = await createNativeMarkdownParser().parse( + input("Before important after."), + ); + expect(artifact.elements.map((element) => element.text)).toEqual(["Before important after."]); + }); + + it("preserves each placement caption when the same Markdown image URI is reused", async () => { + const artifact = await createNativeMarkdownParser().parse( + input('![First](same.png "One") ![Second](same.png "Two")'), + ); + expect( + artifact.elements.map((element) => [ + element.text, + element.metadata.caption, + element.metadata.title, + ]), + ).toEqual([ + ["First", "First", "One"], + ["Second", "Second", "Two"], + ]); + }); + + it("keeps images nested in Markdown headings and tables", async () => { + const artifact = await createNativeMarkdownParser().parse( + input( + "# Heading ![Badge](badge.png)\n\n| Name | Evidence |\n| --- | --- |\n| Ada | ![Chart](chart.png) |", + ), + ); + expect( + artifact.elements + .filter((element) => element.type === "image") + .map((element) => element.text), + ).toEqual(["Badge", "Chart"]); + expect(artifact.elements[0]).toMatchObject({ + sectionPath: ["Heading"], + text: "Heading", + type: "heading", + }); + }); + + it("retains HTML heading/table images without duplicating normal text", async () => { + const artifact = await createNativeHtmlParser().parse( + input( + '

HeadingBadge

NameEvidence
AdaChart
', + "html", + ), + ); + expect(artifact.elements.map((element) => element.type)).toEqual([ + "heading", + "image", + "table", + "image", + ]); + expect( + artifact.elements + .filter((element) => element.type === "image") + .map((element) => element.text), + ).toEqual(["Badge", "Chart"]); + }); + + it("does not extract images hidden inside excluded HTML subtrees", async () => { + const artifact = await createNativeHtmlParser().parse( + input( + '

Heading

Kept
', + "html", + ), + ); + expect(artifact.elements.map((element) => element.type)).toEqual(["heading", "table"]); + expect(artifact.elements.map((element) => element.text).join(" ")).not.toContain("hidden"); + }); + + it("rejects excessive HTML node counts even when the tree is shallow", async () => { + await expect( + createNativeHtmlParser().parse( + input(`
${"".repeat(250_001)}
`, "html"), + ), + ).rejects.toMatchObject({ + code: "provider_input", + name: "ParserResourceLimitError", + retryable: false, + }); + }); + + it("extracts images inside Markdown lists and links without losing surrounding list content", async () => { + const artifact = await createNativeMarkdownParser().parse( + input("- Before [![Linked](a.png)](https://example.test) after\n- Last item"), + ); + expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([ + ["list", "Before"], + ["image", "Linked"], + ["list", "after\nLast item"], + ]); + }); + + it("retains direct container text and nested inline images without duplicate text", async () => { + const artifact = await createNativeHtmlParser().parse( + input( + '

Guide

Bare text

Before Figure after

Trailing text
', + "html", + ), + ); + expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([ + ["heading", "Guide"], + ["paragraph", "Bare text"], + ["paragraph", "Before"], + ["image", "Figure"], + ["paragraph", "after"], + ["paragraph", "Trailing text"], + ]); + expect(artifact.elements.every((element) => element.sectionPath[0] === "Guide")).toBe(true); + }); + + it("retains list images, inline spacing, and line breaks without indexing scripts", async () => { + const artifact = await createNativeHtmlParser().parse( + input( + '
  • BeforeFigureafter
  • Last
    line
', + "html", + ), + ); + expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([ + ["list", "Before"], + ["image", "Figure"], + ["list", "after\nLast\nline"], + ]); + }); + + it("preserves all figure images and captions instead of selecting only the first image", async () => { + const artifact = await createNativeHtmlParser().parse( + input( + '
Shared caption
', + "html", + ), + ); + expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([ + ["image", "Shared caption"], + ["image", "Shared caption"], + ]); + }); + + it("rejects overly deep HTML with a non-retryable resource error instead of overflowing the stack", async () => { + await expect( + createNativeHtmlParser().parse( + input(`${"
".repeat(5_000)}text${"
".repeat(5_000)}`, "html"), + ), + ).rejects.toMatchObject({ + code: "provider_input", + name: "ParserResourceLimitError", + retryable: false, + }); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/office-admission-integration.test.ts b/knowledge-fs/packages/parsers/src/office-admission-integration.test.ts new file mode 100644 index 00000000000..6802f978fbc --- /dev/null +++ b/knowledge-fs/packages/parsers/src/office-admission-integration.test.ts @@ -0,0 +1,91 @@ +import { strToU8, zipSync } from "fflate"; +import { describe, expect, it, vi } from "vitest"; + +import { createUnstructuredParserClient } from "./index"; + +const identity = { documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", version: 1 }; +const spreadsheetMime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + +describe("Office resource admission at the provider boundary", () => { + it("rejects tiny sparse workbooks before the provider or retries and releases the admission slot", async () => { + const fetch = vi.fn(async () => Response.json([{ type: "NarrativeText", text: "retained" }])); + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.invalid", + fetch, + maxRetries: 3, + maxConcurrency: 1, + }); + const rejected = { + ...identity, + filename: "sparse.xlsx", + mimeType: spreadsheetMime, + body: zipSync({ + "xl/worksheets/sheet1.xml": strToU8( + '1', + ), + }), + }; + await expect(parser.parse(rejected)).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + requestOutcomeAmbiguous: false, + }); + expect(fetch).not.toHaveBeenCalled(); + const artifact = await parser.parse({ + ...rejected, + version: 2, + body: zipSync({ + "xl/worksheets/sheet1.xml": strToU8( + '1', + ), + }), + }); + expect(artifact.elements[0]?.text).toBe("retained"); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it.each(["docx", "pptx", "odt", "epub"])( + "rejects oversized XML structure in %s before sending it", + async (extension) => { + const fetch = vi.fn(async () => Response.json([])); + const parser = createUnstructuredParserClient({ endpoint: "https://parser.invalid", fetch }); + await expect( + parser.parse({ + ...identity, + filename: `deep.${extension}`, + mimeType: "application/octet-stream", + body: zipSync({ + "content.xml": strToU8("".repeat(129) + "".repeat(129)), + }), + }), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + expect(fetch).not.toHaveBeenCalled(); + }, + ); + + it("keeps ordinary archive bytes and provider extraction options unchanged", async () => { + const body = zipSync({ + "word/document.xml": strToU8( + 'hello', + ), + }); + const fetch = vi.fn(async (request: RequestInfo | URL) => { + if (!(request instanceof Request)) throw new Error("Expected Request"); + const form = await request.formData(); + const file = form.get("files"); + if (!(file instanceof File)) throw new Error("Expected file"); + expect(new Uint8Array(await file.arrayBuffer())).toEqual(body); + expect(form.get("strategy")).toBe("auto"); + return Response.json([{ type: "NarrativeText", text: "hello" }]); + }); + const parser = createUnstructuredParserClient({ endpoint: "https://parser.invalid", fetch }); + const artifact = await parser.parse({ + ...identity, + body, + filename: "ordinary.docx", + mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + }); + expect(artifact.elements[0]?.text).toBe("hello"); + expect(fetch).toHaveBeenCalledOnce(); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/office-parser-preflight.test.ts b/knowledge-fs/packages/parsers/src/office-parser-preflight.test.ts new file mode 100644 index 00000000000..547c0b6e5c6 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/office-parser-preflight.test.ts @@ -0,0 +1,596 @@ +import { Zip, ZipDeflate, ZipPassThrough, strToU8, zipSync } from "fflate"; +import { describe, expect, it, vi } from "vitest"; + +import { OfficeArchiveAdmissionError, assertOfficeArchiveSafe } from "./office-parser-preflight"; + +const spreadsheetMime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + +function spreadsheetInput(xml: string, extraEntries: Record = {}) { + return { + body: zipSync({ + "[Content_Types].xml": strToU8( + '', + ), + "xl/workbook.xml": strToU8( + '', + ), + "xl/worksheets/sheet1.xml": strToU8(xml), + ...extraEntries, + }), + filename: "workbook.xlsx", + mimeType: spreadsheetMime, + }; +} + +function sheetXml(contents: string, dimension = "A1:B2"): string { + return `${contents}`; +} + +describe("assertOfficeArchiveSafe", () => { + it("rejects a tiny sparse workbook before its rectangular cell span can be materialized", async () => { + const input = spreadsheetInput( + sheetXml( + 'firstlast', + "A1:XFD1048576", + ), + ); + expect(input.body.byteLength).toBeLessThan(2000); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow(OfficeArchiveAdmissionError); + }); + + it("preserves an ordinary spreadsheet without changing source bytes", async () => { + const input = spreadsheetInput( + sheetXml( + 'Name5', + ), + ); + const original = input.body.slice(); + await expect(assertOfficeArchiveSafe(input)).resolves.toBeUndefined(); + expect(input.body).toEqual(original); + }); + + it.each(["xl/alternate.xml", "custom/sheet-part"])( + "detects worksheet content even if relationships select an unconventional part %s", + async (part) => { + const input = spreadsheetInput(sheetXml(""), { + [part]: strToU8( + sheetXml('1', "A1:XFD90000"), + ), + }); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("dense-cell budget"); + }, + ); + + it("does not let leading whitespace hide a renamed XML worksheet", async () => { + const input = spreadsheetInput(sheetXml(""), { + "custom/sheet-part": strToU8(" ".repeat(4096) + sheetXml("", "A1:XFD1048576")), + }); + // Store the member without compression so its prefix spans several admission chunks. + input.body = zipSync( + { + "xl/workbook.xml": strToU8(""), + "custom/sheet-part": strToU8(" ".repeat(4096) + sheetXml("", "A1:XFD1048576")), + }, + { level: 0 }, + ); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("dense-cell budget"); + }); + + it("checks Office contents even when both filename and MIME type are misleading", async () => { + const input = spreadsheetInput(sheetXml("", "A1:XFD1048576")); + await expect( + assertOfficeArchiveSafe({ ...input, filename: "plain.txt", mimeType: "text/plain" }), + ).rejects.toThrow(OfficeArchiveAdmissionError); + }); + + it.each([ + [ + "document.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "word/document.xml", + "document", + ], + [ + "slides.pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "ppt/presentation.xml", + "presentation", + ], + ["document.odt", "application/vnd.oasis.opendocument.text", "content.xml", "document-content"], + ["book.epub", "application/epub+zip", "META-INF/container.xml", "container"], + ])("accepts bounded ordinary %s archives", async (filename, mimeType, part, root) => { + const body = zipSync({ + [part]: strToU8(`<${root}>Ordinary content & text`), + }); + await expect(assertOfficeArchiveSafe({ body, filename, mimeType })).resolves.toBeUndefined(); + }); + + it("preserves an EPUB's harmless HTML doctype", async () => { + await expect( + assertOfficeArchiveSafe({ + body: zipSync({ + "META-INF/container.xml": strToU8(""), + "OPS/chapter.xhtml": strToU8( + '

Chapter

', + ), + }), + filename: "book.epub", + mimeType: "application/epub+zip", + }), + ).resolves.toBeUndefined(); + }); + + it.each([ + ["-//W3C//DTD XHTML 1.1//EN", "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"], + ["-//W3C//DTD XHTML 1.0 Strict//EN", "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"], + [ + "-//W3C//DTD XHTML 1.0 Transitional//EN", + "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd", + ], + ])( + "preserves the standard EPUB XHTML PUBLIC doctype %s without resolving it", + async (publicId, systemId) => { + const body = zipSync({ + "META-INF/container.xml": strToU8(""), + "OPS/chapter.xhtml": strToU8( + `

Chapter

`, + ), + }); + await expect( + assertOfficeArchiveSafe({ body, filename: "book.epub", mimeType: "application/epub+zip" }), + ).resolves.toBeUndefined(); + }, + ); + + it.each([ + '', + ']>', + '', + ])("does not admit custom or internally extended XHTML DTDs", async (doctype) => { + const body = zipSync({ + "META-INF/container.xml": strToU8(""), + "OPS/chapter.xhtml": strToU8(`${doctype}`), + }); + await expect( + assertOfficeArchiveSafe({ body, filename: "book.epub", mimeType: "application/epub+zip" }), + ).rejects.toThrow("external entities"); + }); + + it.each([ + [ + '', + '', + "safe internal", + ], + [ + '', + '', + "external", + ], + ['', "", "missing"], + ['', "", "ambiguous"], + [ + '', + '', + "escapes", + ], + ])("rejects unsafe worksheet relationships", async (sheets, relations, message) => { + const input = spreadsheetInput(sheetXml(""), { + "xl/workbook.xml": strToU8( + `${sheets}`, + ), + "xl/_rels/workbook.xml.rels": strToU8(`${relations}`), + }); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow(message); + }); + + it("bounds logical sheet count rather than only ZIP part count", async () => { + const input = spreadsheetInput(sheetXml(""), { + "xl/workbook.xml": strToU8( + '', + ), + }); + await expect(assertOfficeArchiveSafe(input, { maxWorksheets: 1 })).rejects.toThrow( + "too many worksheets", + ); + }); + + it("keeps column formatting indices inside the format limit", async () => { + await expect( + assertOfficeArchiveSafe( + spreadsheetInput(''), + ), + ).rejects.toThrow("column formatting"); + }); + + it.each(["utf16le", "utf16be"])( + "recognizes %s encoded spreadsheet dimensions", + async (encoding) => { + const bytes = Buffer.from( + `\ufeff${sheetXml("", "A1:XFD1048576")}`, + "utf16le", + ); + if (encoding === "utf16be") bytes.swap16(); + const input = spreadsheetInput(sheetXml(""), { "xl/worksheets/sheet1.xml": bytes }); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("dense-cell budget"); + }, + ); + + it("rejects a UTF-32 part instead of misreading its NUL-padded tags as unimportant XML", async () => { + const xml = sheetXml("", "A1:XFD1048576"); + const bytes = Uint8Array.from( + [...xml].flatMap((character) => [character.charCodeAt(0), 0, 0, 0]), + ); + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml(""), { "custom/worksheet": bytes })), + ).rejects.toThrow(OfficeArchiveAdmissionError); + }); + + it.each(["utf16le", "utf16be"])( + "recognizes BOM-less %s XML with a whitespace prefix", + async (encoding) => { + const bytes = Buffer.from(` \n${sheetXml("", "A1:XFD1048576")}`, "utf16le"); + if (encoding === "utf16be") bytes.swap16(); + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml(""), { "custom/worksheet": bytes })), + ).rejects.toThrow(OfficeArchiveAdmissionError); + }, + ); + + it("handles inferred coordinates and legal column format bounds", async () => { + const xml = + ''; + await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).resolves.toBeUndefined(); + }); + + it("admits a long narrow table inside the dense-cell budget", async () => { + const input = spreadsheetInput( + sheetXml( + '12', + "A1:A90000", + ), + ); + await expect(assertOfficeArchiveSafe(input)).resolves.toBeUndefined(); + }); + + it("does not treat a default column style as populated spreadsheet columns", async () => { + const xml = + '1'; + await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).resolves.toBeUndefined(); + }); + + it.each([ + ['', "positive coordinate"], + ['', "positive coordinate"], + ['', "positive coordinate"], + ['', "coordinates disagree"], + ['', "outside a row"], + ['', "nested rows"], + ['', "increasing"], + ['', "increasing"], + ])("rejects unsafe or ambiguous spreadsheet coordinates: %s", async (contents, message) => { + await expect(assertOfficeArchiveSafe(spreadsheetInput(sheetXml(contents)))).rejects.toThrow( + message, + ); + }); + + it.each(["A0", "A1:B2:C3", "a1", "AAAA1", "A1 B2"])( + "rejects invalid worksheet dimension %s", + async (dimension) => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("", dimension))), + ).rejects.toThrow("invalid"); + }, + ); + + it("bounds inferred coordinates when row and cell references are omitted", async () => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { + maxSheetCellSpan: 4, + }), + ).rejects.toThrow("dense-cell budget"); + }); + + it("bounds rectangular area accumulated across individually safe sheets", async () => { + const input = spreadsheetInput(sheetXml("", "A1:C3"), { + "xl/worksheets/sheet2.xml": strToU8(sheetXml("", "A1:C3")), + }); + await expect( + assertOfficeArchiveSafe(input, { maxSheetCellSpan: 10, maxWorkbookCellSpan: 15 }), + ).rejects.toThrow("workbook exceeds"); + }); + + it("counts worksheet relationships that load the same physical part more than once", async () => { + const input = spreadsheetInput(sheetXml("", "A1:C3"), { + "xl/workbook.xml": strToU8( + '', + ), + "xl/_rels/workbook.xml.rels": strToU8( + '', + ), + }); + await expect(assertOfficeArchiveSafe(input, { maxWorkbookCellSpan: 15 })).rejects.toThrow( + "workbook exceeds", + ); + await expect( + assertOfficeArchiveSafe(input, { maxWorkbookCellSpan: 20 }), + ).resolves.toBeUndefined(); + }); + + it("bounds actual cell count independently of occupied area", async () => { + await expect( + assertOfficeArchiveSafe( + spreadsheetInput(sheetXml('', "A1:C1")), + { maxWorkbookCells: 2 }, + ), + ).rejects.toThrow("too many cells"); + }); + + it("bounds shared string count including rich strings", async () => { + const input = spreadsheetInput(sheetXml(""), { + "xl/sharedStrings.xml": strToU8("ab"), + }); + await expect(assertOfficeArchiveSafe(input, { maxSharedStrings: 1 })).rejects.toThrow( + "shared strings", + ); + }); + + it.each([ + ']>&payload;', + '', + ])("rejects DTDs without resolving entities or fetching URLs", async (xml) => { + await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).rejects.toThrow( + "external entities", + ); + }); + + it("bounds XML depth", async () => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(""), { + maxXmlDepth: 3, + }), + ).rejects.toThrow("structural complexity"); + }); + + it.each([ + ``, + ` `attr${index}="x"`).join(" ")}/>`, + ])("bounds XML attributes", async (xml) => { + await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).rejects.toThrow( + "attributes exceed", + ); + }); + + it("retains bounded binary media without attempting UTF-8 or XML parsing", async () => { + const input = spreadsheetInput(sheetXml(""), { + "xl/media/image.png": Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0xff, 0x00]), + "custom/readme": strToU8(" not XML"), + "custom/small": strToU8("a"), + "custom/empty": new Uint8Array(0), + }); + await expect(assertOfficeArchiveSafe(input)).resolves.toBeUndefined(); + }); + + it("rejects a source that has damaged UTF-8 in a declared XML member", async () => { + await expect( + assertOfficeArchiveSafe( + spreadsheetInput(sheetXml(""), { + "custom/data.xml": Uint8Array.from([0xff, 0x00, 0x81, 0x81]), + }), + ), + ).rejects.toThrow("cannot be inspected safely"); + }); + + it("does not permit local sizes to differ from central-directory sizes", async () => { + const input = spreadsheetInput(sheetXml("")); + new DataView(input.body.buffer).setUint32(22, 1, true); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("local entries disagree"); + }); + + it("bounds actual inflation even when both size headers understate a member", async () => { + const input = { + body: zipSync({ "xl/workbook.xml": strToU8(`${"a".repeat(4000)}`) }), + filename: "book.xlsx", + mimeType: spreadsheetMime, + }; + const central = Buffer.from(input.body).indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + const view = new DataView(input.body.buffer); + view.setUint32(22, 10, true); + view.setUint32(central + 24, 10, true); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("actual expansion"); + }); + + it("detects overstated actual member sizes", async () => { + const input = { + body: zipSync({ "xl/workbook.xml": strToU8("") }), + filename: "book.xlsx", + mimeType: spreadsheetMime, + }; + const central = Buffer.from(input.body).indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + const view = new DataView(input.body.buffer); + view.setUint32(22, 20, true); + view.setUint32(central + 24, 20, true); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("actual entry size"); + }); + + it("rejects unsupported ZIP compression before starting a decoder", async () => { + const input = spreadsheetInput(sheetXml("")); + const central = Buffer.from(input.body).indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + new DataView(input.body.buffer).setUint16(central + 10, 99, true); + await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("compression is unsupported"); + }); + + it("bounds XML nodes across parts", async () => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxXmlNodes: 2 }), + ).rejects.toThrow("structural complexity"); + }); + + it.each(["", "", "", ""])( + "rejects incomplete or structurally ambiguous XML %s", + async (xml) => { + await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).rejects.toThrow( + OfficeArchiveAdmissionError, + ); + }, + ); + + it("rejects unsupported XML encodings explicitly", async () => { + await expect( + assertOfficeArchiveSafe( + spreadsheetInput(''), + ), + ).rejects.toThrow("encoding is unsupported"); + }); + + it("bounds per-part declared XML bytes before decompression", async () => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxXmlEntryBytes: 20 }), + ).rejects.toThrow("XML part"); + }); + + it("bounds aggregate actual XML bytes", async () => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxXmlBytes: 200 }), + ).rejects.toThrow("XML expansion"); + }); + + it("bounds archive bytes including non-XML media", async () => { + const input = spreadsheetInput(sheetXml(""), { "xl/media/image.png": new Uint8Array(600) }); + await expect(assertOfficeArchiveSafe(input, { maxArchiveBytes: 700 })).rejects.toThrow( + "actual expansion", + ); + }); + + it("bounds entries before allocating extracted data", async () => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxEntries: 2 }), + ).rejects.toThrow("too many entries"); + }); + + it.each([ + "../outside.xml", + "word/../outside.xml", + "word\\document.xml", + "/absolute.xml", + "nul\0.xml", + "word/./document.xml", + ])("rejects ambiguous archive path %s", async (name) => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml(""), { [name]: strToU8("") })), + ).rejects.toThrow("entry names"); + }); + + it("rejects case-ambiguous part names", async () => { + await expect( + assertOfficeArchiveSafe( + spreadsheetInput(sheetXml(""), { "XL/WORKBOOK.XML": strToU8("") }), + ), + ).rejects.toThrow("entry names"); + }); + + it.each([ZipDeflate, ZipPassThrough])( + "accepts lawful streamed ZIP data descriptors (%s)", + async (Member) => { + const chunks: Uint8Array[] = []; + const archive = new Zip((error, chunk) => { + if (error) throw error; + chunks.push(chunk); + }); + const member = new Member("xl/worksheets/sheet1.xml"); + archive.add(member); + const xml = strToU8(sheetXml('1')); + member.push(xml.subarray(0, 15), false); + member.push(xml.subarray(15), true); + archive.end(); + await expect( + assertOfficeArchiveSafe({ + body: Buffer.concat(chunks), + filename: "streamed.xlsx", + mimeType: spreadsheetMime, + }), + ).resolves.toBeUndefined(); + }, + ); + + it("retains cancellation reason without wrapping it as an input failure", async () => { + const controller = new AbortController(); + const reason = new Error("lease lost"); + controller.abort(reason); + await expect( + assertOfficeArchiveSafe({ ...spreadsheetInput(sheetXml("")), signal: controller.signal }), + ).rejects.toBe(reason); + }); + + it("yields while inspecting so cancellation can stop further decompression", async () => { + const controller = new AbortController(); + const reason = new Error("client disconnected"); + const body = zipSync( + { + "xl/workbook.xml": strToU8(""), + "xl/media/image.png": new Uint8Array(128 * 1024), + }, + { level: 0 }, + ); + const timer = setTimeout(() => controller.abort(reason), 0); + try { + await expect( + assertOfficeArchiveSafe({ + body, + filename: "large.xlsx", + mimeType: spreadsheetMime, + signal: controller.signal, + }), + ).rejects.toBe(reason); + } finally { + clearTimeout(timer); + } + }); + + it("bounds inspection time independently of provider timeouts", async () => { + const input = spreadsheetInput(sheetXml("")); + const clock = vi.spyOn(Date, "now").mockReturnValueOnce(0).mockReturnValue(2); + try { + await expect(assertOfficeArchiveSafe(input, { timeoutMs: 1 })).rejects.toThrow( + "processing time", + ); + } finally { + clock.mockRestore(); + } + }); + + it("ignores non-Office inputs and unrelated ZIP archives", async () => { + await expect( + assertOfficeArchiveSafe({ + body: strToU8("hello"), + filename: "note.txt", + mimeType: "text/plain", + }), + ).resolves.toBeUndefined(); + await expect( + assertOfficeArchiveSafe({ + body: zipSync({ "notes.txt": strToU8("hello") }), + filename: "archive.zip", + mimeType: "application/zip", + }), + ).resolves.toBeUndefined(); + }); + + it.each([new Uint8Array([1, 2, 3]), new Uint8Array([0x50, 0x4b]), zipSync({})])( + "rejects Office containers that cannot be inspected", + async (body) => { + await expect( + assertOfficeArchiveSafe({ body, filename: "broken.xlsx", mimeType: spreadsheetMime }), + ).rejects.toThrow(OfficeArchiveAdmissionError); + }, + ); + + it.each([0, -1, 1.5, Number.NaN])( + "rejects invalid programmer-supplied limits %s", + async (value) => { + await expect( + assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxEntries: value }), + ).rejects.toThrow("positive safe integer"); + }, + ); +}); diff --git a/knowledge-fs/packages/parsers/src/office-parser-preflight.ts b/knowledge-fs/packages/parsers/src/office-parser-preflight.ts new file mode 100644 index 00000000000..82f0132b6ed --- /dev/null +++ b/knowledge-fs/packages/parsers/src/office-parser-preflight.ts @@ -0,0 +1,588 @@ +import { Unzip, type UnzipFileInfo, UnzipInflate, unzipSync } from "fflate"; +import { Parser } from "htmlparser2"; + +import { classifyUnstructuredWorkload } from "./unstructured-workload-policy"; + +export interface OfficeArchivePreflightInput { + readonly body: Uint8Array; + readonly filename: string; + readonly mimeType: string; + readonly signal?: AbortSignal; +} + +export interface OfficeArchivePreflightLimits { + readonly maxArchiveBytes: number; + readonly maxEntries: number; + readonly maxXmlBytes: number; + readonly maxXmlEntryBytes: number; + readonly maxXmlDepth: number; + readonly maxXmlNodes: number; + readonly maxSheetRows: number; + readonly maxSheetColumns: number; + readonly maxSheetCellSpan: number; + readonly maxWorkbookCellSpan: number; + readonly maxWorkbookCells: number; + readonly maxSharedStrings: number; + readonly maxWorksheets: number; + readonly timeoutMs: number; +} + +/** + * The cell-span caps bound the *dense* Pandas/NetworkX work in the pinned provider. They are not + * Excel format limits. A very sparse or unusually large valid workbook can intentionally fail + * admission; no cells are silently discarded and the original document is never rewritten. + */ +export const defaultOfficeArchivePreflightLimits: OfficeArchivePreflightLimits = Object.freeze({ + maxArchiveBytes: 512 * 1024 * 1024, + maxEntries: 4096, + maxXmlBytes: 64 * 1024 * 1024, + maxXmlEntryBytes: 16 * 1024 * 1024, + maxXmlDepth: 128, + maxXmlNodes: 1_000_000, + maxSheetRows: 100_000, + maxSheetColumns: 16_384, + maxSheetCellSpan: 250_000, + maxWorkbookCellSpan: 500_000, + maxWorkbookCells: 500_000, + maxSharedStrings: 200_000, + maxWorksheets: 256, + timeoutMs: 30_000, +}); + +export class OfficeArchiveAdmissionError extends Error { + readonly code = "office_archive_input"; + + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "OfficeArchiveAdmissionError"; + } +} + +interface ArchiveBudget { + bytes: number; + cells: number; + sharedStrings: number; + workbookCellSpan: number; + xmlBytes: number; + xmlNodes: number; + worksheets: number; + readonly worksheetAreas: Map; + readonly worksheetReferences: { readonly source: string; readonly id: string }[]; + readonly worksheetRelationships: Map>; +} + +const compressedChunkBytes = 1024; +const cooperativeYieldBytes = 64 * 1024; +const maxXmlAttributes = 128; +const maxXmlAttributeChars = 16 * 1024; + +/** + * Resource admission for OOXML/ODF/EPUB; not a document parser or a general-purpose ZIP validator. + * The existing bounded classifier establishes a classic central-directory bound. fflate reads + * that directory without inflating, then streams actual members in small compressed chunks. + * Local names/sizes must match the directory and actual expansion is counted before XML parsing. + * No expanded archive or XML DOM is retained, and no external entities/resources are loaded. + */ +export async function assertOfficeArchiveSafe( + input: OfficeArchivePreflightInput, + options: Partial = {}, +): Promise { + const claimedOffice = claimsOfficeArchive(input); + if (!claimedOffice && !hasZipSignature(input.body)) return; + input.signal?.throwIfAborted(); + const limits = { ...defaultOfficeArchivePreflightLimits, ...options }; + for (const [name, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`Office archive preflight ${name} must be a positive safe integer`); + } + } + + const start = Date.now(); + const assertActive = () => { + input.signal?.throwIfAborted(); + if (Date.now() - start > limits.timeoutMs) { + reject("Office archive inspection exceeded its bounded processing time"); + } + }; + + try { + const classification = classifyUnstructuredWorkload(input); + if (classification.kind === "rejected" || classification.reason === "archive-invalid") { + // Non-Office archives retain their existing router policy. The outer classifier still + // rejects proven ZIP expansion hazards, independently of this Office-specific check. + if (!claimedOffice) return; + reject("Office archive metadata is unsafe or cannot be inspected safely"); + } + const directory = new Map(); + unzipSync(input.body, { + filter: (entry) => { + if (directory.size >= limits.maxEntries) reject("Office archive has too many entries"); + if (directory.has(entry.name)) reject("Office archive contains duplicate entry names"); + directory.set(entry.name, entry); + return false; + }, + }); + if (!claimedOffice && ![...directory.keys()].some(isOfficePart)) return; + if (directory.size === 0) reject("Office archive contains no inspectable entries"); + + const normalizedNames = new Set(); + for (const entry of directory.values()) { + assertActive(); + const normalizedName = entry.name.toLowerCase(); + if (!safeArchivePath(entry.name) || normalizedNames.has(normalizedName)) { + reject("Office archive contains ambiguous or unsafe entry names"); + } + normalizedNames.add(normalizedName); + if (entry.compression !== 0 && entry.compression !== 8) { + reject("Office archive compression is unsupported for safe inspection"); + } + if (entry.originalSize > limits.maxArchiveBytes) + reject("Office archive expansion is too large"); + if (isXmlPart(entry.name) && entry.originalSize > limits.maxXmlEntryBytes) { + reject("Office XML part exceeds the inspection byte limit"); + } + } + + const budget: ArchiveBudget = { + bytes: 0, + cells: 0, + sharedStrings: 0, + workbookCellSpan: 0, + xmlBytes: 0, + xmlNodes: 0, + worksheets: 0, + worksheetAreas: new Map(), + worksheetReferences: [], + worksheetRelationships: new Map(), + }; + const observed = new Set(); + const completed = new Set(); + const unzip = new Unzip((file) => { + assertActive(); + const expected = directory.get(file.name); + if ( + !expected || + observed.has(file.name) || + file.compression !== expected.compression || + (file.size !== undefined && file.size !== expected.size) || + (file.originalSize !== undefined && file.originalSize !== expected.originalSize) + ) { + reject("Office archive local entries disagree with its central directory"); + } + observed.add(file.name); + let entryBytes = 0; + // OPC relationships may target XML parts without a conventional path or extension. Sniff + // each member's prefix while streaming; never let a renamed worksheet bypass cell budgets. + const xml = createXmlInspector(file.name, limits, budget); + file.ondata = (error, chunk, final) => { + assertActive(); + if (error) throw error; + entryBytes += chunk.byteLength; + budget.bytes += chunk.byteLength; + if (entryBytes > expected.originalSize || budget.bytes > limits.maxArchiveBytes) { + reject("Office archive actual expansion exceeds its declared size or byte limit"); + } + xml.push(chunk, final); + if (final) { + if (entryBytes !== expected.originalSize) { + reject("Office archive actual entry size disagrees with its central directory"); + } + completed.add(file.name); + } + }; + file.start(); + }); + unzip.register(UnzipInflate); + let lastYieldAt = start; + for (let offset = 0; offset < input.body.byteLength; offset += compressedChunkBytes) { + assertActive(); + const end = Math.min(input.body.byteLength, offset + compressedChunkBytes); + unzip.push(input.body.subarray(offset, end), end === input.body.byteLength); + if (end % cooperativeYieldBytes === 0 || Date.now() - lastYieldAt >= 8) { + await new Promise((resolve) => setTimeout(resolve, 0)); + lastYieldAt = Date.now(); + } + } + assertActive(); + if (observed.size !== directory.size || completed.size !== directory.size) { + reject("Office archive has incomplete or inconsistent local entries"); + } + let referencedCellSpan = 0; + for (const reference of budget.worksheetReferences) { + const target = budget.worksheetRelationships.get(reference.source)?.get(reference.id); + const area = target === undefined ? undefined : budget.worksheetAreas.get(target); + if (area === undefined) + reject("Spreadsheet worksheet relationship is missing or unsupported"); + referencedCellSpan += area; + if (referencedCellSpan > limits.maxWorkbookCellSpan) { + reject( + "Spreadsheet workbook exceeds the safe dense-cell budget through worksheet references", + ); + } + } + } catch (error) { + if (input.signal?.aborted) throw input.signal.reason; + if (error instanceof OfficeArchiveAdmissionError) throw error; + throw new OfficeArchiveAdmissionError( + "Office archive is invalid or cannot be inspected safely", + { + cause: error, + }, + ); + } +} + +function claimsOfficeArchive(input: OfficeArchivePreflightInput): boolean { + const filename = input.filename.trim().toLowerCase(); + const mime = input.mimeType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + return ( + /\.(?:docx|pptx|xlsx|odt|epub)$/u.test(filename) || + /(?:wordprocessingml|presentationml|spreadsheetml)/u.test(mime) || + mime === "application/vnd.oasis.opendocument.text" || + mime === "application/epub+zip" + ); +} + +function hasZipSignature(body: Uint8Array): boolean { + return body[0] === 0x50 && body[1] === 0x4b; +} + +function isOfficePart(name: string): boolean { + return /^(?:word\/document\.xml|ppt\/presentation\.xml|xl\/workbook\.xml|xl\/worksheets\/[^/]+\.xml|meta-inf\/container\.xml|content\.xml)$/iu.test( + name, + ); +} + +function isXmlPart(name: string): boolean { + return /\.(?:xml|rels|opf|ncx|xhtml)$/iu.test(name); +} + +function safeArchivePath(name: string): boolean { + return ( + !name.startsWith("/") && + !name.includes("\\") && + !name.includes("\0") && + !name.split("/").some((part) => part === ".." || part === ".") + ); +} + +function reject(message: string): never { + throw new OfficeArchiveAdmissionError(message); +} + +function localXmlName(name: string): string { + return name.slice(name.lastIndexOf(":") + 1); +} + +function isAllowedHtmlDoctype(path: string, declaration: string): boolean { + if (!/\.(?:xhtml|html|htm)$/iu.test(path)) return false; + if (/^!DOCTYPE\s+html\s*$/iu.test(declaration)) return true; + // EPUB 2 uses these standard XHTML declarations. They identify vocabulary only: this SAX + // inspector never resolves DTDs, and EPUB chapter content is passed to Pandoc's HTML reader. + // Full matching deliberately excludes internal subsets, entities, and custom external DTDs. + const match = /^!DOCTYPE\s+html\s+PUBLIC\s+(["'])([^"']+)\1\s+(["'])([^"']+)\3\s*$/iu.exec( + declaration, + ); + if (!match) return false; + const standardDtds: Readonly> = { + "-//W3C//DTD XHTML 1.0 Strict//EN": "www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd", + "-//W3C//DTD XHTML 1.0 Transitional//EN": "www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd", + "-//W3C//DTD XHTML 1.0 Frameset//EN": "www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd", + "-//W3C//DTD XHTML 1.1//EN": "www.w3.org/TR/xhtml11/DTD/xhtml11.dtd", + }; + const expected = standardDtds[match[2] ?? ""]; + return ( + expected !== undefined && + /^https?:\/\//u.test(match[4] ?? "") && + match[4]?.replace(/^https?:\/\//u, "") === expected + ); +} + +function createXmlInspector( + path: string, + limits: OfficeArchivePreflightLimits, + budget: ArchiveBudget, +): { push(chunk: Uint8Array, final: boolean): void } { + let worksheet = /^xl\/worksheets\/[^/]+\.xml$/iu.test(path); + let sharedStrings = /^xl\/sharedstrings\.xml$/iu.test(path); + let workbook = false; + const relationshipSource = sourcePartForRelationshipFile(path); + let activeXml = isXmlPart(path); + let ignoredBinary = false; + let memberBytes = 0; + let countedXmlBytes = 0; + let decoder: TextDecoder | undefined; + let pendingPrefix = new Uint8Array(0); + let depth = 0; + let roots = 0; + let attributeCount = 0; + let lastOpenEndIndex = -1; + let row = 0; + let column = 0; + let maxRow = 0; + let maxColumn = 0; + let inRow = false; + + const useCoordinate = (nextRow: number, nextColumn: number) => { + maxRow = Math.max(maxRow, nextRow); + maxColumn = Math.max(maxColumn, nextColumn); + if ( + maxRow > limits.maxSheetRows || + maxColumn > limits.maxSheetColumns || + maxRow * maxColumn > limits.maxSheetCellSpan + ) { + reject("Spreadsheet worksheet dimensions exceed the safe dense-cell budget"); + } + }; + const useRange = (value: string) => { + const parts = value.split(":"); + if (parts.length > 2) reject("Spreadsheet contains an invalid cell range"); + for (const part of parts) { + const coordinate = cellCoordinate(part); + useCoordinate(coordinate.row, coordinate.column); + } + }; + + const parser = new Parser( + { + onopentagname() { + depth += 1; + if (depth === 1) roots += 1; + budget.xmlNodes += 1; + attributeCount = 0; + if (depth > limits.maxXmlDepth || budget.xmlNodes > limits.maxXmlNodes || roots > 1) { + reject("Office XML exceeds its structural complexity budget"); + } + }, + onattribute(_name, value) { + attributeCount += 1; + if (attributeCount > maxXmlAttributes || value.length > maxXmlAttributeChars) { + reject("Office XML attributes exceed their inspection budget"); + } + }, + onprocessinginstruction(name, data) { + if (name.startsWith("!") && !isAllowedHtmlDoctype(path, data)) { + reject("Office XML declarations and external entities are not allowed"); + } + if (name === "?xml") { + const encoding = /\bencoding\s*=\s*["']([^"']+)["']/iu.exec(data)?.[1]; + if (encoding && !/^(?:utf-?8|utf-?16(?:le|be)?)$/iu.test(encoding)) { + reject("Office XML encoding is unsupported for safe inspection"); + } + } + }, + onopentag(name, attributes) { + lastOpenEndIndex = parser.endIndex; + const tag = localXmlName(name); + if (depth === 1) { + worksheet ||= tag === "worksheet"; + sharedStrings ||= tag === "sst"; + workbook = tag === "workbook"; + } + if (workbook && tag === "sheet") { + budget.worksheets += 1; + if (budget.worksheets > limits.maxWorksheets) + reject("Spreadsheet has too many worksheets"); + const ids = Object.entries(attributes).filter(([key]) => localXmlName(key) === "id"); + if (ids.length > 1) reject("Spreadsheet worksheet references are ambiguous"); + const id = ids[0]?.[1]; + if (id) budget.worksheetReferences.push({ id, source: path }); + } + if ( + relationshipSource !== undefined && + tag === "Relationship" && + attributes.Type?.endsWith("/relationships/worksheet") + ) { + const id = attributes.Id; + const target = attributes.Target; + if (!id || !target || attributes.TargetMode?.toLowerCase() === "external") { + reject("Spreadsheet worksheet relationship is missing or external"); + } + const resolved = resolveInternalPartTarget(relationshipSource, target); + const relations = + budget.worksheetRelationships.get(relationshipSource) ?? new Map(); + if (relations.has(id) || relations.size >= limits.maxWorksheets) { + reject("Spreadsheet worksheet relationships are ambiguous or excessive"); + } + relations.set(id, resolved); + budget.worksheetRelationships.set(relationshipSource, relations); + } + if (sharedStrings && tag === "si") { + budget.sharedStrings += 1; + if (budget.sharedStrings > limits.maxSharedStrings) { + reject("Spreadsheet shared strings exceed the safe item budget"); + } + } + if (!worksheet) return; + if ((tag === "dimension" || tag === "mergeCell") && attributes.ref !== undefined) { + useRange(attributes.ref); + } else if (tag === "row") { + if (inRow) reject("Spreadsheet contains nested rows"); + const nextRow = attributes.r === undefined ? row + 1 : positiveXmlInteger(attributes.r); + if (nextRow <= row) reject("Spreadsheet row coordinates must be strictly increasing"); + row = nextRow; + inRow = true; + column = 0; + useCoordinate(row, 0); + } else if (tag === "c") { + if (!inRow) reject("Spreadsheet contains a cell outside a row"); + budget.cells += 1; + if (budget.cells > limits.maxWorkbookCells) reject("Spreadsheet has too many cells"); + if (attributes.r === undefined) { + column += 1; + } else { + const coordinate = cellCoordinate(attributes.r); + if (coordinate.row !== row) reject("Spreadsheet cell and row coordinates disagree"); + if (coordinate.column <= column) + reject("Spreadsheet cell coordinates must be strictly increasing"); + column = coordinate.column; + } + useCoordinate(row, column); + } else if (tag === "col") { + // Column formatting does not make otherwise empty columns part of Pandas' data frame. + // Validate its indices without increasing the occupied rectangular cell span. + for (const value of [attributes.min, attributes.max]) { + if (value !== undefined && positiveXmlInteger(value) > limits.maxSheetColumns) { + reject("Spreadsheet column formatting exceeds the safe column budget"); + } + } + } + }, + onclosetag(name, implied) { + if (implied && parser.endIndex !== lastOpenEndIndex) { + reject("Office XML contains unbalanced elements"); + } + if (worksheet && localXmlName(name) === "row") inRow = false; + depth -= 1; + }, + }, + { decodeEntities: true, xmlMode: true }, + ); + + return { + push(chunk, final) { + if (ignoredBinary) return; + memberBytes += chunk.byteLength; + let bytes = chunk; + if (!decoder) { + if (pendingPrefix.byteLength > 0) { + bytes = new Uint8Array(pendingPrefix.byteLength + chunk.byteLength); + bytes.set(pendingPrefix); + bytes.set(chunk, pendingPrefix.byteLength); + } + if (bytes.byteLength < 4 && !final) { + pendingPrefix = bytes.slice(); + return; + } + pendingPrefix = new Uint8Array(0); + if (!activeXml && !couldBeXmlPrefix(bytes)) { + ignoredBinary = true; + return; + } + const encoding = + (bytes[0] === 0xff && bytes[1] === 0xfe) || (isAsciiXmlPrefix(bytes[0]) && bytes[1] === 0) + ? "utf-16le" + : (bytes[0] === 0xfe && bytes[1] === 0xff) || + (bytes[0] === 0 && isAsciiXmlPrefix(bytes[1])) + ? "utf-16be" + : "utf-8"; + decoder = new TextDecoder(encoding, { fatal: true }); + } + let text: string; + try { + text = decoder.decode(bytes, { stream: !final }); + } catch (error) { + if (activeXml) throw error; + ignoredBinary = true; + return; + } + if (!activeXml) { + text = text.replace(/^[\t\r\n ]+/u, ""); + if (!text) return; + if (!text.startsWith("<")) { + ignoredBinary = true; + return; + } + activeXml = true; + } + budget.xmlBytes += memberBytes - countedXmlBytes; + countedXmlBytes = memberBytes; + if (memberBytes > limits.maxXmlEntryBytes || budget.xmlBytes > limits.maxXmlBytes) { + reject("Office XML expansion exceeds the inspection byte limit"); + } + if (text.includes("\0")) + reject("Office XML contains an unsupported encoding or NUL character"); + parser.write(text); + if (final) { + if (depth !== 0 || roots !== 1) + reject("Office XML is incomplete or has no document element"); + parser.end(); + if (worksheet) { + budget.worksheetAreas.set(path, maxRow * maxColumn); + budget.workbookCellSpan += maxRow * maxColumn; + if (budget.workbookCellSpan > limits.maxWorkbookCellSpan) { + reject("Spreadsheet workbook exceeds the safe dense-cell budget"); + } + } + } + }, + }; +} + +function couldBeXmlPrefix(bytes: Uint8Array): boolean { + if ( + (bytes[0] === 0xff && bytes[1] === 0xfe) || + (bytes[0] === 0xfe && bytes[1] === 0xff) || + (bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) || + (bytes[0] === 0 && isAsciiXmlPrefix(bytes[1])) || + (isAsciiXmlPrefix(bytes[0]) && bytes[1] === 0) + ) + return true; + for (const byte of bytes) { + if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) continue; + return byte === 0x3c; + } + return bytes.byteLength > 0; +} + +function isAsciiXmlPrefix(byte: number | undefined): boolean { + return byte === 0x3c || byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d; +} + +function sourcePartForRelationshipFile(path: string): string | undefined { + const match = /^(.*\/)?_rels\/([^/]+)\.rels$/u.exec(path); + return match ? `${match[1] ?? ""}${match[2]}` : undefined; +} + +function resolveInternalPartTarget(source: string, target: string): string { + if (!target || /[\\\0?#]/u.test(target) || /^[a-z][a-z0-9+.-]*:/iu.test(target)) + reject("Spreadsheet worksheet target is not a safe internal part"); + const segments = target.startsWith("/") ? [] : source.split("/").slice(0, -1); + for (const segment of target.split("/")) { + if (!segment || segment === ".") continue; + if (segment === "..") { + if (segments.length === 0) reject("Spreadsheet worksheet target escapes the archive"); + segments.pop(); + } else segments.push(segment); + } + const resolved = segments.join("/"); + if (!resolved) reject("Spreadsheet worksheet target is empty"); + return resolved; +} + +function positiveXmlInteger(value: string): number { + if (!/^\d+$/u.test(value)) reject("Spreadsheet contains an invalid positive coordinate"); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1) { + reject("Spreadsheet contains an invalid positive coordinate"); + } + return parsed; +} + +function cellCoordinate(value: string): { row: number; column: number } { + const match = /^\$?([A-Z]{1,3})\$?(\d+)$/u.exec(value); + if (!match) reject("Spreadsheet contains an invalid cell coordinate"); + let column = 0; + for (const letter of match[1] as string) column = column * 26 + letter.charCodeAt(0) - 64; + return { column, row: positiveXmlInteger(match[2] as string) }; +} diff --git a/knowledge-fs/packages/parsers/src/parse-coverage.test.ts b/knowledge-fs/packages/parsers/src/parse-coverage.test.ts new file mode 100644 index 00000000000..da6fd74e573 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/parse-coverage.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { createArchiveMediaReportCollector } from "./parse-coverage"; + +describe("bounded parse coverage reports", () => { + it("bounds diagnostic references and explicitly reports truncation", () => { + const report = createArchiveMediaReportCollector(true); + report.skip("x".repeat(1025), "unsupported-media-format"); + for (let index = 0; index < 4096; index += 1) + report.skip(`word/media/${index}.svg`, "unsupported-media-format"); + const metadata = report.metadata(); + expect(metadata.archiveMediaReport.skippedResources).toHaveLength(4096); + expect(metadata.archiveMediaReport.skippedResources[0]?.archivePath).toHaveLength(1024); + expect(metadata.archiveMediaReport.omittedReferences).toBe(1); + expect(metadata.parseCoverage.media.reasons).toContain("resource-reference-truncated"); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/parse-coverage.ts b/knowledge-fs/packages/parsers/src/parse-coverage.ts new file mode 100644 index 00000000000..8d744e68cd4 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/parse-coverage.ts @@ -0,0 +1,38 @@ +export interface ParseCoveragePart { + readonly status: "complete" | "partial" | "unknown" | "not-requested"; + readonly reasons: readonly string[]; +} + +/** Diagnostic references are bounded independently from decoded images and never inline bytes. */ +export function createArchiveMediaReportCollector(requiresImages: boolean | undefined) { + const skippedResources: { archivePath: string; reason: string }[] = []; + const reasons = new Set(); + let omittedReferences = 0; + let observedResources = 0; + const maxDiagnosticReferences = 4096; + return { + observe() { + observedResources += 1; + }, + skip(archivePath: string, reason: string) { + reasons.add(reason); + if (skippedResources.length < maxDiagnosticReferences) { + skippedResources.push({ archivePath: archivePath.slice(0, 1024), reason }); + if (archivePath.length > 1024) reasons.add("resource-reference-truncated"); + } else omittedReferences += 1; + }, + metadata() { + const unknown = { reasons: ["provider-coverage-not-reported"], status: "unknown" } as const; + const media: ParseCoveragePart = + requiresImages === false + ? { reasons: ["image-extraction-disabled"], status: "not-requested" } + : reasons.size > 0 + ? { reasons: [...reasons].sort(), status: "partial" } + : unknown; + return { + archiveMediaReport: { observedResources, omittedReferences, skippedResources }, + parseCoverage: { media, tables: unknown, text: unknown }, + }; + }, + }; +} diff --git a/knowledge-fs/packages/parsers/src/parser-resource-budget.test.ts b/knowledge-fs/packages/parsers/src/parser-resource-budget.test.ts new file mode 100644 index 00000000000..48898a14b5a --- /dev/null +++ b/knowledge-fs/packages/parsers/src/parser-resource-budget.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "vitest"; + +import { + ParserResourceLimitError, + assertJsonDepth, + assertParserResourceBudget, +} from "./parser-resource-budget"; + +describe("parser resource budgets", () => { + it("bounds nested JSON before a recursive decoder sees it", () => { + expect(() => assertJsonDepth(`${"[".repeat(129)}0${"]".repeat(129)}`)).toThrow( + ParserResourceLimitError, + ); + expect(() => assertJsonDepth('{"text":"[[[\\"{}"}')).not.toThrow(); + }); + it("accepts the nesting boundary", () => { + expect(() => assertJsonDepth(`${"[".repeat(128)}0${"]".repeat(128)}`)).not.toThrow(); + }); + it("counts escaped strings and object keys against a byte budget", () => { + const value = { key: '\u0000\n\\"中文\ud800' }; + const bytes = Buffer.byteLength(JSON.stringify(value)); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes })).not.toThrow(); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes - 1 })).toThrow( + ParserResourceLimitError, + ); + }); + it("rejects cycles and excessive nodes without recursive traversal", () => { + const cycle: Record = {}; + cycle.self = cycle; + expect(() => assertParserResourceBudget(cycle)).toThrow(ParserResourceLimitError); + expect(() => assertParserResourceBudget([1, 2, 3], { maxNodes: 3 })).toThrow( + ParserResourceLimitError, + ); + }); + it("counts repeated references as their expanded serialization", () => { + const child = { text: "sample" }; + expect(() => assertParserResourceBudget([child, child], { maxBytes: 20 })).toThrow( + ParserResourceLimitError, + ); + }); + it("preserves cancellation", () => { + const reason = new Error("cancelled"); + expect(() => assertJsonDepth("{}", AbortSignal.abort(reason))).toThrow(reason); + expect(() => assertParserResourceBudget({}, { signal: AbortSignal.abort(reason) })).toThrow( + reason, + ); + }); + + it.each([ + null, + true, + false, + 0, + -123, + "", + "plain ASCII", + 'quote: " slash: \\', + "\b\t\n\f\r\u0000\u001f", + "中文😀\u2028\u2029", + "\ud800", + "\udc00", + "\ud800\ud800\udc00\udc00", + [], + [null, true, "😀", 12], + { '\u0000"\\😀': [1, "\ud800", {}] }, + ])("counts exact compact JSON bytes for %j", (value) => { + const bytes = Buffer.byteLength(JSON.stringify(value)); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes })).not.toThrow(); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes - 1 })).toThrow( + ParserResourceLimitError, + ); + }); + + it("does not count inherited prototype fields as serialized document data", () => { + const value = Object.assign(Object.create({ inherited: "ignored" }), { own: "keep" }); + const bytes = Buffer.byteLength(JSON.stringify(value)); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes })).not.toThrow(); + }); + + it("accepts the node boundary and rejects its next item", () => { + expect(() => assertParserResourceBudget([1, 2], { maxNodes: 3 })).not.toThrow(); + expect(() => assertParserResourceBudget([1, 2, 3], { maxNodes: 3 })).toThrow( + ParserResourceLimitError, + ); + }); + + it("counts repeated aliases by serialized expansion rather than unique identity", () => { + const child = { value: ["shared", 42] }; + const value = [child, child, child]; + const bytes = Buffer.byteLength(JSON.stringify(value)); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes })).not.toThrow(); + expect(() => assertParserResourceBudget(value, { maxBytes: bytes - 1 })).toThrow( + ParserResourceLimitError, + ); + }); + + it("stops traversing a wide object once its node budget is exhausted", () => { + let reads = 0; + const value: Record = {}; + for (let index = 0; index < 100; index += 1) { + Object.defineProperty(value, String(index), { + enumerable: true, + get: () => { + reads += 1; + return "data"; + }, + }); + } + expect(() => assertParserResourceBudget(value, { maxNodes: 3 })).toThrow( + ParserResourceLimitError, + ); + expect(reads).toBeLessThanOrEqual(3); + }); + + it("handles escaped quotes and backslashes without treating text as JSON structure", () => { + const source = JSON.stringify({ text: '"\\'.repeat(150) + "[".repeat(300) }); + expect(() => assertJsonDepth(source)).not.toThrow(); + }); + + it.each([Number.NaN, Number.POSITIVE_INFINITY, -1, 0, 1.5])( + "rejects an invalid node budget instead of disabling it: %s", + (maxNodes) => { + expect(() => assertParserResourceBudget(null, { maxNodes })).toThrow(); + }, + ); + + it.each([Number.NaN, Number.POSITIVE_INFINITY, -1, 0, 1.5])( + "rejects an invalid byte budget instead of disabling it: %s", + (maxBytes) => { + expect(() => assertParserResourceBudget(null, { maxBytes })).toThrow(); + }, + ); +}); diff --git a/knowledge-fs/packages/parsers/src/parser-resource-budget.ts b/knowledge-fs/packages/parsers/src/parser-resource-budget.ts new file mode 100644 index 00000000000..ad30247989a --- /dev/null +++ b/knowledge-fs/packages/parsers/src/parser-resource-budget.ts @@ -0,0 +1,109 @@ +import { Buffer } from "node:buffer"; + +export const parserResourceLimits = Object.freeze({ + maxDepth: 128, + maxNodes: 250_000, + maxOutputBytes: 32 * 1024 * 1024, + maxRawElements: 50_000, + maxArtifactNodes: 1_000_000, + maxTableCells: 500_000, + maxTableColumns: 4_096, +}); + +/** Stable input classification without a dependency on the parser facade. */ +export class ParserResourceLimitError extends Error { + readonly code = "provider_input"; + readonly retryable = false; + constructor(limit: string) { + super( + `Document parser resource limit exceeded: ${limit}. Reduce the document structure or split the input.`, + ); + this.name = "ParserResourceLimitError"; + } +} + +/** A non-recursive, allocation-free structural pass before recursive JSON decoding. */ +export function assertJsonDepth(text: string, signal?: AbortSignal): void { + signal?.throwIfAborted(); + let depth = 0; + let quoted = false; + let escaped = false; + for (let index = 0; index < text.length; index += 1) { + if ((index & 4095) === 0) signal?.throwIfAborted(); + const character = text[index]; + if (quoted) { + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') quoted = false; + } else if (character === '"') quoted = true; + else if (character === "[" || character === "{") { + depth += 1; + if (depth > parserResourceLimits.maxDepth) + throw new ParserResourceLimitError("JSON nesting depth"); + } else if (character === "]" || character === "}") depth -= 1; + } +} + +export function assertParserResourceBudget( + value: unknown, + options: { + readonly maxBytes?: number; + readonly maxNodes?: number; + readonly signal?: AbortSignal | undefined; + } = {}, +): void { + options.signal?.throwIfAborted(); + const maxBytes = options.maxBytes ?? parserResourceLimits.maxOutputBytes; + const maxNodes = options.maxNodes ?? parserResourceLimits.maxNodes; + if ( + !Number.isSafeInteger(maxBytes) || + maxBytes < 1 || + !Number.isSafeInteger(maxNodes) || + maxNodes < 1 + ) { + throw new ParserResourceLimitError("invalid resource budget"); + } + const pending = [{ value, depth: 0 }]; + let bytes = 0; + let nodes = 0; + while (pending.length > 0) { + const entry = pending.pop(); + if (!entry) break; + nodes += 1; + if ((nodes & 4095) === 0) options.signal?.throwIfAborted(); + if (nodes > maxNodes || entry.depth > parserResourceLimits.maxDepth) { + throw new ParserResourceLimitError("expanded node count or depth"); + } + const current = entry.value; + if (typeof current === "string") bytes += serializedStringBytes(current); + else if (current !== null && typeof current === "object") { + bytes += 2; + let count = 0; + for (const key in current) { + if (!Object.prototype.hasOwnProperty.call(current, key)) continue; + if (count > 0) bytes += 1; + if (!Array.isArray(current)) bytes += serializedStringBytes(key) + 1; + pending.push({ value: (current as Record)[key], depth: entry.depth + 1 }); + count += 1; + if (nodes + pending.length > maxNodes || bytes > maxBytes) + throw new ParserResourceLimitError("expanded node count or bytes"); + } + } else bytes += String(current).length; + if (bytes > maxBytes) throw new ParserResourceLimitError("expanded output bytes"); + } +} + +function serializedStringBytes(value: string): number { + let bytes = Buffer.byteLength(value, "utf8") + 2; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code === 34 || code === 92) bytes += 1; + else if (code < 32) bytes += [8, 9, 10, 12, 13].includes(code) ? 1 : 5; + else if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (next >= 0xdc00 && next <= 0xdfff) index += 1; + else bytes += 3; + } else if (code >= 0xdc00 && code <= 0xdfff) bytes += 3; + } + return bytes; +} diff --git a/knowledge-fs/packages/parsers/src/parser.test.ts b/knowledge-fs/packages/parsers/src/parser.test.ts index fb3c0e706ce..8fb94c4e3a4 100644 --- a/knowledge-fs/packages/parsers/src/parser.test.ts +++ b/knowledge-fs/packages/parsers/src/parser.test.ts @@ -1,5 +1,5 @@ -import { zipSync } from "fflate"; -import { describe, expect, it } from "vitest"; +import { unzipSync, zipSync } from "fflate"; +import { describe, expect, it, vi } from "vitest"; import { ProviderInputError, @@ -28,7 +28,37 @@ function compactPdf(pageCount: number): Uint8Array { } function compactDocx(): Uint8Array { - return zipSync({ "word/document.xml": textBytes("") }); + return zipSync({ + "[Content_Types].xml": textBytes( + '', + ), + "_rels/.rels": textBytes( + '', + ), + "word/document.xml": textBytes( + 'Ordinary document', + ), + }); +} + +function compactXlsx(): Uint8Array { + return zipSync({ + "[Content_Types].xml": textBytes( + '', + ), + "_rels/.rels": textBytes( + '', + ), + "xl/workbook.xml": textBytes( + '', + ), + "xl/_rels/workbook.xml.rels": textBytes( + '', + ), + "xl/worksheets/sheet1.xml": textBytes( + 'Ordinary sheet', + ), + }); } function utf8Length(text: string): number { @@ -144,7 +174,7 @@ describe("parser adapters", () => { metadata: { filename: "architecture.md", mimeType: "text/markdown", - parserVersion: "native-markdown@2", + parserVersion: "native-markdown@4", }, parser: "native-markdown", version: 1, @@ -152,7 +182,7 @@ describe("parser adapters", () => { // Locks the byte-compatible digest while the implementation hashes incrementally to avoid a // second whole-document allocation. expect(artifact.artifactHash).toBe( - "509d7ac5f29bab6e0c579da62f18ea120e5952529c052183bdd0188793c2af98", + "cc85d1d5ee391b775dca15620b06d9fd116aa7888ce99595e1f772d3b8a0cd02", ); expect(artifact.elements).toEqual([ { @@ -237,11 +267,11 @@ describe("parser adapters", () => { "Overview", "MDX keeps this searchable.\nNested detail", ]); - expect(artifact.metadata.parserVersion).toBe("native-mdx@2"); + expect(artifact.metadata.parserVersion).toBe("native-mdx@4"); }, ); - it("keeps plain Markdown raw HTML behavior while using the table-aware parser version", async () => { + it("preserves static text inside ordinary Markdown HTML blocks", async () => { const parser = createNativeMarkdownParser({ generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c96", now: () => createdAt, @@ -255,8 +285,10 @@ describe("parser adapters", () => { }), ); - expect(artifact.elements).toEqual([]); - expect(artifact.metadata.parserVersion).toBe("native-markdown@2"); + expect(artifact.elements.map((element) => element.text)).toEqual([ + "Plain Markdown keeps its existing behavior.", + ]); + expect(artifact.metadata.parserVersion).toBe("native-markdown@4"); }); it("preserves the schema of a Markdown table that has no data records", async () => { @@ -356,7 +388,7 @@ describe("parser adapters", () => { expect(artifact.parser).toBe("native-html"); expect(artifact.metadata.documentTitle).toBe("Ignored Title"); - expect(artifact.metadata.parserVersion).toBe("native-html@3"); + expect(artifact.metadata.parserVersion).toBe("native-html@5"); expect(artifact.elements.map((element) => element.type)).toEqual([ "heading", "paragraph", @@ -1026,14 +1058,16 @@ describe("parser adapters", () => { }), parserHints: { language: "en", layoutComplexity: "simple" }, }); - await router.parse({ - ...createParseInput({ - body: "# too large", - filename: "large.md", - mimeType: "text/markdown", + await expect( + router.parse({ + ...createParseInput({ + body: "# too large", + filename: "large.md", + mimeType: "text/markdown", + }), + parserHints: { language: "en", layoutComplexity: "simple" }, }), - parserHints: { language: "en", layoutComplexity: "simple" }, - }); + ).rejects.toMatchObject({ code: "provider_input" }); await router.parse({ ...createParseInput({ body: "# scan", @@ -1059,13 +1093,7 @@ describe("parser adapters", () => { parserHints: { language: "ja" }, }); - expect(selected).toEqual([ - "markdown", - "unstructured", - "unstructured", - "unstructured", - "unstructured", - ]); + expect(selected).toEqual(["markdown", "unstructured", "unstructured", "unstructured"]); }); it("parses native structured data formats into structured artifacts", async () => { @@ -1120,7 +1148,7 @@ describe("parser adapters", () => { metadata: { filename: "scores.csv", mimeType: "text/csv", - parserVersion: "native-structured@2", + parserVersion: "native-structured@4", }, parser: "native-structured", }); @@ -1199,7 +1227,7 @@ describe("parser adapters", () => { elements: [ { metadata: { format: "xml", rootType: "object" }, - text: '{\n "record": {\n "name": "Ada",\n "score": 10\n }\n}', + text: '{\n "record": {\n "name": "Ada",\n "score": "10"\n }\n}', type: "code", }, ], @@ -1306,8 +1334,8 @@ describe("parser adapters", () => { }), ), ).resolves.toMatchObject({ - metadata: { routeReason: "native-size-limit", routedParser: "unstructured" }, - parser: "unstructured", + metadata: { routeReason: "structured-file-type", routedParser: "native-structured" }, + parser: "native-structured", }); }); @@ -1330,7 +1358,9 @@ describe("parser adapters", () => { await expect( router.parse( createParseInput({ - body: "first line\nsecond line", + body: filename.endsWith(".vtt") + ? "WEBVTT\n\n00:00.000 --> 00:01.000\nfirst line\nsecond line" + : "first line\nsecond line", filename, mimeType, }), @@ -1359,7 +1389,7 @@ describe("parser adapters", () => { mimeType: "application/json", }), ), - ).rejects.toThrow("Structured parser returned an invalid response"); + ).rejects.toThrow("Structured parser input is malformed"); }); it("maps Unstructured API responses into parse artifacts", async () => { @@ -1443,7 +1473,7 @@ describe("parser adapters", () => { metadata: { filename: "report.pdf", mimeType: "application/pdf", - parserVersion: "unstructured@10", + parserVersion: "unstructured@12", }, parser: "unstructured", version: 1, @@ -1556,7 +1586,7 @@ describe("parser adapters", () => { "column_1: 施工条件 | column_2: 临时用电", "随着道路工程推进", ]); - expect(artifact.metadata.parserVersion).toBe("unstructured@10"); + expect(artifact.metadata.parserVersion).toBe("unstructured@12"); }); it("projects an Unstructured spreadsheet table into independently retrievable records", async () => { @@ -1582,7 +1612,7 @@ describe("parser adapters", () => { }); const artifact = await parser.parse({ - body: new Uint8Array([1, 2, 3]), + body: compactXlsx(), documentAssetId, filename: "issues.xlsx", mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -1636,7 +1666,7 @@ describe("parser adapters", () => { now: () => createdAt, }); const artifact = await parser.parse({ - body: new Uint8Array([1, 2, 3]), + body: compactXlsx(), documentAssetId, filename: "multi-sheet.xlsx", mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", @@ -1688,7 +1718,7 @@ describe("parser adapters", () => { version: 1, }), ).resolves.toMatchObject({ - metadata: { parserVersion: "unstructured@10" }, + metadata: { parserVersion: "unstructured@12" }, parser: "unstructured", }); }, @@ -1841,7 +1871,6 @@ describe("parser adapters", () => { async (filename, mimeType, archivePath) => { const body = zipSync( { - "../outside.png": new Uint8Array([9, 9, 9]), "metadata/readme.txt": textBytes("not an image"), [archivePath]: new Uint8Array([1, 2, 3, 4]), }, @@ -1898,13 +1927,27 @@ describe("parser adapters", () => { sectionPath: [], type: "image", }); - expect(artifact.elements).not.toEqual( - expect.arrayContaining([ - expect.objectContaining({ - metadata: expect.objectContaining({ archivePath: "../outside.png" }), - }), - ]), - ); + }, + ); + + it.each(["docx", "pptx", "xlsx", "odt", "epub"])( + "rejects unsafe %s archive member paths before calling the provider", + async (extension) => { + const fetch = vi.fn(async () => new Response("[]")); + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch, + }); + await expect( + parser.parse({ + body: zipSync({ "../outside.png": new Uint8Array([9, 9, 9]) }), + documentAssetId, + filename: `unsafe.${extension}`, + mimeType: "application/octet-stream", + version: 1, + }), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + expect(fetch).not.toHaveBeenCalled(); }, ); @@ -2310,194 +2353,227 @@ describe("parser adapters", () => { }); }); - it("matches worksheet anchors without trusting malformed or external OOXML relationships", async () => { - const body = zipSync( - { - "xl/_rels/workbook.xml.rels": textBytes( - [ - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - "", - ].join(""), - ), - "xl/drawings/_rels/drawing1.xml.rels": textBytes( - [ - '', - '', - '', - '', - '', - "", - ].join(""), - ), - "xl/drawings/drawing1.xml": textBytes( - [ - '', - spreadsheetImageAnchorXml({ - column: 2, - kind: "oneCellAnchor", - relationshipId: "rIdImage", - row: 1, - }), - spreadsheetImageAnchorXml({ column: 3, relationshipId: "rIdBlank", row: 2 }), - spreadsheetImageAnchorXml({ - column: 2, - kind: "oneCellAnchor", - relationshipId: "rIdImage", - row: 1, - }), - spreadsheetImageAnchorXml({ column: -1, relationshipId: "rIdImage", row: 1 }), - spreadsheetImageAnchorXml({ column: 2, relationshipId: "missing", row: 1 }), - spreadsheetImageAnchorXml({ column: 2, relationshipId: "rIdWrong", row: 1 }), - spreadsheetImageAnchorXml({ column: 2, relationshipId: "rIdText", row: 1 }), - "21", - "", - ].join(""), - ), - "xl/media/image1.png": new Uint8Array([1, 2, 3, 4]), - "xl/media/blank.png": new Uint8Array([5, 6, 7, 8]), - "xl/media/orphan.png": new Uint8Array([9, 10, 11, 12]), - "xl/workbook.xml": textBytes( - [ - '', - '', - '', - '', - '', - '', - '', - "", - ].join(""), - ), - "xl/worksheets/_rels/sheet1.xml.rels": textBytes( - [ - '', - '', - '', - "", - ].join(""), - ), - "xl/worksheets/sheet1.xml": textBytes( - [ - '', - 'header', - 'record', - 'invalid', - '', - "", - ].join(""), - ), - "xl/worksheets/sheet2.xml": textBytes( - [ - '', - 'header', - 'note', - "", - ].join(""), - ), - }, - { level: 0 }, - ); - const parser = createUnstructuredParserClient({ - endpoint: "https://unstructured.example.test", - fetch: async () => - new Response( - JSON.stringify([ - { - metadata: { - page_number: 1, - page_name: "Issues", - text_as_html: - "
Issue
First issue
", - }, - type: "Table", - }, - { - metadata: { - sheet_name: "Notes", - text_as_html: - "
Note
First note
", - }, - type: "Table", - }, - { - metadata: { - text_as_html: - "
Other
Other record
", - }, - type: "Table", - }, - ]), - { headers: { "content-type": "application/json" }, status: 200 }, - ), - generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c62", - now: () => createdAt, - }); - - const artifact = await parser.parse({ - body, - documentAssetId, - filename: "multiple-sheets.xlsx", - mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - parserHints: { requiresImages: true, requiresTables: true }, - version: 1, - }); - const images = artifact.elements.filter((element) => element.type === "image"); - const imageByArchivePath = new Map( - images.map((image) => [image.metadata.archivePath, image] as const), - ); - - expect(images).toHaveLength(3); - expect(imageByArchivePath.get("xl/media/image1.png")).toMatchObject({ - metadata: { - archivePath: "xl/media/image1.png", - endOffset: utf8Length("Issue: First issue"), - spreadsheetAnchor: { - sheetIndex: 0, - sheetName: "Issues", - sourceColumn: 3, - sourceRow: 2, + it.each(["unsafe worksheet metadata", "safe worksheets with optional invalid image metadata"])( + "applies Office admission and image fallback independently for %s", + async (fixtureKind) => { + let body = zipSync( + { + "xl/_rels/workbook.xml.rels": textBytes( + [ + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + "", + ].join(""), + ), + "xl/drawings/_rels/drawing1.xml.rels": textBytes( + [ + '', + '', + '', + '', + '', + "", + ].join(""), + ), + "xl/drawings/drawing1.xml": textBytes( + [ + '', + spreadsheetImageAnchorXml({ + column: 2, + kind: "oneCellAnchor", + relationshipId: "rIdImage", + row: 1, + }), + spreadsheetImageAnchorXml({ column: 3, relationshipId: "rIdBlank", row: 2 }), + spreadsheetImageAnchorXml({ + column: 2, + kind: "oneCellAnchor", + relationshipId: "rIdImage", + row: 1, + }), + spreadsheetImageAnchorXml({ column: -1, relationshipId: "rIdImage", row: 1 }), + spreadsheetImageAnchorXml({ column: 2, relationshipId: "missing", row: 1 }), + spreadsheetImageAnchorXml({ column: 2, relationshipId: "rIdWrong", row: 1 }), + spreadsheetImageAnchorXml({ column: 2, relationshipId: "rIdText", row: 1 }), + "21", + "", + ].join(""), + ), + "xl/media/image1.png": new Uint8Array([1, 2, 3, 4]), + "xl/media/blank.png": new Uint8Array([5, 6, 7, 8]), + "xl/media/orphan.png": new Uint8Array([9, 10, 11, 12]), + "xl/workbook.xml": textBytes( + [ + '', + '', + '', + '', + '', + '', + '', + "", + ].join(""), + ), + "xl/worksheets/_rels/sheet1.xml.rels": textBytes( + [ + '', + '', + '', + "", + ].join(""), + ), + "xl/worksheets/sheet1.xml": textBytes( + [ + '', + 'header', + 'record', + 'invalid', + '', + "", + ].join(""), + ), + "xl/worksheets/sheet2.xml": textBytes( + [ + '', + 'header', + 'note', + "", + ].join(""), + ), }, - startOffset: 0, - }, - }); - expect(imageByArchivePath.get("xl/media/blank.png")).toMatchObject({ - metadata: { - archivePath: "xl/media/blank.png", - positionUnknown: true, - spreadsheetAnchor: { - sheetIndex: 0, - sheetName: "Issues", - sourceColumn: 4, - sourceRow: 3, - }, - }, - }); - expect(imageByArchivePath.get("xl/media/orphan.png")).toMatchObject({ - metadata: { - archivePath: "xl/media/orphan.png", - positionUnknown: true, - }, - }); - }); + { level: 0 }, + ); + if (fixtureKind === "safe worksheets with optional invalid image metadata") { + const entries: Record = unzipSync(body); + entries["xl/_rels/workbook.xml.rels"] = textBytes( + '', + ); + entries["xl/workbook.xml"] = textBytes( + '', + ); + entries["xl/worksheets/sheet1.xml"] = textBytes( + new TextDecoder() + .decode(entries["xl/worksheets/sheet1.xml"]) + .replace('invalid', ""), + ); + body = zipSync(entries); + } + const fetch = vi.fn( + async () => + new Response( + JSON.stringify([ + { + metadata: { + page_number: 1, + page_name: "Issues", + text_as_html: + "
Issue
First issue
", + }, + type: "Table", + }, + { + metadata: { + sheet_name: "Notes", + text_as_html: + "
Note
First note
", + }, + type: "Table", + }, + { + metadata: { + text_as_html: + "
Other
Other record
", + }, + type: "Table", + }, + ]), + { headers: { "content-type": "application/json" }, status: 200 }, + ), + ); + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch, + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c62", + now: () => createdAt, + }); - it("keeps parser text usable when optional spreadsheet media metadata is malformed", async () => { - const parser = createUnstructuredParserClient({ - endpoint: "https://unstructured.example.test", - fetch: async () => + const input = { + body, + documentAssetId, + filename: "multiple-sheets.xlsx", + mimeType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + parserHints: { requiresImages: true, requiresTables: true }, + version: 1, + }; + if (fixtureKind === "unsafe worksheet metadata") { + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + }); + expect(fetch).not.toHaveBeenCalled(); + return; + } + const artifact = await parser.parse(input); + const images = artifact.elements.filter((element) => element.type === "image"); + const imageByArchivePath = new Map( + images.map((image) => [image.metadata.archivePath, image] as const), + ); + + expect(images).toHaveLength(3); + expect(imageByArchivePath.get("xl/media/image1.png")).toMatchObject({ + metadata: { + archivePath: "xl/media/image1.png", + endOffset: utf8Length("Issue: First issue"), + spreadsheetAnchor: { + sheetIndex: 0, + sheetName: "Issues", + sourceColumn: 3, + sourceRow: 2, + }, + startOffset: 0, + }, + }); + expect(imageByArchivePath.get("xl/media/blank.png")).toMatchObject({ + metadata: { + archivePath: "xl/media/blank.png", + positionUnknown: true, + spreadsheetAnchor: { + sheetIndex: 0, + sheetName: "Issues", + sourceColumn: 4, + sourceRow: 3, + }, + }, + }); + expect(imageByArchivePath.get("xl/media/orphan.png")).toMatchObject({ + metadata: { + archivePath: "xl/media/orphan.png", + positionUnknown: true, + }, + }); + }, + ); + + it("rejects malformed spreadsheet ZIP and XML before remote parsing", async () => { + const fetch = vi.fn( + async () => new Response(JSON.stringify([{ text: "Provider text", type: "NarrativeText" }]), { headers: { "content-type": "application/json" }, status: 200, }), + ); + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch, generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c63", now: () => createdAt, }); @@ -2511,26 +2587,26 @@ describe("parser adapters", () => { version: 1, }); - const malformedZip = await parse(new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0x01])); - expect(malformedZip.elements).toHaveLength(1); - - const malformedRelationships = await parse( - zipSync( - { - "xl/media/image1.png": new Uint8Array([1, 2, 3, 4]), - "xl/workbook.xml": textBytes("<"), - }, - { level: 0 }, - ), - ); - expect(malformedRelationships.elements).toHaveLength(2); - expect(malformedRelationships.elements[1]).toMatchObject({ - metadata: { - archivePath: "xl/media/image1.png", - positionUnknown: true, - }, - type: "image", + await expect(parse(new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0x01]))).rejects.toMatchObject({ + code: "provider_input", + retryable: false, }); + + await expect( + parse( + zipSync( + { + "xl/media/image1.png": new Uint8Array([1, 2, 3, 4]), + "xl/workbook.xml": textBytes("<"), + }, + { level: 0 }, + ), + ), + ).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + }); + expect(fetch).not.toHaveBeenCalled(); }); it("deduplicates provider images while filling archive images that the provider omitted", async () => { @@ -2680,7 +2756,7 @@ describe("parser adapters", () => { }); const artifact = await parser.parse({ - body: new Uint8Array([1, 2, 3]), + body: compactDocx(), documentAssetId, filename: "manual.docx", mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -3675,7 +3751,7 @@ describe("parser adapters", () => { }); const parses = [ parser.parse({ - body: new Uint8Array([3]), + body: compactDocx(), documentAssetId, filename: "first.docx", mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -3696,7 +3772,7 @@ describe("parser adapters", () => { version: 1, }), parser.parse({ - body: new Uint8Array([4]), + body: compactDocx(), documentAssetId, filename: "second.docx", mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", @@ -4048,10 +4124,15 @@ describe("parser adapters", () => { }); it("uses the heavy deadline for every PDF and keeps ordinary Office on the standard deadline", async () => { + // A real transport retains its Request. Keep these mock requests alive until both + // deadlines settle as well, including the Request.signal follower subscriptions. + const requests: Request[] = []; const parser = createUnstructuredParserClient({ endpoint: "https://unstructured.example.test", fetch: async (input) => { const request = input instanceof Request ? input : new Request(input); + requests.push(request); + request.signal.throwIfAborted(); return await new Promise((_resolve, reject) => { request.signal.addEventListener("abort", () => reject(request.signal.reason), { once: true, @@ -4080,6 +4161,7 @@ describe("parser adapters", () => { version: 1, }), ).rejects.toThrow(/^Unstructured parser request timed out after requestTimeoutMs=10$/u); + expect(requests).toHaveLength(2); }); it("rejects hazardous OOXML expansion before starting a provider transport", async () => { @@ -4322,7 +4404,7 @@ describe("structured data parser coverage", () => { structured().parse( createParseInput({ body: "{broken", filename: "bad.json", mimeType: "application/json" }), ), - ).rejects.toThrow("invalid response"); + ).rejects.toThrow("input is malformed"); }); }); diff --git a/knowledge-fs/packages/parsers/src/structured-data-safety.test.ts b/knowledge-fs/packages/parsers/src/structured-data-safety.test.ts new file mode 100644 index 00000000000..a96c46e74f7 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/structured-data-safety.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from "vitest"; + +import { + createNativeHtmlParser, + createNativeMarkdownParser, + createNativeStructuredDataParser, + createParserRouter, +} from "./index"; + +const documentAssetId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44"; +function input(text: string, extension = "json") { + return { + body: new TextEncoder().encode(text), + documentAssetId, + filename: `data.${extension}`, + mimeType: extension === "csv" ? "text/csv" : `application/${extension}`, + version: 1, + }; +} + +describe("structured document fidelity and resource safety", () => { + it("preserves duplicate CSV columns without overwriting either value", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input("name,name\nleft,right", "csv"), + ); + expect(artifact.elements[0]?.text).toBe("name: left | name_2: right"); + }); + + it("does not collide generated duplicate names with original CSV column names", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input("name,name,name_2\nleft,middle,right", "csv"), + ); + expect(artifact.elements[0]?.text).toContain("left"); + expect(artifact.elements[0]?.text).toContain("middle"); + expect(artifact.elements[0]?.text).toContain("right"); + expect(new Set(artifact.elements[0]?.metadata.columns as string[]).size).toBe(3); + }); + + it("preserves integer and decimal JSON numeric lexemes", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input('{"id":9007199254740993,"decimal":0.1234567890123456789,"large":1e400}'), + ); + expect(artifact.elements[0]?.text).toContain("9007199254740993"); + expect(artifact.elements[0]?.text).toContain("0.1234567890123456789"); + expect(artifact.elements[0]?.text).toContain("1e400"); + }); + + it("preserves nested and tabular JSON numbers", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input('[{"id":9007199254740993,"nested":{"amount":1e400}}]'), + ); + expect(artifact.elements[0]?.text).toBe('id: 9007199254740993 | nested: {"amount":1e400}'); + }); + + it("preserves scalar, null and array JSONL records in source order", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input('42\ntrue\n"abc"\nnull\n[1,2]', "jsonl"), + ); + const text = artifact.elements.map((element) => element.text).join("\n"); + expect(text).toContain("42"); + expect(text).toContain("true"); + expect(text).toContain('"abc"'); + expect(text).toContain("null"); + expect(text).toContain("[1,2]"); + }); + + it("retains XML attributes and leading-zero identifiers", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input('00123', "xml"), + ); + expect(artifact.elements[0]?.text).toContain("A1"); + expect(artifact.elements[0]?.text).toContain("active"); + expect(artifact.elements[0]?.text).toContain("00123"); + }); + + it("keeps sparse records sparse without losing their field labels", async () => { + const source = Array.from({ length: 300 }, (_, index) => ({ [`field${index}`]: index })); + const artifact = await createNativeStructuredDataParser().parse(input(JSON.stringify(source))); + const text = artifact.elements.map((element) => element.text).join("\n"); + expect(text.length).toBeLessThan(20_000); + expect(text).toContain("field0: 0"); + expect(text).toContain("field299: 299"); + }); + + it("rejects excessive JSON nesting as a non-retryable input error before recursive parsing", async () => { + await expect( + createNativeStructuredDataParser().parse(input(`${"[".repeat(500)}1${"]".repeat(500)}`)), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + }); + + it("does not parse an already-cancelled native request", async () => { + const reason = new Error("cancelled by caller"); + await expect( + createNativeStructuredDataParser().parse({ + ...input("{}"), + signal: AbortSignal.abort(reason), + }), + ).rejects.toBe(reason); + }); + + it.each([createNativeMarkdownParser, createNativeHtmlParser])( + "cancels native markup before inspecting an over-limit input", + async (createParser) => { + const reason = new Error("cancelled before parsing"); + await expect( + createParser({ maxInputBytes: 1 }).parse({ + ...input("long input", "md"), + signal: AbortSignal.abort(reason), + }), + ).rejects.toBe(reason); + }, + ); + + it("keeps structured files on their parser instead of using an incompatible size fallback", async () => { + let remoteCalls = 0; + const router = createParserRouter({ + html: createNativeHtmlParser(), + markdown: createNativeMarkdownParser(), + maxNativeInputBytes: 8, + structured: createNativeStructuredDataParser({ maxInputBytes: 128 }), + unstructured: { + kind: "unstructured", + parse: async () => { + remoteCalls += 1; + throw new Error("incompatible remote parser"); + }, + }, + }); + const artifact = await router.parse(input('{"value":123}')); + expect(artifact.metadata.routedParser).toBe("native-structured"); + expect(remoteCalls).toBe(0); + }); + + it("preserves the ordinary homogeneous table representation", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input('[{"name":"A","value":2},{"name":"B","value":3}]'), + ); + expect(artifact.elements[0]?.text).toBe("name: A | value: 2\nname: B | value: 3"); + }); + + it("keeps duplicate Markdown headers distinct even when their suffix already exists", async () => { + const artifact = await createNativeMarkdownParser().parse( + input("| name | name | name_2 |\n| --- | --- | --- |\n| left | middle | right |", "md"), + ); + expect(artifact.elements[0]?.text).toBe("name: left | name_3: middle | name_2: right"); + }); + + it("bounds native Markdown table width before constructing projected rows", async () => { + const columns = Array.from({ length: 4097 }, (_, index) => `c${index}`); + const source = `| ${columns.join(" | ")} |\n| ${columns.map(() => "---").join(" | ")} |\n| ${columns.map(() => "x").join(" | ")} |`; + await expect(createNativeMarkdownParser().parse(input(source, "md"))).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + }); + }); + + it("preserves prototype-shaped CSV column names as ordinary own fields", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input("__proto__,constructor,prototype\nleft,middle,right", "csv"), + ); + expect(artifact.elements[0]?.text).toBe( + "__proto__: left | constructor: middle | prototype: right", + ); + }); + + it("does not manufacture inherited values for missing JSON record columns", async () => { + const artifact = await createNativeStructuredDataParser().parse( + input('[{"constructor":"real","__proto__":"own"},{"value":"other"}]'), + ); + expect(artifact.elements[0]?.text).toBe( + "constructor: real | __proto__: own | value: \nconstructor: | __proto__: | value: other", + ); + }); + + it("retains CSV headers when the document has no data rows", async () => { + const artifact = await createNativeStructuredDataParser().parse(input("name,score", "csv")); + expect(artifact.elements[0]?.text).toBe("name | score"); + expect(artifact.elements[0]?.metadata.columns).toEqual(["name", "score"]); + }); + + it("rejects overwide CSV headers before naming duplicates or reading data rows", async () => { + const result = await createNativeStructuredDataParser() + .parse(input(Array(4097).fill("name").join(","), "csv")) + .then( + () => null, + (error: unknown) => error, + ); + expect(result).toMatchObject({ code: "provider_input", retryable: false }); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/structured-json.test.ts b/knowledge-fs/packages/parsers/src/structured-json.test.ts new file mode 100644 index 00000000000..66f6f9a3696 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/structured-json.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from "vitest"; + +import { ParserResourceLimitError } from "./parser-resource-budget"; +import { + documentJsonRootType, + isDocumentRecord, + parseDocumentJson, + stringifyDocumentJson, +} from "./structured-json"; + +describe("lossless document JSON", () => { + it.each([ + "9007199254740993", + "-9007199254740993", + "1.2300e+05", + "-0", + "1e10000", + "0.00000000000000000000000000000000000000000000000000000001", + ])("preserves numeric lexemes without conversion: %s", (text) => { + const parsed = parseDocumentJson(text); + expect(stringifyDocumentJson(parsed)).toBe(text); + expect(documentJsonRootType(parsed)).toBe("number"); + expect(isDocumentRecord(parsed)).toBe(false); + }); + + it.each(["true", "false", "null", '"abc"', "[]", "{}"])( + "preserves scalar and empty container values: %s", + (text) => { + expect(stringifyDocumentJson(parseDocumentJson(text))).toBe(text); + }, + ); + + it("preserves mixed arrays without turning scalar values into records", () => { + const source = '[9007199254740993,1.2300e+05,-0,true,false,null,"abc",[],{}]'; + const parsed = parseDocumentJson(source); + expect(stringifyDocumentJson(parsed)).toBe(source); + expect(documentJsonRootType(parsed)).toBe("array"); + expect(isDocumentRecord(parsed)).toBe(false); + }); + + it("identifies ordinary record and scalar root types", () => { + expect(documentJsonRootType(parseDocumentJson('{"name":"record"}'))).toBe("object"); + expect(documentJsonRootType(parseDocumentJson("true"))).toBe("boolean"); + expect(documentJsonRootType(parseDocumentJson('"text"'))).toBe("string"); + }); + + it("preserves a native bigint supplied by a structured decoder", () => { + expect(stringifyDocumentJson({ id: 9007199254740993n })).toBe('{"id":9007199254740993}'); + }); + + it("treats library marker and serialization method names as ordinary fields", () => { + const source = '{"isLosslessNumber":true,"value":"keep","toJSON":"not a method"}'; + const parsed = parseDocumentJson(source); + expect(stringifyDocumentJson(parsed)).toBe(source); + expect(isDocumentRecord(parsed)).toBe(true); + }); + + it("preserves __proto__ as an own data property without changing the object prototype", () => { + const source = '{"__proto__":{"polluted":true},"constructor":"keep","prototype":"keep"}'; + const parsed = parseDocumentJson(source) as Record; + expect(Object.prototype.hasOwnProperty.call(parsed, "__proto__")).toBe(true); + expect([null, Object.prototype]).toContain(Object.getPrototypeOf(parsed)); + expect(stringifyDocumentJson(parsed)).toBe(source); + expect(Object.prototype).not.toHaveProperty("polluted"); + }); + + it("preserves nested and Unicode-escaped prototype field names", () => { + const source = '[{"nested":{"__pro\\u0074o__":null}},{"__proto__":"string value"}]'; + const parsed = parseDocumentJson(source); + expect(stringifyDocumentJson(parsed)).toBe( + '[{"nested":{"__proto__":null}},{"__proto__":"string value"}]', + ); + }); + + it("does not invoke toJSON hooks on serializer input", () => { + const toJSON = vi.fn(() => "unexpected hook"); + const output = stringifyDocumentJson({ toJSON, value: 42 }); + expect(toJSON).not.toHaveBeenCalled(); + expect(output).toBe('{"toJSON":null,"value":42}'); + }); + + it("ignores inherited fields when serializing document objects", () => { + const value = Object.assign(Object.create({ inherited: "not document data" }), { own: "keep" }); + expect(stringifyDocumentJson(value)).toBe('{"own":"keep"}'); + }); + + it("preserves numeric text when pretty-printing nested records", () => { + const value = parseDocumentJson('{"items":[{"id":9007199254740993}],"empty":{}}'); + expect(stringifyDocumentJson(value, true)).toBe( + '{\n "items": [\n {\n "id": 9007199254740993\n }\n ],\n "empty": {}\n}', + ); + }); + + it("preserves escaped controls, astral Unicode, and lone surrogate strings", () => { + const source = JSON.stringify({ '\u0000"\\😀': "\b\t\n\f\r\u001f中文😀\ud800\udc00\ud800" }); + expect(stringifyDocumentJson(parseDocumentJson(source))).toBe(source); + }); + + it.each(["", "[1,]", '{"a":}', "01", "NaN", "Infinity", '"unterminated'])( + "rejects invalid document JSON instead of returning a partial value: %s", + (text) => { + expect(() => parseDocumentJson(text)).toThrow(); + }, + ); + + it("rejects excessive input nesting before lossless parsing", () => { + expect(() => parseDocumentJson(`${"[".repeat(129)}0${"]".repeat(129)}`)).toThrow( + ParserResourceLimitError, + ); + }); + + it("rejects cyclic serializer input without recursive stack exhaustion", () => { + const cycle: unknown[] = []; + cycle.push(cycle); + expect(() => stringifyDocumentJson(cycle)).toThrow(ParserResourceLimitError); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/structured-json.ts b/knowledge-fs/packages/parsers/src/structured-json.ts new file mode 100644 index 00000000000..1dbea86fac1 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/structured-json.ts @@ -0,0 +1,85 @@ +import { Buffer } from "node:buffer"; + +import { + ParserResourceLimitError, + assertJsonDepth, + assertParserResourceBudget, + parserResourceLimits, +} from "./parser-resource-budget"; + +class DocumentJsonNumber { + constructor(readonly source: string) {} +} + +export function parseDocumentJson(text: string): unknown { + assertJsonDepth(text); + // Node 22 (the deployed runtime) supplies the original primitive token. Native JSON.parse + // also defines __proto__ as ordinary own data, unlike assignment-based JSON decoders. + const value: unknown = JSON.parse( + text, + (_key, current: unknown, context?: { source?: string }) => { + if (typeof current !== "number") return current; + if (context?.source === undefined) + throw new Error("Document JSON parsing requires Node.js 22 or newer"); + return new DocumentJsonNumber(context.source); + }, + ); + assertParserResourceBudget(value); + return value; +} + +export function isDocumentRecord(value: unknown): value is Record { + return ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + !(value instanceof DocumentJsonNumber) + ); +} + +export function documentJsonRootType(value: unknown): string { + return value instanceof DocumentJsonNumber + ? "number" + : Array.isArray(value) + ? "array" + : typeof value; +} + +/** + * Only actual numeric wrappers are special, never user-controlled isLosslessNumber/toJSON keys. + * Bounded depth plus a single output sink avoids quadratic pretty-printing intermediates. + */ +export function stringifyDocumentJson(value: unknown, pretty = false): string { + assertParserResourceBudget(value); + const chunks: string[] = []; + let bytes = 0; + const append = (text: string) => { + bytes += Buffer.byteLength(text, "utf8"); + if (bytes > parserResourceLimits.maxOutputBytes) + throw new ParserResourceLimitError("structured output bytes"); + chunks.push(text); + }; + const visit = (current: unknown, depth: number): void => { + if (current instanceof DocumentJsonNumber) { + append(current.source); + return; + } + if (current !== null && typeof current === "object") { + const array = Array.isArray(current); + const keys = Object.keys(current); + append(array ? "[" : "{"); + for (const [index, key] of keys.entries()) { + if (index > 0) append(","); + if (pretty) append(`\n${" ".repeat(depth + 1)}`); + if (!array) append(JSON.stringify(key) + (pretty ? ": " : ":")); + visit((current as Record)[key], depth + 1); + } + if (pretty && keys.length > 0) append(`\n${" ".repeat(depth)}`); + append(array ? "]" : "}"); + } else { + append(typeof current === "bigint" ? String(current) : (JSON.stringify(current) ?? "null")); + } + }; + visit(value, 0); + return chunks.join(""); +} diff --git a/knowledge-fs/packages/parsers/src/structured-stream-admission.test.ts b/knowledge-fs/packages/parsers/src/structured-stream-admission.test.ts new file mode 100644 index 00000000000..ed147a07b40 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/structured-stream-admission.test.ts @@ -0,0 +1,88 @@ +import { XMLParser } from "fast-xml-parser"; +import { describe, expect, it, vi } from "vitest"; +import { createNativeStructuredDataParser } from "./index"; +import { assertXmlStructureBudget, iterateDocumentLines } from "./structured-stream-admission"; + +const input = (body: string, extension: string) => ({ + body: new TextEncoder().encode(body), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: `x.${extension}`, + mimeType: `application/${extension}`, + version: 1, +}); + +describe("structured parser incremental admission", () => { + it.each([ + ["", [""]], + ["first", ["first"]], + ["a\n", ["a", ""]], + ["a\r\nb\n\nc\r", ["a\r", "b", "", "c\r"]], + ])("iterates line slices without a document-wide line array %#", (source, expected) => { + expect([...iterateDocumentLines(source)]).toEqual(expected); + }); + it("stops JSONL admission before decoding a malformed tail after the row budget", async () => { + await expect( + createNativeStructuredDataParser({ maxRows: 1 }).parse( + input("9007199254740993\r\n\n{", "jsonl"), + ), + ).rejects.toThrow("maxRows=1"); + }); + it("keeps CSV admission ahead of later invalid records", async () => { + await expect( + createNativeStructuredDataParser({ maxRows: 1 }).parse( + input('key\none\ntwo\n"unclosed', "csv"), + ), + ).rejects.toThrow("maxRows=1"); + }); + it("rejects deep XML before constructing its object graph", async () => { + const parse = vi.spyOn(XMLParser.prototype, "parse"); + try { + await expect( + createNativeStructuredDataParser().parse( + input(`${"".repeat(129)}x${"".repeat(129)}`, "xml"), + ), + ).rejects.toMatchObject({ code: "provider_input" }); + expect(parse).not.toHaveBeenCalled(); + } finally { + parse.mockRestore(); + } + }); + it("charges repeated nodes, attributes and text against a preallocation budget", () => { + expect(() => + assertXmlStructureBudget('text', { maxNodes: 4 }), + ).toThrow("XML node count"); + expect(() => + assertXmlStructureBudget("", { maxNodes: 3 }), + ).toThrow("XML node count"); + expect(() => + assertXmlStructureBudget("value", { maxDepth: 1 }), + ).toThrow("XML nesting depth"); + }); + it("does not mistake comments, CDATA, namespace tags or escaped literals for structure", () => { + const xml = + ']]><text>'; + expect(() => assertXmlStructureBudget(xml, { maxDepth: 2, maxNodes: 12 })).not.toThrow(); + }); + it("checks cancellation before scanning XML or yielding lines", () => { + const signal = AbortSignal.abort(new Error("stop")); + expect(() => assertXmlStructureBudget("", { signal })).toThrow("stop"); + expect(() => [...iterateDocumentLines("first\nsecond", signal)]).toThrow("stop"); + }); + it.each([{ maxNodes: 0 }, { maxNodes: Number.NaN }, { maxDepth: -1 }, { maxDepth: 1.2 }])( + "rejects invalid XML budgets %#", + (options) => { + expect(() => assertXmlStructureBudget("", options)).toThrow( + "invalid XML admission budget", + ); + }, + ); + it("scans across chunk boundaries without materializing a tree", () => { + const text = `${"x".repeat(131_072)}`; + expect(() => assertXmlStructureBudget(text, { maxDepth: 2, maxNodes: 12 })).not.toThrow(); + }); + it("does not charge indentation that the authoritative XML projection discards", () => { + expect(() => + assertXmlStructureBudget("\n \n \n \n", { maxNodes: 4 }), + ).not.toThrow(); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/structured-stream-admission.ts b/knowledge-fs/packages/parsers/src/structured-stream-admission.ts new file mode 100644 index 00000000000..99f8e526158 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/structured-stream-admission.ts @@ -0,0 +1,70 @@ +import { Parser } from "htmlparser2"; +import { ParserResourceLimitError, parserResourceLimits } from "./parser-resource-budget"; + +/** Only the current line is sliced; consumers can stop before materializing the remainder. */ +export function* iterateDocumentLines(text: string, signal?: AbortSignal): Generator { + let offset = 0; + while (true) { + signal?.throwIfAborted(); + const end = text.indexOf("\n", offset); + if (end < 0) { + yield text.slice(offset); + return; + } + yield text.slice(offset, end); + offset = end + 1; + } +} + +/** SAX admission is independent from the authoritative XML projection; no DOM is constructed. */ +export function assertXmlStructureBudget( + text: string, + options: { + readonly maxDepth?: number; + readonly maxNodes?: number; + readonly signal?: AbortSignal | undefined; + } = {}, +): void { + const maxDepth = options.maxDepth ?? parserResourceLimits.maxDepth; + const maxNodes = options.maxNodes ?? parserResourceLimits.maxNodes; + if ( + !Number.isSafeInteger(maxDepth) || + maxDepth < 1 || + !Number.isSafeInteger(maxNodes) || + maxNodes < 1 + ) + throw new ParserResourceLimitError("invalid XML admission budget"); + options.signal?.throwIfAborted(); + let depth = 0; + let nodes = 0; + const consumeNode = () => { + nodes += 1; + if (nodes > maxNodes) throw new ParserResourceLimitError("XML node count"); + }; + const parser = new Parser( + { + onopentagname() { + depth += 1; + if (depth > maxDepth) throw new ParserResourceLimitError("XML nesting depth"); + consumeNode(); + }, + onattribute: consumeNode, + ontext(value) { + // Match the authoritative projection's default trimValues behavior: indentation alone + // must not make a pretty-printed file consume twice the compact file's node budget. + if (value.trim()) consumeNode(); + }, + onprocessinginstruction: consumeNode, + onclosetag() { + depth -= 1; + }, + }, + { decodeEntities: false, xmlMode: true }, + ); + for (let offset = 0; offset < text.length; offset += 64 * 1024) { + options.signal?.throwIfAborted(); + parser.write(text.slice(offset, offset + 64 * 1024)); + } + parser.end(); + options.signal?.throwIfAborted(); +} diff --git a/knowledge-fs/packages/parsers/src/table-projection-bytes.test.ts b/knowledge-fs/packages/parsers/src/table-projection-bytes.test.ts new file mode 100644 index 00000000000..9e2fea3502e --- /dev/null +++ b/knowledge-fs/packages/parsers/src/table-projection-bytes.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createNativeMarkdownParser, createUnstructuredParserClient } from "./index"; + +vi.mock("./parser-resource-budget", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + parserResourceLimits: { ...actual.parserResourceLimits, maxOutputBytes: 64 }, + }; +}); + +const parse = (body: string) => + createNativeMarkdownParser().parse({ + body: new TextEncoder().encode(body), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "table.md", + mimeType: "text/markdown", + version: 1, + }); + +describe("table projection byte admission", () => { + it("classifies provider table expansion as a terminal invalid provider response", async () => { + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.invalid", + fetch: async () => + Response.json([ + { + type: "Table", + text: "table", + metadata: { text_as_html: `
${"中".repeat(30)}
` }, + }, + ]), + }); + await expect( + parser.parse({ + body: new TextEncoder().encode("%PDF-1.4"), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "provider.pdf", + mimeType: "application/pdf", + version: 1, + }), + ).rejects.toMatchObject({ code: "provider_response_invalid", retryable: false }); + }); + it("counts header-only UTF-8 output before joining the labels", async () => { + await expect( + parse(`| ${"中".repeat(12)} | ${"文".repeat(12)} |\n| --- | --- |`), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + }); + it("counts repeated column labels before joining data rows", async () => { + await expect( + parse(`| ${"中".repeat(8)} |\n| --- |\n| one |\n| two |\n| three |`), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + }); + it("keeps a normal table within the byte limit unchanged", async () => { + const artifact = await parse("| name | score |\n| --- | --- |\n| Ada | 2 |"); + expect(artifact.elements[0]?.text).toBe("name: Ada | score: 2"); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/text-document-contracts.test.ts b/knowledge-fs/packages/parsers/src/text-document-contracts.test.ts new file mode 100644 index 00000000000..064a56aed7c --- /dev/null +++ b/knowledge-fs/packages/parsers/src/text-document-contracts.test.ts @@ -0,0 +1,206 @@ +import { Buffer } from "node:buffer"; +import { describe, expect, it } from "vitest"; +import { + createNativeHtmlParser, + createNativeMarkdownParser, + createNativeStructuredDataParser, + createUnstructuredParserClient, +} from "./index"; + +const input = (body: Uint8Array | string, filename = "fixture.md", mimeType = "text/markdown") => ({ + body: typeof body === "string" ? new TextEncoder().encode(body) : body, + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename, + mimeType, + version: 1, +}); + +describe("text document decoding contracts", () => { + it.each([ + ["x.json", "{"], + ["x.csv", 'a,b\n"unterminated'], + ["x.jsonl", "null\n{"], + ])( + "classifies malformed %s as terminal parser input rather than generic compilation failure", + async (filename, body) => { + await expect( + createNativeStructuredDataParser().parse(input(body, filename)), + ).rejects.toMatchObject({ code: "provider_input", retryable: false }); + }, + ); + it("classifies native element and row limits as terminal input failures", async () => { + await expect( + createNativeMarkdownParser({ maxElements: 1 }).parse(input("# heading\n\nbody")), + ).rejects.toMatchObject({ code: "provider_input" }); + await expect( + createNativeStructuredDataParser({ maxRows: 1 }).parse(input("a\n1\n2", "x.csv")), + ).rejects.toMatchObject({ code: "provider_input" }); + }); + it.each(["utf16le", "utf16be"])( + "decodes BOM-marked %s without indexing NULs", + async (encoding) => { + const bytes = Buffer.from("\ufeff# 中文\n\nhello", "utf16le"); + if (encoding === "utf16be") bytes.swap16(); + const artifact = await createNativeMarkdownParser().parse(input(bytes)); + expect(artifact.elements.map((element) => element.text)).toEqual(["中文", "hello"]); + expect(artifact.metadata.textEncoding).toBe(encoding === "utf16le" ? "utf-16le" : "utf-16be"); + }, + ); + it.each([ + new Uint8Array([0xc3, 0x28]), + new Uint8Array([0xff, 0xfe, 0x00]), + new Uint8Array([0xfe, 0xff, 0xd8, 0x00]), + new Uint8Array([0xff, 0xfe, 0x00, 0x00]), + ])("rejects malformed or unsupported text encoding %#", async (bytes) => { + await expect(createNativeMarkdownParser().parse(input(bytes))).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + }); + }); + it("uses strict decoding in HTML and structured text", async () => { + const bytes = new Uint8Array([0xc3, 0x28]); + await expect( + createNativeHtmlParser().parse(input(bytes, "x.html", "text/html")), + ).rejects.toMatchObject({ code: "provider_input" }); + await expect( + createNativeStructuredDataParser().parse(input(bytes, "x.json", "application/json")), + ).rejects.toMatchObject({ code: "provider_input" }); + }); + it("decodes properties escapes, continuations and separators without indexing comments", async () => { + const artifact = await createNativeMarkdownParser().parse( + input( + "# comment\n! also comment\nhello\\ world = \\u4e2d\\u6587\nlong: first\\\n second\nempty\n__proto__=safe", + "x.properties", + "text/plain", + ), + ); + expect(artifact.elements.map((element) => element.text)).toEqual([ + "hello world = 中文", + "long = firstsecond", + "empty = ", + "__proto__ = safe", + ]); + expect(artifact.elements[0]?.metadata).toMatchObject({ + propertyKey: "hello world", + sourceLine: 3, + format: "properties", + }); + }); + it("rejects malformed property Unicode escapes", async () => { + await expect( + createNativeMarkdownParser().parse(input("key=\\uXYZW", "x.properties")), + ).rejects.toMatchObject({ code: "provider_input" }); + }); + it("preserves escaped separators, whitespace, Unicode surrogate pairs and terminal continuation", async () => { + const artifact = await createNativeMarkdownParser().parse( + input(" a\\:b\\=c : \\t\\n\\r\\f\\q\\\\\nemoji=\\uD83D\\uDE00\nlast=tail\\", "x.properties"), + ); + expect(artifact.elements.map((element) => element.metadata.propertyValue)).toEqual([ + "\t\n\r\fq\\", + "😀", + "tail", + ]); + expect(artifact.elements[0]?.metadata.propertyKey).toBe("a:b=c"); + }); + it.each(["key=\\uD800", "key=\\uDC00"])( + "rejects unpaired property surrogate %s", + async (body) => { + await expect( + createNativeMarkdownParser().parse(input(body, "x.properties")), + ).rejects.toMatchObject({ code: "provider_input" }); + }, + ); + it.each(["properties", "vtt"])( + "bounds %s elements before accumulating the complete file", + async (extension) => { + const body = + extension === "properties" + ? "first=one\nsecond=two" + : "WEBVTT\n\n00:00.000 --> 00:01.000\na\n\n00:01.000 --> 00:02.000\nb"; + await expect( + createNativeMarkdownParser({ maxElements: 1 }).parse(input(body, `x.${extension}`)), + ).rejects.toMatchObject({ code: "provider_input" }); + }, + ); + it("decodes UTF-8 BOM and rejects unmarked binary NUL text", async () => { + const artifact = await createNativeMarkdownParser().parse(input("\ufeffhello")); + expect(artifact.metadata.textEncoding).toBe("utf-8"); + expect(artifact.elements[0]?.text).toBe("hello"); + await expect(createNativeMarkdownParser().parse(input("h\0i\0"))).rejects.toMatchObject({ + code: "provider_input", + }); + await expect( + createNativeMarkdownParser().parse(input(new Uint8Array([0, 0, 0xfe, 0xff]))), + ).rejects.toMatchObject({ code: "provider_input" }); + }); + it("keeps VTT hour timings, multiline payload and empty cues", async () => { + const artifact = await createNativeMarkdownParser().parse( + input( + "WEBVTT title\r\n\r\nREGION\r\nid:region1\r\n\r\n01:01:01.100 --> 01:01:02.500\r\na\r\nb\r\n\r\n01:01:03.000 --> 01:01:04.000\r\n", + "x.vtt", + ), + ); + expect(artifact.elements.map((element) => element.text)).toEqual(["a\nb", ""]); + expect(artifact.elements[0]?.metadata.startTimeMs).toBe(3661100); + }); + it("rejects unsafe VTT hour magnitudes and malformed cue timing lines", async () => { + for (const body of [ + "WEBVTT\n\n999999999999999:01:01.000 --> 999999999999999:01:02.000\na", + "WEBVTT\n\nidentifier\nnot timings\na", + ]) { + await expect(createNativeMarkdownParser().parse(input(body, "x.vtt"))).rejects.toMatchObject({ + code: "provider_input", + }); + } + }); + it.each(["", " ", "x".repeat(257)])("rejects invalid backend revision %#", (backendRevision) => { + expect(() => + createUnstructuredParserClient({ endpoint: "https://parser.test", backendRevision }), + ).toThrow("backendRevision"); + }); + it("retains VTT cues and timings while excluding control blocks", async () => { + const artifact = await createNativeMarkdownParser().parse( + input( + "WEBVTT\n\nNOTE ignore\nnot speech\n\nSTYLE\n::cue { color: red; }\n\ncue-1\n00:01.000 --> 00:02.500 align:start\nHello world & team\n\n00:02.500 --> 00:04.000\nSecond line", + "x.vtt", + "text/vtt", + ), + ); + expect(artifact.elements.map((element) => element.text)).toEqual([ + "Hello world & team", + "Second line", + ]); + expect(artifact.elements[0]?.metadata).toMatchObject({ + cueId: "cue-1", + startTimeMs: 1000, + endTimeMs: 2500, + settings: "align:start", + format: "vtt", + }); + }); + it.each([ + "no header", + "WEBVTT\n\n00:03.000 --> 00:02.000\nbackwards", + "WEBVTT\n\n00:90.000 --> 00:91.000\nbad", + "WEBVTT\n\n00:01.000 --> 00:02.000\nhello\n00:03.000 --> 00:04.000", + ])("rejects malformed VTT without silently dropping content %#", async (body) => { + await expect(createNativeMarkdownParser().parse(input(body, "x.vtt"))).rejects.toMatchObject({ + code: "provider_input", + }); + }); + it("versions remote policy and artifacts by backend semantic revision", async () => { + const remote = (backendRevision: string) => + createUnstructuredParserClient({ + endpoint: "https://parser.test", + backendRevision, + fetch: async () => Response.json([{ type: "NarrativeText", text: "hello", metadata: {} }]), + }); + const source = input("hello", "x.txt", "text/plain"); + const first = remote("pinned-a"); + const second = remote("pinned-b"); + expect(first.policyFingerprint?.(source)).not.toBe(second.policyFingerprint?.(source)); + const [a, b] = await Promise.all([first.parse(source), second.parse(source)]); + expect(a.artifactHash).not.toBe(b.artifactHash); + expect(a.metadata.backendRevision).toBe("pinned-a"); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/text-document-contracts.ts b/knowledge-fs/packages/parsers/src/text-document-contracts.ts new file mode 100644 index 00000000000..b9b9a304d4c --- /dev/null +++ b/knowledge-fs/packages/parsers/src/text-document-contracts.ts @@ -0,0 +1,184 @@ +import type { ParseElement } from "@knowledge/core"; +import { DomUtils, parseDocument } from "htmlparser2"; + +/** Classified input errors remain terminal across the isolated parser IPC boundary. */ +export class TextDocumentInputError extends Error { + readonly code = "provider_input"; + readonly retryable = false; + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "TextDocumentInputError"; + } +} + +export function decodeDocumentText(bytes: Uint8Array): { + readonly text: string; + readonly encoding: "utf-8" | "utf-16le" | "utf-16be"; +} { + if ( + (bytes[0] === 0xff && bytes[1] === 0xfe && bytes[2] === 0 && bytes[3] === 0) || + (bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 0xfe && bytes[3] === 0xff) + ) + throw new TextDocumentInputError("UTF-32 text is not supported; convert to UTF-8 or UTF-16"); + const encoding = + bytes[0] === 0xff && bytes[1] === 0xfe + ? "utf-16le" + : bytes[0] === 0xfe && bytes[1] === 0xff + ? "utf-16be" + : "utf-8"; + try { + const text = new TextDecoder(encoding, { fatal: true }).decode(bytes); + if (text.includes("\0")) throw new Error("Text contains NUL characters"); + return { encoding, text }; + } catch (cause) { + throw new TextDocumentInputError(`Invalid ${encoding} text; use UTF-8 or BOM-marked UTF-16`, { + cause, + }); + } +} + +type Element = Omit; + +export function propertiesElements(text: string, maxElements: number): Element[] { + const lines = text.split(/\r\n|\r|\n/u); + const elements: Element[] = []; + for (let index = 0; index < lines.length; index += 1) { + let line = (lines[index] ?? "").replace(/^[ \t\f]+/u, ""); + if (!line || line.startsWith("#") || line.startsWith("!")) continue; + const sourceLine = index + 1; + const fragments: string[] = []; + while (true) { + let trailing = 0; + for (let cursor = line.length - 1; cursor >= 0 && line[cursor] === "\\"; cursor -= 1) + trailing += 1; + const continued = trailing % 2 === 1; + fragments.push(continued ? line.slice(0, -1) : line); + if (!continued || index + 1 >= lines.length) break; + index += 1; + line = (lines[index] ?? "").replace(/^[ \t\f]+/u, ""); + } + const logicalLine = fragments.join(""); + let separator = 0; + for (; separator < logicalLine.length; separator += 1) { + if (logicalLine[separator] === "\\") separator += 1; + else if (/[=: \t\f]/u.test(logicalLine[separator] ?? "")) break; + } + let valueStart = separator; + while (/[ \t\f]/u.test(logicalLine[valueStart] ?? "")) valueStart += 1; + if (logicalLine[valueStart] === "=" || logicalLine[valueStart] === ":") valueStart += 1; + while (/[ \t\f]/u.test(logicalLine[valueStart] ?? "")) valueStart += 1; + const key = decodePropertyEscapes(logicalLine.slice(0, separator)); + const value = decodePropertyEscapes(logicalLine.slice(valueStart)); + assertElementCapacity(elements, maxElements); + elements.push({ + metadata: { format: "properties", propertyKey: key, propertyValue: value, sourceLine }, + sectionPath: [], + text: `${key} = ${value}`, + type: "paragraph", + }); + } + return elements; +} + +function decodePropertyEscapes(value: string): string { + const output: string[] = []; + for (let index = 0; index < value.length; index += 1) { + const character = value[index]; + if (character !== "\\") { + output.push(character ?? ""); + continue; + } + index += 1; + const escaped = value[index]; + if (escaped === "u") { + const hex = value.slice(index + 1, index + 5); + if (!/^[0-9a-f]{4}$/iu.test(hex)) + throw new TextDocumentInputError("Malformed properties Unicode escape"); + output.push(String.fromCharCode(Number.parseInt(hex, 16))); + index += 4; + } else { + output.push( + escaped === "t" + ? "\t" + : escaped === "n" + ? "\n" + : escaped === "r" + ? "\r" + : escaped === "f" + ? "\f" + : (escaped ?? ""), + ); + } + } + const decoded = output.join(""); + for (let index = 0; index < decoded.length; index += 1) { + const code = decoded.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const low = decoded.charCodeAt(index + 1); + if (!(low >= 0xdc00 && low <= 0xdfff)) + throw new TextDocumentInputError("Malformed properties Unicode surrogate"); + index += 1; + } else if (code >= 0xdc00 && code <= 0xdfff) + throw new TextDocumentInputError("Malformed properties Unicode surrogate"); + } + return decoded; +} + +export function vttElements(text: string, maxElements: number): Element[] { + const normalized = text.replace(/\r\n|\r/gu, "\n"); + const blocks = normalized.split(/\n[ \t]*\n/u); + const header = blocks.shift() ?? ""; + if (!/^WEBVTT(?:[ \t][^\n]*)?(?:\n[^\n]*)*$/u.test(header) || header.includes("-->")) { + throw new TextDocumentInputError("WebVTT requires a valid WEBVTT header"); + } + const elements: Element[] = []; + for (const block of blocks) { + if (!block.trim() || /^(?:NOTE(?:[ \t\n]|$)|STYLE(?:\n|$)|REGION(?:\n|$))/u.test(block)) + continue; + const lines = block.split("\n"); + const identified = !lines[0]?.includes("-->"); + const cueId = identified ? lines.shift() : undefined; + const timings = lines.shift() ?? ""; + const match = /^(\S+)[ \t]+-->[ \t]+(\S+)(?:[ \t]+(.*))?$/u.exec(timings); + if (!match || lines.some((line) => line.includes("-->"))) + throw new TextDocumentInputError("Malformed WebVTT cue"); + const startTimeMs = vttTimestamp(match[1] ?? ""); + const endTimeMs = vttTimestamp(match[2] ?? ""); + if (endTimeMs <= startTimeMs) + throw new TextDocumentInputError("WebVTT cue end must follow its start"); + const payload = lines.join("\n"); + const document = parseDocument(payload); + const cueText = DomUtils.textContent(document).trim(); + assertElementCapacity(elements, maxElements); + elements.push({ + metadata: { + ...(cueId ? { cueId } : {}), + cuePayload: payload, + endTimeMs, + format: "vtt", + settings: match[3] ?? "", + startTimeMs, + }, + sectionPath: [], + text: cueText, + type: "paragraph", + }); + } + return elements; +} + +function vttTimestamp(value: string): number { + const match = /^(?:(\d{2,}):)?([0-5]\d):([0-5]\d)\.(\d{3})$/u.exec(value); + if (!match) throw new TextDocumentInputError("Invalid WebVTT cue timestamp"); + const result = + ((Number(match[1] ?? 0) * 60 + Number(match[2])) * 60 + Number(match[3])) * 1000 + + Number(match[4]); + if (!Number.isSafeInteger(result)) + throw new TextDocumentInputError("WebVTT cue timestamp is too large"); + return result; +} + +function assertElementCapacity(elements: readonly Element[], limit: number): void { + if (elements.length >= limit) + throw new TextDocumentInputError(`Text parser output exceeds maxElements=${limit}`); +} diff --git a/knowledge-fs/packages/parsers/src/unstructured-glyph-index.test.ts b/knowledge-fs/packages/parsers/src/unstructured-glyph-index.test.ts new file mode 100644 index 00000000000..48d52be7d03 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-glyph-index.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; + +import { createUnstructuredGlyphIndex } from "./unstructured-glyph-index"; + +function glyph(index: number, x: number, y: number, width = 10, height = 10, pageNumber = 1) { + return { box: { bottom: y + height, height, width, x, y }, index, pageNumber }; +} + +describe("page-local glyph candidate index", () => { + it("keeps a long sparse page local instead of scanning every glyph", () => { + const glyphs = Array.from({ length: 1000 }, (_, index) => glyph(index, 10, index * 100)); + const index = createUnstructuredGlyphIndex(glyphs); + let candidateCount = 0; + for (const current of glyphs) { + index.remove(current); + candidateCount += [...index.candidates(current)].length; + } + expect(candidateCount).toBe(0); + }); + + it("excludes other pages and consumed candidates", () => { + const upper = glyph(0, 10, 10); + const lower = glyph(1, 10, 21); + const otherPage = glyph(2, 10, 21, 10, 10, 2); + const index = createUnstructuredGlyphIndex([upper, lower, otherPage]); + index.remove(upper); + expect([...index.candidates(upper)]).toEqual([lower]); + index.remove(lower); + expect([...index.candidates(upper)]).toEqual([]); + index.remove(glyph(3, 10, 10, 10, 10, 3)); + expect([...index.candidates(glyph(3, 10, 10, 10, 10, 3))]).toEqual([]); + }); + + it("includes all geometrically eligible neighbors across signed bucket boundaries", () => { + const glyphs = Array.from({ length: 240 }, (_, index) => ({ + ...glyph( + index, + ((index * 37) % 131) - 65, + ((index * 19) % 167) - 83, + index % 13, + index % 17, + index % 3, + ), + box: { + ...glyph( + index, + ((index * 37) % 131) - 65, + ((index * 19) % 167) - 83, + index % 13, + index % 17, + ).box, + layoutWidth: index % 2 ? 10000 : undefined, + }, + })); + const index = createUnstructuredGlyphIndex(glyphs); + for (const upper of glyphs) { + const candidates = new Set([...index.candidates(upper)].map((candidate) => candidate.index)); + for (const lower of glyphs) { + if (upper.pageNumber !== lower.pageNumber) continue; + const height = Math.max(upper.box.height, lower.box.height); + const gap = lower.box.y - upper.box.bottom; + const tolerance = Math.max( + 2, + Math.max(upper.box.width, lower.box.width) * 0.35, + Math.max(upper.box.layoutWidth ?? 0, lower.box.layoutWidth ?? 0) * 0.002, + ); + if ( + lower.box.y + lower.box.height / 2 > upper.box.y + upper.box.height / 2 && + Math.abs(upper.box.x + upper.box.width / 2 - lower.box.x - lower.box.width / 2) <= + tolerance && + gap >= -height * 0.25 && + gap <= height * 1.1 + ) { + expect(candidates.has(lower.index)).toBe(true); + } + } + } + }); + + it("does not emit duplicate buckets at large finite coordinates", () => { + const first = glyph(0, 1e100, 1e100, 0, 0); + const second = glyph(1, 1e100, 1e100, 0, 0); + const index = createUnstructuredGlyphIndex([first, second]); + index.remove(first); + expect([...index.candidates(first)]).toEqual([second]); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/unstructured-glyph-index.ts b/knowledge-fs/packages/parsers/src/unstructured-glyph-index.ts new file mode 100644 index 00000000000..e4c86e3259a --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-glyph-index.ts @@ -0,0 +1,89 @@ +interface IndexedGlyph { + readonly box: { + readonly bottom: number; + readonly height: number; + readonly layoutWidth?: number | undefined; + readonly width: number; + readonly x: number; + readonly y: number; + }; + readonly index: number; + readonly pageNumber: number; +} + +interface GlyphPage { + readonly buckets: Map>; + cellHeight: number; + cellWidth: number; +} + +/** + * A conservative page-local grid for finite glyph geometry. Every adjacent pair under the + * vertical-text predicate is in neighboring buckets: horizontal tolerance is at most cellWidth, + * and lower.y - upper.bottom lies between -0.25 and +1.1 cellHeight. Exact adjacency and tie + * breaking remain the caller's responsibility. Dense/adversarial buckets need a caller-owned + * comparison budget; the index never silently drops a possible neighbor. + */ +export function createUnstructuredGlyphIndex( + glyphs: readonly T[], +): { + candidates(upper: T): Iterable; + remove(glyph: T): void; +} { + const pages = new Map>(); + for (const glyph of glyphs) { + let page = pages.get(glyph.pageNumber); + if (!page) { + page = { buckets: new Map(), cellHeight: 1, cellWidth: 2 }; + pages.set(glyph.pageNumber, page); + } + page.cellHeight = Math.max(page.cellHeight, glyph.box.height); + page.cellWidth = Math.max( + page.cellWidth, + glyph.box.width * 0.35, + (glyph.box.layoutWidth ?? 0) * 0.002, + ); + } + + const bucketKey = (glyph: T, page: GlyphPage): string => + `${Math.floor((glyph.box.x + glyph.box.width / 2) / page.cellWidth)}:${Math.floor(glyph.box.y / page.cellHeight)}`; + for (const glyph of glyphs) { + const page = pages.get(glyph.pageNumber) as GlyphPage; + const key = bucketKey(glyph, page); + let bucket = page.buckets.get(key); + if (!bucket) { + bucket = new Map(); + page.buckets.set(key, bucket); + } + bucket.set(glyph.index, glyph); + } + + return { + candidates: function* (upper) { + const page = pages.get(upper.pageNumber); + if (!page) return; + const x = Math.floor((upper.box.x + upper.box.width / 2) / page.cellWidth); + const y = Math.floor(upper.box.bottom / page.cellHeight); + // Fixed offsets, rather than incrementing huge coordinate values in a numeric range, keep + // traversal bounded even beyond the safe-integer range. Deduplicate rounded bucket keys. + const visited = new Set(); + for (const dx of [-1, 0, 1]) { + for (const dy of [-1, 0, 1, 2]) { + const key = `${x + dx}:${y + dy}`; + if (visited.has(key)) continue; + visited.add(key); + const bucket = page.buckets.get(key); + if (bucket) yield* bucket.values(); + } + } + }, + remove: (glyph) => { + const page = pages.get(glyph.pageNumber); + if (!page) return; + const key = bucketKey(glyph, page); + const bucket = page.buckets.get(key); + bucket?.delete(glyph.index); + if (bucket?.size === 0) page.buckets.delete(key); + }, + }; +} diff --git a/knowledge-fs/packages/parsers/src/unstructured-normalization-policy.ts b/knowledge-fs/packages/parsers/src/unstructured-normalization-policy.ts new file mode 100644 index 00000000000..5f0004d5669 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-normalization-policy.ts @@ -0,0 +1,4 @@ +/** Independent expansion budgets for provider-controlled layout and heading metadata. */ +export const maxUnstructuredSectionDepth = 64; +export const maxUnstructuredSectionPathItems = 100_000; +export const maxUnstructuredVerticalCandidateComparisons = 1_000_000; diff --git a/knowledge-fs/packages/parsers/src/unstructured-normalization.test.ts b/knowledge-fs/packages/parsers/src/unstructured-normalization.test.ts new file mode 100644 index 00000000000..b3f48c473cb --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-normalization.test.ts @@ -0,0 +1,224 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createUnstructuredParserClient } from "./index"; + +vi.mock("./unstructured-normalization-policy", () => ({ + maxUnstructuredSectionDepth: 4, + maxUnstructuredSectionPathItems: 16, + maxUnstructuredVerticalCandidateComparisons: 8, +})); + +const input = { + body: new TextEncoder().encode("%PDF-1.4\n"), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "layout.pdf", + mimeType: "application/pdf", + version: 1, +}; + +function parserFor(elements: readonly unknown[]) { + return createUnstructuredParserClient({ + endpoint: "http://parser.invalid", + fetch: async () => Response.json(elements), + }); +} + +function titleChain(count: number) { + return Array.from({ length: count }, (_, index) => ({ + element_id: `title-${index}`, + metadata: index === 0 ? {} : { parent_id: `title-${index - 1}` }, + text: `Heading ${index + 1}`, + type: "Title", + })); +} + +function glyph(index: number, x: number, y: number, pageNumber = 1) { + return { + element_id: `glyph-${index}`, + metadata: { + coordinates: { + layout_height: 10000, + layout_width: 1000, + points: [ + [x, y], + [x, y + 10], + [x + 10, y + 10], + [x + 10, y], + ], + system: "PixelSpace", + }, + page_number: pageNumber, + }, + text: "中", + type: "UncategorizedText", + }; +} + +describe("bounded Unstructured normalization", () => { + afterEach(() => vi.restoreAllMocks()); + + it("rejects an over-deep provider parent chain before expanding its paths", async () => { + await expect(parserFor(titleChain(5)).parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + message: expect.stringContaining("maxSectionDepth=4"), + }); + }); + + it("applies the same depth limit to category-depth headings", async () => { + const elements = titleChain(5).map((element, index) => ({ + ...element, + metadata: { category_depth: index }, + })); + await expect(parserFor(elements).parse(input)).rejects.toThrow("maxSectionDepth=4"); + }); + + it("rejects excessive cumulative section-path expansion across paragraphs", async () => { + const elements = [ + ...titleChain(4), + { metadata: {}, text: "First body", type: "NarrativeText" }, + { metadata: {}, text: "Second body", type: "NarrativeText" }, + ]; + await expect(parserFor(elements).parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + message: expect.stringContaining("maxSectionPathItems=16"), + }); + }); + + it("preserves admitted hierarchical paths and sibling headings", async () => { + const artifact = await parserFor([ + ...titleChain(3), + { metadata: {}, text: "Body", type: "NarrativeText" }, + { + element_id: "sibling", + metadata: { parent_id: "title-0" }, + text: "Sibling", + type: "Title", + }, + ]).parse(input); + expect(artifact.elements.map((element) => element.sectionPath)).toEqual([ + ["Heading 1"], + ["Heading 1", "Heading 2"], + ["Heading 1", "Heading 2", "Heading 3"], + ["Heading 1", "Heading 2", "Heading 3"], + ["Heading 1", "Sibling"], + ]); + }); + + it("rejects pathological dense layout when its comparison budget is exhausted", async () => { + const elements = Array.from({ length: 6 }, (_, index) => glyph(index, 10, 10)); + await expect(parserFor(elements).parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + message: expect.stringContaining("maxVerticalCandidateComparisons=8"), + }); + }); + + it("does not charge unrelated pages against the local candidate budget", async () => { + const elements = Array.from({ length: 100 }, (_, index) => glyph(index, 10, 10, index + 1)); + const artifact = await parserFor(elements).parse(input); + expect(artifact.elements).toHaveLength(100); + expect(artifact.elements.map((element) => element.pageNumber)).toEqual( + Array.from({ length: 100 }, (_, index) => index + 1), + ); + }); + + it("does not compare spatially distant glyphs on the same page", async () => { + const elements = Array.from({ length: 50 }, (_, index) => glyph(index, 10, index * 100)); + const artifact = await parserFor(elements).parse(input); + expect(artifact.elements).toHaveLength(50); + }); + + it("merges a bounded vertical line in reading order", async () => { + const artifact = await parserFor([ + { ...glyph(0, 10, 10), text: "中" }, + { ...glyph(1, 10, 21), text: "国" }, + { ...glyph(2, 10, 32), text: "人" }, + ]).parse(input); + expect(artifact.elements).toHaveLength(1); + expect(artifact.elements[0]?.text).toBe("中国人"); + expect(artifact.elements[0]?.metadata.layout_normalization).toEqual({ + operation: "merge_vertical_text", + source_element_count: 3, + }); + }); + + it("preserves the original leftmost tie-break across candidate buckets", async () => { + const artifact = await parserFor([ + { ...glyph(0, 10, 10), text: "中" }, + { ...glyph(1, 13, 21), text: "日" }, + { ...glyph(2, 11, 22), text: "月" }, + ]).parse(input); + expect(artifact.elements.map((element) => element.text)).toEqual(["中月", "日"]); + }); + + it("does not merge glyphs from incompatible coordinate systems", async () => { + const lower = glyph(1, 10, 21); + const artifact = await parserFor([ + glyph(0, 10, 10), + { + ...lower, + metadata: { + ...lower.metadata, + coordinates: { ...lower.metadata.coordinates, system: "PointSpace" }, + }, + }, + ]).parse(input); + expect(artifact.elements).toHaveLength(2); + }); + + it("rejects finite points whose inferred geometry overflows", async () => { + const element = glyph(0, 0, 0); + const parser = parserFor([ + { + ...element, + metadata: { + ...element.metadata, + coordinates: { + ...element.metadata.coordinates, + points: [ + [-1e308, 0], + [1e308, 10], + ], + }, + }, + }, + ]); + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + message: "Unstructured parser returned invalid glyph geometry", + }); + }); + + it("recognizes a monotonic deadline overrun even before timers can run", async () => { + const clock = vi.spyOn(performance, "now").mockReturnValue(0); + const parser = createUnstructuredParserClient({ + endpoint: "http://parser.invalid", + fetch: async () => { + clock.mockReturnValue(101); + return Response.json([{ metadata: {}, text: "Body", type: "NarrativeText" }]); + }, + requestTimeoutMs: 100, + }); + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_timeout", + retryable: false, + }); + }); + + it("accepts a response strictly within the monotonic deadline", async () => { + const clock = vi.spyOn(performance, "now").mockReturnValue(1000); + const parser = createUnstructuredParserClient({ + endpoint: "http://parser.invalid", + fetch: async () => { + clock.mockReturnValue(1099); + return Response.json([{ metadata: {}, text: "Body", type: "NarrativeText" }]); + }, + requestTimeoutMs: 100, + }); + const artifact = await parser.parse(input); + expect(artifact.elements[0]?.text).toBe("Body"); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/unstructured-preflight.test.ts b/knowledge-fs/packages/parsers/src/unstructured-preflight.test.ts new file mode 100644 index 00000000000..36a43fad903 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-preflight.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it, vi } from "vitest"; + +import { ProviderInputError, createUnstructuredParserClient } from "./index"; + +const input = { + body: new TextEncoder().encode("%PDF-1.7"), + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: "banner.pdf", + mimeType: "application/pdf", + version: 1, +}; + +describe("Unstructured request preflight", () => { + it("treats the provider raster guard as terminal input failure before inline retries", async () => { + const fetch = vi.fn(async () => + Response.json( + { + detail: + "PDF page would render to too many pixels for safe processing: page=1, pixels=273439296, maximum=25000000. Try splitting the PDF, reducing the page dimensions, or using a lower render DPI.", + }, + { status: 500 }, + ), + ); + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.example.test", + fetch, + maxRetries: 3, + retryDelayMs: 0, + }); + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_input", + retryable: false, + requestOutcomeAmbiguous: false, + }); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it("rejects unsafe pages without sending or retrying a provider request", async () => { + const fetch = vi.fn(async () => new Response("[]")); + const check = vi.fn(async () => { + throw new ProviderInputError("PDF page 1 exceeds the raster pixel budget"); + }); + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.example.test", + fetch, + maxRetries: 3, + requestPreflight: { check }, + }); + + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_input", + requestOutcomeAmbiguous: false, + retryable: false, + }); + expect(check).toHaveBeenCalledOnce(); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("coalesces preflight with identical requests and preserves the original provider input", async () => { + let finishPreflight = () => {}; + const waiting = new Promise((resolve) => { + finishPreflight = resolve; + }); + const check = vi.fn(async () => waiting); + const fetch = vi.fn(async (request: RequestInfo | URL) => { + if (!(request instanceof Request)) throw new Error("Expected a Request"); + const form = await request.formData(); + const file = form.get("files"); + if (!(file instanceof File)) throw new Error("Expected original file"); + expect(new Uint8Array(await file.arrayBuffer())).toEqual(input.body); + expect(form.get("strategy")).toBe("hi_res"); + return new Response("[]"); + }); + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.example.test", + fetch, + requestPreflight: { check }, + }); + const imageInput = { ...input, parserHints: { requiresImages: true } }; + const first = parser.parse(imageInput); + const second = parser.parse(imageInput); + await vi.waitFor(() => expect(check).toHaveBeenCalledOnce()); + finishPreflight(); + const [a, b] = await Promise.all([first, second]); + expect(a).toEqual(b); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it("cancels before transport and releases admission for the next file", async () => { + let notifyStarted = () => {}; + const started = new Promise((resolve) => { + notifyStarted = resolve; + }); + const controller = new AbortController(); + const check = vi.fn(async (candidate: typeof input & { signal?: AbortSignal }) => { + if (candidate.filename !== "banner.pdf") return; + notifyStarted(); + await new Promise((_, reject) => { + candidate.signal?.addEventListener("abort", () => reject(candidate.signal?.reason), { + once: true, + }); + }); + }); + const fetch = vi.fn(async () => new Response("[]")); + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.example.test", + fetch, + maxConcurrency: 1, + requestPreflight: { check }, + }); + const first = parser.parse({ ...input, signal: controller.signal }); + const rejected = expect(first).rejects.toThrow("Import canceled"); + await started; + controller.abort(new Error("Import canceled")); + await rejected; + await parser.parse({ ...input, filename: "ordinary.pdf", version: 2 }); + expect(fetch).toHaveBeenCalledOnce(); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/unstructured-response-budget.test.ts b/knowledge-fs/packages/parsers/src/unstructured-response-budget.test.ts new file mode 100644 index 00000000000..dc80e90e913 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-response-budget.test.ts @@ -0,0 +1,79 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { createUnstructuredParserClient } from "./index"; + +vi.mock("./parser-resource-budget", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + parserResourceLimits: { + ...actual.parserResourceLimits, + // Allow the bounded artifact-level provenance/coverage envelope as well as elements. + maxArtifactNodes: 64, + maxRawElements: 3, + }, + }; +}); + +const input = { + body: new TextEncoder().encode("%PDF-1.4\n"), + documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", + filename: "raw-response.pdf", + mimeType: "application/pdf", + version: 1, +}; + +function parserFor(body: string) { + return createUnstructuredParserClient({ + endpoint: "http://parser.invalid", + fetch: async () => new Response(body), + }); +} + +describe("provider response admission before normalization", () => { + afterEach(() => vi.restoreAllMocks()); + + it("rejects excessive raw elements even when normalization would discard them all", async () => { + await expect(parserFor("[{},{},{},{}]").parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + }); + + it("admits the raw-element boundary without changing normal output filtering", async () => { + const artifact = await parserFor("[{},{},{}]").parse(input); + expect(artifact.elements).toEqual([]); + }); + + it("rejects excessive metadata nodes that output filtering would otherwise hide", async () => { + const body = JSON.stringify([ + { metadata: { nested: Array.from({ length: 100 }, () => null) } }, + ]); + await expect(parserFor(body).parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + }); + + it("rejects excessive JSON depth before invoking the recursive decoder", async () => { + const body = `[{"metadata":{"nested":${"[".repeat(129)}0${"]".repeat(129)}}}]`; + const parser = parserFor(body); + const decode = vi.spyOn(JSON, "parse"); + await expect(parser.parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + expect(decode).not.toHaveBeenCalled(); + }); + + it("keeps invalid JSON and invalid top-level shapes classified as provider responses", async () => { + await expect(parserFor("[{]").parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + await expect(parserFor('{"text":"not an element array"}').parse(input)).rejects.toMatchObject({ + code: "provider_response_invalid", + retryable: false, + }); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/unstructured-response-budget.ts b/knowledge-fs/packages/parsers/src/unstructured-response-budget.ts new file mode 100644 index 00000000000..6a0a4398e74 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-response-budget.ts @@ -0,0 +1,32 @@ +import { + ParserResourceLimitError, + assertJsonDepth, + assertParserResourceBudget, + parserResourceLimits, +} from "./parser-resource-budget"; + +/** + * The transport has already bounded response bytes. Check JSON depth before decoding and the raw + * tree before schema cloning or normalization: filtering empty/noisy elements later is not an + * admission policy. Provider output gets the larger artifact-node budget, not the native-record + * budget, so ordinary coordinate metadata remains compatible with the final element limit. + */ +export function parseUnstructuredResponsePayload( + text: string, + options: { + readonly maxResponseBytes: number; + readonly signal?: AbortSignal | undefined; + }, +): unknown { + assertJsonDepth(text, options.signal); + const payload: unknown = JSON.parse(text); + if (Array.isArray(payload) && payload.length > parserResourceLimits.maxRawElements) { + throw new ParserResourceLimitError("raw provider element count"); + } + assertParserResourceBudget(payload, { + maxBytes: options.maxResponseBytes, + maxNodes: parserResourceLimits.maxArtifactNodes, + signal: options.signal, + }); + return payload; +} diff --git a/knowledge-fs/packages/parsers/src/unstructured-response-safety.test.ts b/knowledge-fs/packages/parsers/src/unstructured-response-safety.test.ts new file mode 100644 index 00000000000..e87d4183ce4 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-response-safety.test.ts @@ -0,0 +1,172 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { isPdfRasterLimitResponse } from "./unstructured-response-safety"; + +const guardDetail = + "PDF page would render to too many pixels for safe processing: " + + "page=1, pixels=273439296, maximum=25000000. " + + "Try splitting the PDF, reducing the page dimensions, or using a lower render DPI."; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("Unstructured PDF raster-limit responses", () => { + it.each([500, 422])("recognizes the exact provider guard in HTTP %s", async (status) => { + const response = Response.json({ detail: guardDetail }, { status }); + await expect(isPdfRasterLimitResponse(response, new AbortController().signal)).resolves.toBe( + true, + ); + expect(response.body?.locked).toBe(false); + }); + + it.each([ + null, + { error: guardDetail }, + { detail: null }, + { detail: "Internal server error" }, + { detail: `${guardDetail} More untrusted text` }, + { detail: guardDetail.replace("273439296", "1") }, + { detail: guardDetail.replace("page=1", "page=0") }, + { detail: guardDetail.replace("maximum=25000000", "maximum=0") }, + { detail: guardDetail.replace("273439296", "9007199254740999") }, + ])("keeps unknown errors in their original HTTP status class: %j", async (payload) => { + await expect( + isPdfRasterLimitResponse( + Response.json(payload, { status: 500 }), + new AbortController().signal, + ), + ).resolves.toBe(false); + }); + + it("does not recognize malformed JSON", async () => { + await expect( + isPdfRasterLimitResponse( + new Response("{bad-json", { status: 500 }), + new AbortController().signal, + ), + ).resolves.toBe(false); + }); + + it("handles a missing error body", async () => { + await expect( + isPdfRasterLimitResponse(new Response(null, { status: 500 }), new AbortController().signal), + ).resolves.toBe(false); + }); + + it("leaves an already-locked response body with its current owner", async () => { + const response = Response.json({ detail: guardDetail }, { status: 500 }); + const reader = response.body?.getReader(); + await expect(isPdfRasterLimitResponse(response, new AbortController().signal)).resolves.toBe( + false, + ); + expect(response.body?.locked).toBe(true); + reader?.releaseLock(); + }); + + it("accepts a complete streamed error at the 4 KiB limit", async () => { + const payload = new TextEncoder().encode(JSON.stringify({ detail: guardDetail }).padEnd(4096)); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(payload.subarray(0, 100)); + controller.enqueue(payload.subarray(100)); + controller.close(); + }, + }); + await expect( + isPdfRasterLimitResponse(new Response(body, { status: 500 }), new AbortController().signal), + ).resolves.toBe(true); + }); + + it("bounds the sum of streamed chunks and ignores a rejected cancellation", async () => { + const cancel = vi.fn(async () => { + throw new Error("connection already closed"); + }); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(2048)); + controller.enqueue(new Uint8Array(2049)); + }, + cancel, + }); + await expect( + isPdfRasterLimitResponse(new Response(body, { status: 500 }), new AbortController().signal), + ).resolves.toBe(false); + expect(cancel).toHaveBeenCalledOnce(); + expect(body.locked).toBe(false); + }); + + it("does not read ordinary HTTP 429 responses", async () => { + const pull = vi.fn(); + const body = new ReadableStream({ pull }, { highWaterMark: 0 }); + await expect( + isPdfRasterLimitResponse(new Response(body, { status: 429 }), new AbortController().signal), + ).resolves.toBe(false); + expect(pull).not.toHaveBeenCalled(); + expect(body.locked).toBe(false); + }); + + it("stops at 4 KiB and cancels oversized error bodies", async () => { + const cancel = vi.fn(); + const body = new ReadableStream({ + pull(controller) { + controller.enqueue(new Uint8Array(4097)); + }, + cancel, + }); + await expect( + isPdfRasterLimitResponse(new Response(body, { status: 500 }), new AbortController().signal), + ).resolves.toBe(false); + expect(cancel).toHaveBeenCalledOnce(); + expect(body.locked).toBe(false); + }); + + it("bounds a stalled error-body inspection to one second even if cancellation stalls", async () => { + vi.useFakeTimers(); + const cancel = vi.fn(() => new Promise(() => {})); + const body = new ReadableStream({ cancel }); + const result = isPdfRasterLimitResponse( + new Response(body, { status: 500 }), + new AbortController().signal, + ); + await vi.advanceTimersByTimeAsync(1000); + await expect(result).resolves.toBe(false); + expect(cancel).toHaveBeenCalledOnce(); + expect(body.locked).toBe(false); + expect(vi.getTimerCount()).toBe(0); + }); + + it("preserves parent cancellation and releases the response reader", async () => { + const controller = new AbortController(); + const reason = new Error("parser lease expired"); + const cancel = vi.fn(); + const body = new ReadableStream({ cancel }); + const result = isPdfRasterLimitResponse(new Response(body, { status: 500 }), controller.signal); + const rejected = expect(result).rejects.toBe(reason); + controller.abort(reason); + await rejected; + expect(cancel).toHaveBeenCalledOnce(); + expect(body.locked).toBe(false); + }); + + it("preserves an already-aborted parent's reason without locking the stream", async () => { + const controller = new AbortController(); + const reason = new Error("already cancelled"); + controller.abort(reason); + const response = Response.json({ detail: guardDetail }, { status: 500 }); + await expect(isPdfRasterLimitResponse(response, controller.signal)).rejects.toBe(reason); + expect(response.body?.locked).toBe(false); + }); + + it("preserves the HTTP classification when the response body itself fails", async () => { + const body = new ReadableStream({ + start(controller) { + controller.error(new Error("connection closed")); + }, + }); + await expect( + isPdfRasterLimitResponse(new Response(body, { status: 500 }), new AbortController().signal), + ).resolves.toBe(false); + expect(body.locked).toBe(false); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/unstructured-response-safety.ts b/knowledge-fs/packages/parsers/src/unstructured-response-safety.ts new file mode 100644 index 00000000000..4478a193235 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-response-safety.ts @@ -0,0 +1,83 @@ +const maxErrorBodyBytes = 4 * 1024; +const inspectionTimeoutMs = 1_000; + +// unstructured-inference 1.6.11 emits this exact message before allocating a PDF page bitmap. +// The pinned API preserves it in JSON `detail`, but wraps the ValueError as HTTP 500. +const rasterLimitDetailPattern = + /^PDF page would render to too many pixels for safe processing: page=([1-9]\d*), pixels=([1-9]\d*), maximum=([1-9]\d*)\. Try splitting the PDF, reducing the page dimensions, or using a lower render DPI\.$/; + +/** Consumes only recognized error-status bodies; unknown errors keep their HTTP classification. */ +export async function isPdfRasterLimitResponse( + response: Response, + signal: AbortSignal, +): Promise { + return Boolean( + await inspectUnstructuredErrorPayload( + response, + signal, + [500, 422], + (payload) => isPdfRasterLimitPayload(payload) || undefined, + ), + ); +} + +/** One bounded read shared by classifiers, so HTTP 422 bodies are never consumed twice. */ +export async function inspectUnstructuredErrorPayload( + response: Response, + signal: AbortSignal, + statuses: readonly number[], + classify: (payload: unknown) => T | undefined, +): Promise { + signal.throwIfAborted(); + if (!statuses.includes(response.status)) return undefined; + if (!response.body || response.body.locked) return undefined; + + const reader = response.body.getReader(); + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + try { + const interrupted = new Promise((resolve, reject) => { + timer = setTimeout(() => resolve(undefined), inspectionTimeoutMs); + onAbort = () => reject(signal.reason); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + }); + const payload = await Promise.race([readErrorPayload(reader), interrupted]); + signal.throwIfAborted(); + return classify(payload); + } catch { + signal.throwIfAborted(); + return undefined; + } finally { + if (timer !== undefined) clearTimeout(timer); + if (onAbort) signal.removeEventListener("abort", onAbort); + // A stalled or broken provider must not prolong the inspection while acknowledging cancel. + void reader.cancel().catch(() => undefined); + reader.releaseLock(); + } +} + +async function readErrorPayload(reader: ReadableStreamDefaultReader): Promise { + const body = new Uint8Array(maxErrorBodyBytes); + let length = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value.byteLength > maxErrorBodyBytes - length) return undefined; + body.set(value, length); + length += value.byteLength; + } + + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(body.subarray(0, length))); +} + +export function isPdfRasterLimitPayload(payload: unknown): boolean { + if (typeof payload !== "object" || payload === null || !("detail" in payload)) return false; + if (typeof payload.detail !== "string") return false; + const match = rasterLimitDetailPattern.exec(payload.detail); + if (!match) return false; + const page = Number(match[1]); + const pixels = Number(match[2]); + const maximum = Number(match[3]); + return [page, pixels, maximum].every(Number.isSafeInteger) && pixels > maximum; +} diff --git a/knowledge-fs/packages/parsers/src/unstructured-sandbox-response.test.ts b/knowledge-fs/packages/parsers/src/unstructured-sandbox-response.test.ts new file mode 100644 index 00000000000..1737a857938 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-sandbox-response.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it, vi } from "vitest"; +import { createUnstructuredParserClient } from "./index"; +import { classifyUnstructuredResourceResponse } from "./unstructured-sandbox-response"; + +const payload = (reason: string) => ({ + detail: { + code: "PARSER_RESOURCE_REJECTED", + reason, + revision: "knowledgefs-unstructured-sandbox-v1", + }, +}); + +describe("sandbox response resource classification", () => { + it.each([ + [413, "memory_bytes", "input"], + [422, "attachments", "input"], + [504, "wall_seconds", "timeout"], + ] as const)( + "classifies known %s %s without transport ambiguity", + async (status, reason, kind) => { + await expect( + classifyUnstructuredResourceResponse( + Response.json(payload(reason), { status }), + new AbortController().signal, + ), + ).resolves.toEqual({ kind, reason }); + }, + ); + + it.each([ + [504, payload("attachments")], + [413, payload("wall_seconds")], + [422, payload("arbitrary-untrusted-text")], + [503, payload("memory_bytes")], + [429, payload("memory_bytes")], + [422, { detail: { ...payload("attachments").detail, revision: "unknown-v2" } }], + [422, { detail: { ...payload("attachments").detail, extra: true } }], + [422, { detail: "attachments" }], + [422, null], + ])("does not reinterpret unknown contracts %s %j", async (status, value) => { + await expect( + classifyUnstructuredResourceResponse( + Response.json(value, { status: status as number }), + new AbortController().signal, + ), + ).resolves.toBeUndefined(); + }); + + it.each([ + [413, "memory_bytes", "provider_input"], + [422, "attachments", "provider_input"], + [504, "wall_seconds", "provider_timeout"], + ] as const)("does not retry confirmed worker rejections %s", async (status, reason, code) => { + const fetchImpl = vi.fn(async () => Response.json(payload(reason), { status })); + const parser = createUnstructuredParserClient({ + endpoint: "https://parser.invalid", + fetch: fetchImpl, + maxRetries: 2, + }); + await expect( + parser.parse({ + body: new TextEncoder().encode("body"), + documentAssetId: "asset", + filename: "sample.pdf", + mimeType: "application/pdf", + version: 1, + }), + ).rejects.toMatchObject({ code, retryable: false, requestOutcomeAmbiguous: false }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); +}); diff --git a/knowledge-fs/packages/parsers/src/unstructured-sandbox-response.ts b/knowledge-fs/packages/parsers/src/unstructured-sandbox-response.ts new file mode 100644 index 00000000000..c50341d99c0 --- /dev/null +++ b/knowledge-fs/packages/parsers/src/unstructured-sandbox-response.ts @@ -0,0 +1,95 @@ +import { + inspectUnstructuredErrorPayload, + isPdfRasterLimitPayload, +} from "./unstructured-response-safety"; + +export type UnstructuredResourceRejection = + | { readonly kind: "pdf" } + | { readonly kind: "input" | "timeout"; readonly reason: string }; + +const sandboxRevision = "knowledgefs-unstructured-sandbox-v1"; +const inputReasons = new Set([ + "input_bytes", + "response_bytes", + "request_metadata_bytes", + "response_metadata_bytes", + "memory_bytes", + "temp_bytes", + "temp_entries", + "cpu_seconds", + "process_limit", + "worker_resource_limit", + "worker_terminated", +]); +const documentReasons = new Set([ + "depth", + "attachments", + "mime_parts", + "decoded_bytes", + "unsupported_attachment", + "msg_invalid", + "ole_invalid", + "mime_invalid", + "archive_entries", + "expanded_bytes", + "archive_path", + "archive_encrypted", + "archive_invalid", + "xml_bytes", + "xml_member_bytes", + "xml_entity", + "xml_depth", + "xml_nodes", + "sheets", + "sheet_extent", + "sheet_cells", + "sheet_cell_reference", + "workbook_cells", + "multipart_required", + "multipart_invalid", + "multipart_fields", + "multipart_files", +]); + +/** Only a confirmed, versioned supervisor rejection can remove transport ambiguity. */ +export function classifyUnstructuredResourceResponse( + response: Response, + signal: AbortSignal, +): Promise { + return inspectUnstructuredErrorPayload(response, signal, [413, 422, 500, 504], (payload) => { + if ((response.status === 500 || response.status === 422) && isPdfRasterLimitPayload(payload)) { + return { kind: "pdf" }; + } + if ( + typeof payload !== "object" || + payload === null || + Object.keys(payload).length !== 1 || + !("detail" in payload) + ) + return undefined; + const detail = payload.detail; + if ( + typeof detail !== "object" || + detail === null || + Object.keys(detail).length !== 3 || + !("code" in detail) || + !("revision" in detail) || + !("reason" in detail) + ) + return undefined; + if ( + detail.code !== "PARSER_RESOURCE_REJECTED" || + detail.revision !== sandboxRevision || + typeof detail.reason !== "string" + ) + return undefined; + if (response.status === 504 && detail.reason === "wall_seconds") + return { kind: "timeout", reason: detail.reason }; + if ( + (response.status === 413 && inputReasons.has(detail.reason)) || + (response.status === 422 && documentReasons.has(detail.reason)) + ) + return { kind: "input", reason: detail.reason }; + return undefined; + }); +} diff --git a/knowledge-fs/scripts/api-image-bundle-smoke.mjs b/knowledge-fs/scripts/api-image-bundle-smoke.mjs index b89cfbfbe1d..5f5cbb52626 100644 --- a/knowledge-fs/scripts/api-image-bundle-smoke.mjs +++ b/knowledge-fs/scripts/api-image-bundle-smoke.mjs @@ -52,6 +52,7 @@ try { const port = await dockerPort(containerId); const imageProcessing = await verifySharpRuntime(containerId); const pdfRasterizer = await verifyPdfRasterizerRuntime(containerId); + const nativeParserWorker = await verifyNativeParserWorkerRuntime(containerId); const health = await waitForHealth(`http://127.0.0.1:${port}/health`); console.log( @@ -61,6 +62,7 @@ try { healthOk: health.ok, imageTag, imageProcessing, + nativeParserWorker, ok: true, pdfRasterizer, port, @@ -75,6 +77,37 @@ try { } } +async function verifyNativeParserWorkerRuntime(containerId) { + const script = ` + const { fork } = await import('node:child_process'); + const child = fork('/workspace/native-parser-worker.mjs', [], { + execArgv: ['--max-old-space-size=256'], serialization: 'advanced', + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], env: {} + }); + const timer = setTimeout(() => { child.kill('SIGKILL'); process.exitCode = 1; }, 10000); + let passed = false; + child.on('message', result => { + passed = result.ok === true && result.artifact.elements[0]?.text.includes('worker-smoke') + && result.artifact.metadata.parserExecution.isolation === 'child-process'; + }); + child.on('error', () => { process.exitCode = 1; }); + child.on('close', (code, signal) => { clearTimeout(timer); if (!passed || code !== 0 || signal !== null) process.exitCode = 1; else console.log('native-worker-ok'); }); + child.send({ kind: 'native-structured', options: {}, input: { + body: new TextEncoder().encode('{"value":"worker-smoke"}'), + documentAssetId: '00000000-0000-4000-8000-000000000001', filename: 'smoke.json', + mimeType: 'application/json', version: 1 + } }); + `; + const { stdout } = await execFileAsync( + docker, + ["exec", containerId, "node", "--input-type=module", "--eval", script], + { timeout: 15_000, maxBuffer: 65_536 }, + ); + if (!stdout.includes("native-worker-ok")) + throw new Error("Native parser worker bundle smoke failed"); + return { ok: true, isolation: "child-process" }; +} + async function verifyPdfRasterizerRuntime(containerId) { const [ { stderr, stdout }, @@ -82,6 +115,7 @@ async function verifyPdfRasterizerRuntime(containerId) { materializationConcurrencyResult, fallbackConcurrencyResult, fallbackReservedBytesResult, + pdfInfoResult, ] = await Promise.all([ execFileAsync(docker, ["exec", containerId, "pdftoppm", "-v"]), execFileAsync(docker, [ @@ -108,8 +142,12 @@ async function verifyPdfRasterizerRuntime(containerId) { "printenv", "KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_RESERVED_BYTES", ]), + execFileAsync(docker, ["exec", containerId, "pdfinfo", "-v"]), ]); const version = `${stdout}${stderr}`.trim(); + if (!/^pdfinfo version\b/m.test(`${pdfInfoResult.stdout}${pdfInfoResult.stderr}`)) { + throw new Error("Poppler PDF safety inspector is unavailable in the API image"); + } const maxConcurrency = Number(concurrencyResult.stdout.trim()); const materializationMaxConcurrency = Number(materializationConcurrencyResult.stdout.trim()); const fallbackMaxConcurrency = Number(fallbackConcurrencyResult.stdout.trim()); @@ -157,6 +195,20 @@ async function verifySharpRuntime(containerId) { if (data.byteLength < 1 || info.format !== "png" || info.width !== 2 || info.height !== 1) { throw new Error("sharp native runtime did not produce the expected PNG"); } + const { fork } = await import('node:child_process'); + const isolated = await new Promise((resolve, reject) => { + const child = fork('/workspace/image-variant-worker.mjs', [], { + execArgv: ['--max-old-space-size=128'], serialization: 'advanced', + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], env: { VIPS_CONCURRENCY: '1' } + }); + let result; + const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('Image worker deadline')); }, 10000); + child.on('message', message => { result = message; }); + child.on('error', reject); + child.on('close', code => { clearTimeout(timer); if (code !== 0 || result?.ok !== true) reject(new Error('Image worker failed')); else resolve(result); }); + child.send({ options: { analysisMaxDimension: 2048 }, input: { body: data, contentType: 'image/png', elementId: 'smoke' } }); + }); + if (isolated.variants.length !== 2 || !isolated.variants.some(v => v.name === 'analysis')) throw new Error('Missing analysis variant'); console.log(JSON.stringify({ format: info.format, height: info.height, diff --git a/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs b/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs index 1ef14e3504f..ab9d6978fba 100644 --- a/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs +++ b/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs @@ -23,6 +23,7 @@ test("isolated API bundle smoke starts the container and checks compute health", assert.match(smokeScript, /difyDependencyConnected/); assert.match(smokeScript, /verifyPdfRasterizerRuntime/); assert.match(smokeScript, /pdftoppm/); + assert.match(smokeScript, /"pdfinfo", "-v"/); assert.match(smokeScript, /KNOWLEDGE_PDF_RASTERIZER_MAX_CONCURRENCY/); assert.match(smokeScript, /KNOWLEDGE_DOCUMENT_MATERIALIZATION_MAX_CONCURRENCY/); assert.match(smokeScript, /materializationMaxConcurrency !== 2/); @@ -30,6 +31,9 @@ test("isolated API bundle smoke starts the container and checks compute health", assert.match(smokeScript, /fallbackMaxReservedBytes !== 31457280/); assert.match(smokeScript, /maxConcurrency !== 2/); assert.match(smokeScript, /verifySharpRuntime/); + assert.match(smokeScript, /verifyNativeParserWorkerRuntime/); + assert.match(smokeScript, /native-parser-worker\.mjs/); + assert.match(smokeScript, /image-variant-worker\.mjs/); assert.match(smokeScript, /await import\("sharp"\)/); assert.match(smokeScript, /sharp\.versions\.vips/); assert.match(smokeScript, /imageProcessing/); @@ -41,6 +45,16 @@ test("isolated API bundle smoke starts the container and checks compute health", test("production API image carries and executes the target platform sharp runtime", () => { assert.equal(apiPackageJson.dependencies.sharp, "^0.35.3"); assert.match(apiPackageJson.scripts["build:prod"], /--external:sharp/); + assert.match(apiPackageJson.scripts["build:prod"], /src\/native-parser-worker\.ts/); + assert.match(apiPackageJson.scripts["build:prod"], /src\/image-variant-worker\.ts/); + assert.match( + apiDockerfile, + /COPY --from=builder .*native-parser-worker\.mjs \.\/native-parser-worker\.mjs/, + ); + assert.match( + apiDockerfile, + /COPY --from=builder .*image-variant-worker\.mjs \.\/image-variant-worker\.mjs/, + ); assert.match(apiDockerfile, /realpath apps\/api\/node_modules\/sharp/); assert.match(apiDockerfile, /cp -LR/); assert.match(apiDockerfile, /COPY --from=builder \/runtime\/node_modules \.\/node_modules/); @@ -48,6 +62,10 @@ test("production API image carries and executes the target platform sharp runtim assert.match(apiDockerfile, /sharp native runtime smoke failed/); }); +test("native worker smoke requires a clean process exit as well as a response", () => { + assert.match(smokeScript, /!passed \|\| code !== 0 \|\| signal !== null/); +}); + test("production API image carries and executes the Poppler PDF rasterizer", () => { assert.match(apiDockerfile, /apt-get install --yes --no-install-recommends poppler-utils/); assert.match(apiDockerfile, /KNOWLEDGE_PDF_RASTERIZER=poppler/); @@ -65,4 +83,6 @@ test("production API image carries and executes the Poppler PDF rasterizer", () assert.match(apiDockerfile, /KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_RESERVED_BYTES=31457280/); assert.match(apiDockerfile, /command -v pdftoppm/); assert.match(apiDockerfile, /pdftoppm -v/); + assert.match(apiDockerfile, /command -v pdfinfo/); + assert.match(apiDockerfile, /pdfinfo -v/); }); diff --git a/knowledge-fs/scripts/compose-apps.test.mjs b/knowledge-fs/scripts/compose-apps.test.mjs index df71a45445d..d1aa3d91df5 100644 --- a/knowledge-fs/scripts/compose-apps.test.mjs +++ b/knowledge-fs/scripts/compose-apps.test.mjs @@ -252,6 +252,8 @@ test("local parser isolates every bounded heavy document workload", () => { ); assert.match(localUnstructured, /^ {10}cpus: "4\.0"$/m); assert.match(localUnstructured, /^ {10}memory: 6G$/m); + assert.match(localUnstructured, /^ {6}PDF_RENDER_DPI: "350"$/m); + assert.match(localUnstructured, /^ {6}PDF_RENDER_MAX_PIXELS_PER_PAGE: "25000000"$/m); assert.match(localEnvExample, /^UNSTRUCTURED_MAX_CONCURRENCY=2$/m); assert.match(localEnvExample, /^UNSTRUCTURED_MAX_INPUT_BYTES=15728640$/m); assert.match(localEnvExample, /^UNSTRUCTURED_REQUEST_TIMEOUT_MS=600000$/m); @@ -470,6 +472,7 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", ( "KNOWLEDGE_RESEARCH_MAX_RERANK_CANDIDATES", "KNOWLEDGE_QUERY_IMAGE_EXPANSION_TIMEOUT_MS", "UNSTRUCTURED_API_URL", + "UNSTRUCTURED_BACKEND_REVISION", "UNSTRUCTURED_API_KEY", "UNSTRUCTURED_MAX_CONCURRENCY", "UNSTRUCTURED_HEAVY_MAX_CONCURRENCY", @@ -531,12 +534,21 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", ( test("KnowledgeFS has an isolated page-parallel parser without changing legacy Unstructured", () => { assert.deepEqual(envVariableNames(difyKnowledgeFsUnstructuredEnv), [ + "PDF_RENDER_DPI", + "PDF_RENDER_MAX_PIXELS_PER_PAGE", "UNSTRUCTURED_PARALLEL_MODE_ENABLED", "UNSTRUCTURED_PARALLEL_MODE_URL", "UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE", "UNSTRUCTURED_PARALLEL_MODE_THREADS", "UNSTRUCTURED_PARALLEL_RETRY_ATTEMPTS", ]); + for (const configuration of [ + difyKnowledgeFsUnstructuredEnv, + difyKnowledgeFsUnstructuredServiceDefaults, + ]) { + assert.match(configuration, /^PDF_RENDER_DPI=350$/m); + assert.match(configuration, /^PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000$/m); + } assert.match(difyKnowledgeFsUnstructuredEnv, /^UNSTRUCTURED_PARALLEL_MODE_ENABLED=true$/m); assert.match( difyKnowledgeFsUnstructuredEnv, diff --git a/knowledge-fs/scripts/github-actions-workflow.test.mjs b/knowledge-fs/scripts/github-actions-workflow.test.mjs index daa779e6373..50af0580eba 100644 --- a/knowledge-fs/scripts/github-actions-workflow.test.mjs +++ b/knowledge-fs/scripts/github-actions-workflow.test.mjs @@ -37,6 +37,25 @@ function qualityStep(name) { return step; } +test("dedicated parser process and attachment regressions run with locked Python dependencies", () => { + const check = qualityStep("Test dedicated parser sandbox"); + assert.equal(check["working-directory"], "."); + assert.match(check.run, /uv run --project api python -m unittest discover/); + assert.match(check.env.PYTHONPATH, /services\/unstructured-sandbox/); + assert.ok( + qualitySteps.indexOf(check) > + qualitySteps.indexOf(qualityStep("Install Dify contract dependencies")), + ); +}); + +test("PDF geometry integration tests have the production metadata inspector available", () => { + const inspector = qualityStep("Install PDF metadata inspector"); + assert.match(inspector.run, /apt-get install --yes --no-install-recommends poppler-utils/); + assert.match(inspector.run, /pdfinfo -v/); + const checks = qualitySteps.findIndex((step) => step.run === "pnpm check"); + assert.ok(qualitySteps.indexOf(inspector) < checks); +}); + test("root workflow always emits a stable PR and merge-queue gate", () => { assert.match(workflow, /^name: KnowledgeFS CI$/m); assert.match(workflow, /^ {2}pull_request:$/m); diff --git a/knowledge-fs/scripts/parser-benchmark.mjs b/knowledge-fs/scripts/parser-benchmark.mjs new file mode 100644 index 00000000000..d211e5054cf --- /dev/null +++ b/knowledge-fs/scripts/parser-benchmark.mjs @@ -0,0 +1,122 @@ +import { pathToFileURL } from "node:url"; + +/** Marker coverage is a narrow golden assertion, not an OCR/retrieval quality score. */ +export function summarizeParserSamples(samples) { + if (!samples.length) throw new Error("Parser benchmark requires samples"); + const times = samples.map((sample) => sample.elapsedMs).sort((a, b) => a - b); + return { + p50Ms: times[Math.ceil(times.length * 0.5) - 1], + p95Ms: times[Math.ceil(times.length * 0.95) - 1], + peakRssKiB: Math.max(...samples.map((sample) => sample.peakRssKiB)), + maxOutputAmplification: Math.max( + ...samples.map((sample) => sample.outputBytes / Math.max(1, sample.inputBytes)), + ), + minimumMarkerCoverage: Math.min(...samples.map((sample) => sample.markerCoverage)), + samples: samples.length, + }; +} + +export function nativeGoldenFixtures(rows) { + if (!Number.isSafeInteger(rows) || rows < 1 || rows > 10_000) + throw new Error("Golden fixture rows must be between 1 and 10000"); + const marker = "知识解析证据"; + const records = Array.from({ length: rows }, (_, index) => ({ + index, + text: `${marker}${index}【完】`, + })); + const fixture = ( + extension, + mimeType, + text, + markers = [`${marker}0【完】`, `${marker}${rows - 1}【完】`], + ) => ({ extension, mimeType, body: new TextEncoder().encode(text), markers }); + return [ + fixture("md", "text/markdown", records.map((record) => `> ${record.text}\n`).join("\n")), + fixture("mdx", "text/mdx", records.map((record) => `

${record.text}

`).join("\n\n")), + fixture("txt", "text/plain", records.map((record) => record.text).join("\n\n")), + fixture( + "html", + "text/html", + `${records.map((record) => `

${record.text}

`).join("")}`, + ), + fixture("json", "application/json", JSON.stringify(records)), + fixture( + "jsonl", + "application/x-ndjson", + records.map((record) => JSON.stringify(record)).join("\n"), + ), + fixture( + "csv", + "text/csv", + `index,text\n${records.map((record) => `${record.index},${record.text}`).join("\n")}`, + ), + fixture( + "xml", + "application/xml", + `${records.map((record) => `${record.text}`).join("")}`, + ), + fixture( + "properties", + "text/x-java-properties", + records.map((record) => `key${record.index}=${record.text}`).join("\n"), + ), + fixture( + "vtt", + "text/vtt", + `WEBVTT\n\n${records.map((record) => `${record.index}\n00:00:00.000 --> 00:00:01.000\n${record.text}`).join("\n\n")}\n`, + ), + ]; +} + +export async function runNativeParserBenchmark({ repetitions = 3, sizes = [10, 1000] } = {}) { + if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > 20) + throw new Error("Repetitions must be between 1 and 20"); + const { createApiDocumentParser } = await import("../apps/api/src/parser-options.ts"); + const parser = createApiDocumentParser({ env: {} }); + const results = []; + for (const rows of sizes) { + for (const fixture of nativeGoldenFixtures(rows)) { + const samples = []; + for (let iteration = 0; iteration < repetitions; iteration++) { + const started = performance.now(); + const artifact = await parser.parse({ + body: fixture.body, + documentAssetId: "00000000-0000-4000-8000-000000000001", + filename: `golden.${fixture.extension}`, + mimeType: fixture.mimeType, + version: 1, + }); + const elapsedMs = performance.now() - started; + const text = artifact.elements.map((element) => element.text ?? "").join("\n"); + const markerCoverage = + fixture.markers.filter((marker) => text.includes(marker)).length / fixture.markers.length; + if (markerCoverage !== 1) + throw new Error(`Golden marker missing: ${fixture.extension}/${rows}`); + samples.push({ + elapsedMs, + inputBytes: fixture.body.byteLength, + outputBytes: artifact.metadata.parserExecution.outputBytes, + peakRssKiB: artifact.metadata.parserExecution.peakRssKiB, + markerCoverage, + }); + } + results.push({ + format: fixture.extension, + rows, + inputBytes: fixture.body.byteLength, + ...summarizeParserSamples(samples), + }); + } + } + return { + benchmark: "parser-native-resource-golden-v1", + environment: { node: process.version, platform: process.platform, arch: process.arch }, + scope: "real-native-parsers-through-isolated-api-adapter", + excludes: ["Unstructured image", "OCR accuracy", "retrieval recall", "production capacity"], + results, + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + console.log(JSON.stringify(await runNativeParserBenchmark(), null, 2)); +} diff --git a/knowledge-fs/scripts/parser-benchmark.test.mjs b/knowledge-fs/scripts/parser-benchmark.test.mjs new file mode 100644 index 00000000000..c16f3555bea --- /dev/null +++ b/knowledge-fs/scripts/parser-benchmark.test.mjs @@ -0,0 +1,40 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { nativeGoldenFixtures, summarizeParserSamples } from "./parser-benchmark.mjs"; + +test("resource benchmark contracts are part of the standard verification command", () => { + const scripts = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ).scripts; + assert.ok(scripts.check.includes("pnpm parser:regression:test")); + assert.ok(scripts["parser:regression:test"].includes("parser-benchmark.test.mjs")); + assert.ok(scripts["benchmark:parsers"].includes("parser-benchmark.mjs")); +}); + +test("benchmark keeps latency, memory, amplification and marker coverage separate", () => { + const summary = summarizeParserSamples([ + { elapsedMs: 2, inputBytes: 10, outputBytes: 30, peakRssKiB: 100, markerCoverage: 1 }, + { elapsedMs: 8, inputBytes: 10, outputBytes: 40, peakRssKiB: 200, markerCoverage: 0.5 }, + { elapsedMs: 4, inputBytes: 10, outputBytes: 20, peakRssKiB: 150, markerCoverage: 1 }, + ]); + assert.equal(summary.p50Ms, 4); + assert.equal(summary.p95Ms, 8); + assert.equal(summary.peakRssKiB, 200); + assert.equal(summary.maxOutputAmplification, 4); + assert.equal(summary.minimumMarkerCoverage, 0.5); + assert.throws(() => summarizeParserSamples([]), /samples/); +}); + +test("golden corpus covers all native upload families and non-ASCII content with bounded fixtures", () => { + const fixtures = nativeGoldenFixtures(20); + assert.deepEqual( + fixtures.map((fixture) => fixture.extension), + ["md", "mdx", "txt", "html", "json", "jsonl", "csv", "xml", "properties", "vtt"], + ); + for (const fixture of fixtures) { + assert.ok(fixture.body.byteLength < 100_000); + assert.ok(fixture.markers.length >= 1); + } + assert.throws(() => nativeGoldenFixtures(1_000_001), /rows/); +}); diff --git a/knowledge-fs/scripts/pdf-thumbnail-benchmark.mjs b/knowledge-fs/scripts/pdf-thumbnail-benchmark.mjs new file mode 100644 index 00000000000..487f269f70a --- /dev/null +++ b/knowledge-fs/scripts/pdf-thumbnail-benchmark.mjs @@ -0,0 +1,81 @@ +import { createRequire } from "node:module"; +import { pathToFileURL } from "node:url"; + +// Small deterministic source fixture; never render the user-supplied oversized banner here. +export function thumbnailGoldenPdf() { + const content = Array.from( + { length: 18 }, + (_, index) => + `BT /F1 ${6 + (index % 3)} Tf 12 ${275 - index * 14} Td (Table row ${index}: 0123456789 ABC xyz) Tj ET\n0.5 w 10 ${272 - index * 14} m 205 ${272 - index * 14} l S`, + ).join("\n"); + const objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 216 288] /Resources << /Font << /F1 5 0 R >> >> /Contents 4 0 R >>", + `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`, + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ]; + let pdf = "%PDF-1.7\n"; + const offsets = []; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.byteLength(pdf)); + pdf += `${index + 1} 0 obj\n${object}\nendobj\n`; + } + const start = Buffer.byteLength(pdf); + pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n${offsets.map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`).join("")}trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${start}\n%%EOF\n`; + return new TextEncoder().encode(pdf); +} + +export async function runPdfThumbnailBenchmark() { + const sharp = createRequire(new URL("../apps/api/package.json", import.meta.url))("sharp"); + const { createPopplerPdfRasterizer } = await import( + "../packages/api/src/document-pdf-rasterizer.ts" + ); + const request = { documentBody: thumbnailGoldenPdf(), elementId: "golden-page", pageNumber: 1 }; + const baseline = createPopplerPdfRasterizer({ dpi: 144, thumbnailDpi: 48 }); + // Equal DPI uses the same rasterized page; derive the preview from its full-resolution crop. + const onePass = createPopplerPdfRasterizer({ dpi: 144, thumbnailDpi: 144 }); + const samples = []; + for (let iteration = 0; iteration < 5; iteration++) { + let started = performance.now(); + const original = await baseline.render(request); + const twoRenderMs = performance.now() - started; + const expected = original?.variants?.thumbnail?.body; + if (!expected) throw new Error("Baseline thumbnail is missing"); + const geometry = await sharp(expected).metadata(); + started = performance.now(); + const candidate = await onePass.render(request); + if (!candidate) throw new Error("Candidate image is missing"); + const resized = await sharp(candidate.body, { limitInputPixels: 1_000_000 }) + .resize(geometry.width, geometry.height) + .png() + .toBuffer(); + const oneRenderAndResizeMs = performance.now() - started; + const a = await sharp(expected).removeAlpha().raw().toBuffer(); + const b = await sharp(resized).removeAlpha().raw().toBuffer(); + if (a.length !== b.length) throw new Error("Thumbnail geometry mismatch"); + let squaredError = 0; + for (let index = 0; index < a.length; index++) squaredError += (a[index] - b[index]) ** 2; + const mse = squaredError / a.length; + samples.push({ + twoRenderMs, + oneRenderAndResizeMs, + equalPixels: mse === 0, + psnrDb: mse === 0 ? null : 10 * Math.log10(255 ** 2 / mse), + originalBytes: expected.byteLength, + derivedBytes: resized.byteLength, + }); + } + return { + benchmark: "pdf-thumbnail-single-raster-evaluation-v1", + fixture: "216x288pt ASCII small text/table grid", + maxRasterPixels: 248832, + samples, + decision: + "Keep production two-render default: resampling is not pixel-equivalent; OCR/non-Latin/diagram quality and process-tree RSS need the wider provider golden corpus.", + excludes: ["OCR accuracy", "non-Latin text", "process-tree peak RSS", "production throughput"], + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) + console.log(JSON.stringify(await runPdfThumbnailBenchmark(), null, 2)); diff --git a/knowledge-fs/scripts/pdf-thumbnail-benchmark.test.mjs b/knowledge-fs/scripts/pdf-thumbnail-benchmark.test.mjs new file mode 100644 index 00000000000..f777ee5fc92 --- /dev/null +++ b/knowledge-fs/scripts/pdf-thumbnail-benchmark.test.mjs @@ -0,0 +1,13 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { thumbnailGoldenPdf } from "./pdf-thumbnail-benchmark.mjs"; + +test("thumbnail benchmark uses a small deterministic PDF, not an unbounded user document", () => { + const first = thumbnailGoldenPdf(); + assert.deepEqual(first, thumbnailGoldenPdf()); + assert.ok(first.byteLength < 10_000); + const text = new TextDecoder().decode(first); + assert.ok(text.includes("/MediaBox [0 0 216 288]")); + const offset = Number(/startxref\n(\d+)/.exec(text)[1]); + assert.equal(text.slice(offset, offset + 4), "xref"); +}); diff --git a/knowledge-fs/services/unstructured-sandbox/.coveragerc b/knowledge-fs/services/unstructured-sandbox/.coveragerc new file mode 100644 index 00000000000..792a30550e4 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/.coveragerc @@ -0,0 +1,9 @@ +[run] +branch = true +source = kfs_sandbox +parallel = true +patch = subprocess + +[report] +fail_under = 90 +show_missing = true diff --git a/knowledge-fs/services/unstructured-sandbox/.dockerignore b/knowledge-fs/services/unstructured-sandbox/.dockerignore new file mode 100644 index 00000000000..e93b5aab2c8 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/.dockerignore @@ -0,0 +1,10 @@ +* +!Dockerfile +!kfs_sandbox +!kfs_sandbox/*.py +!golden.py +!build-converter-manifest.py +!bin +!bin/* +!fixtures +!fixtures/* diff --git a/knowledge-fs/services/unstructured-sandbox/Dockerfile b/knowledge-fs/services/unstructured-sandbox/Dockerfile new file mode 100644 index 00000000000..1480067b45b --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/Dockerfile @@ -0,0 +1,20 @@ +FROM downloads.unstructured.io/unstructured-io/unstructured-api@sha256:0df934a22e4e893cf15e7aeaf35c463ecc75937758a83099aefdc13041619a1d + +# The image is deliberately opt-in until its real-format golden contract passes. Reuse +# the provider's installed libraries/models; never pip-install an unpinned runtime here. +USER 1000:1000 +RUN python3 -c 'from importlib.metadata import version; assert version("unstructured") == "0.22.18"; import uvicorn, psutil, oxmsg; from prepline_general.api.app import app' +COPY --chown=1000:1000 kfs_sandbox /opt/kfs-sandbox/kfs_sandbox +COPY --chown=1000:1000 golden.py /opt/kfs-sandbox/golden.py +COPY --chown=1000:1000 fixtures /opt/kfs-sandbox/fixtures +USER root +COPY bin /opt/kfs-sandbox/bin +COPY build-converter-manifest.py /opt/kfs-sandbox/build-converter-manifest.py +RUN python3 /opt/kfs-sandbox/build-converter-manifest.py && chmod 0555 /opt/kfs-sandbox/bin/soffice /opt/kfs-sandbox/bin/pandoc +ENV PYTHONPATH=/opt/kfs-sandbox:/home/notebook-user +ENV PYTHONDONTWRITEBYTECODE=1 +ENV UNSTRUCTURED_PARALLEL_MODE_ENABLED=false +ENV PATH=/opt/kfs-sandbox/bin:${PATH} +ENV PYPANDOC_PANDOC=/opt/kfs-sandbox/bin/pandoc +USER 1000:1000 +ENTRYPOINT ["python3", "-m", "kfs_sandbox"] diff --git a/knowledge-fs/services/unstructured-sandbox/README.md b/knowledge-fs/services/unstructured-sandbox/README.md new file mode 100644 index 00000000000..51fee99e1ae --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/README.md @@ -0,0 +1,149 @@ +# Optional request-isolated Unstructured runtime + +Status: **implemented and locally process-tested; not approved for production activation**. +The default deployment is unchanged. This is an override of the existing dedicated +KnowledgeFS Unstructured service, not another per-format service. Legacy Dify's +`unstructured` service is unaffected. + +## Boundary and contract + +The single supervisor accepts `POST /general/v0/general`, reserves a bounded shared slot, +spools a bounded multipart body, and starts a disposable process group. That process applies +resource limits, checks nested mail and Office archives, and replays the **unchanged** ASGI +request into the pinned original API. Its original API-key check, form interpretation, +response status, and response body remain in that worker. No upstream monkeypatch is used. +All KnowledgeFS API replicas pointing to this one service share its admission queue. +Multiple sandbox replicas still multiply capacity; there is no distributed semaphore. + +This internal endpoint accepts exactly one file, at most 128 form parts, and no arbitrary +upstream routes. `/healthcheck` is **supervisor liveness**, not proof that models or every +format are ready. The real-format golden gate below is required for readiness validation. + +The optional worker disables upstream HTTP self-recursive PDF parallelism to avoid waiting +on its own service-wide admission slot. Text/layout semantics are retained, but long-PDF +throughput may decrease. This must be measured before replacing the default page-parallel +service. No production default is changed to hide this trade-off. + +## Resource protection + +| Boundary | Default | Enforcement | +| --- | --- | --- | +| Accepted / waiting requests | 1 / 8 | Atomic supervisor admission, 30 s queue deadline | +| Input / response / metadata | 51 MiB / 32 MiB / 64 KiB | Bounded spool / worker output / metadata | +| Request wall time | 2,400 s | Supervisor kill on expiry; upstream client disconnect also kills | +| Worker address space / CPU / file size | 32 GiB / 2,400 CPU s / 256 MiB | Linux inherited rlimits before provider imports | +| Process-tree RSS / processes / temporary bytes | 4 GiB / 64 / 512 MiB | 50 ms sampled supervisor checks | +| Container RAM / PIDs / temporary filesystem | 6 GiB / 192 / 1 GiB | Docker hard limits and tmpfs; read-only root | +| Attachment depth / count / decoded bytes | 8 / 128 / 128 MiB | Shared admission tree, not fresh limits per attachment | +| Archive entries / expanded bytes | 4,096 / 512 MiB | Declared precheck and actual incremental inflation | +| XML total / member / depth / nodes | 64 MiB / 16 MiB / 128 / 1,000,000 | Streaming Expat admission, no entity expansion | +| Worksheet rows / columns / rectangle | 100,000 / 16,384 / 250,000 cells | Actual explicit and implicit cells; merged ranges included | +| All worksheet rectangles / sheets | 500,000 cells / 256 | Shared across nested attachments | + +The supervisor kills the request's process group after success as well as failure, and +observed descendants that changed sessions. Temporary files are removed before admission +is released; a stalled response consumer cannot hold a slot beyond the deadline. +Unknown attachment extensions are **rejected explicitly**, not silently omitted. This is +all-or-nothing admission, not a partial-success attachment extraction implementation. + +These limits are not a native-code exploit sandbox. Linux `RLIMIT_AS` limits virtual address +space, not RSS; process-tree RSS/CPU/disk checks are sampled and can overshoot between polls. +`RLIMIT_FSIZE` is per file, not a disk quota; aggregate hard tmpfs/RAM/PID limits belong to +the container, not each request. A process that immediately detaches before being observed +is not proven captured by the process-tree monitor. Do not advertise hard per-request +cgroup isolation or complete protection against arbitrary native-code compromise. + +Conversion processes and in-process legacy XLS readers inherit the request resource boundary. +Version-scoped executable adapters intercept the pinned Unstructured DOC/PPT LibreOffice and +RTF/EPUB/ODT Pandoc invocations. The original executable paths are captured into an immutable +build-time manifest before wrapper installation; no Python parser function is monkeypatched. +Generated DOCX/PPTX/HTML stays in a private directory until structural admission succeeds. +Office archives use the same archive/XML/worksheet guards; HTML has node/depth and rectangular +table limits, including rowspan/colspan. A file-locked ledger shares the root/mail admission +budget across all conversions. Products and converter stdout are bounded to 64 MiB; stderr +is bounded to 64 KiB. Validated files are atomically published without overwriting an existing +file, and validated HTML stdout is then returned unchanged. + +Unknown invocation shapes, malformed/oversized products, signal-killed converters and budget +failures reject the whole request through a sticky ledger, even if upstream catches an +attachment exception. Ordinary positive converter error statuses retain upstream behavior. +The adapters accept only inspected argument shapes, including Pandoc version/format probes; +the actual pinned image's converter discovery and every real-format fixture remain mandatory +activation gates. Local fake-executable tests alone do not prove image compatibility. + +## Local tests + +From the Dify repository root, using the existing API development environment: + +```sh +PYTHONPATH=knowledge-fs/services/unstructured-sandbox uv run --project api python -m unittest discover -s knowledge-fs/services/unstructured-sandbox/tests +uv run --project api ruff check knowledge-fs/services/unstructured-sandbox +``` + +Runtime tests need Python 3.12, `psutil`, `uvicorn`, `python-oxmsg`, and `olefile`; they do not +require Unstructured, Torch, Docker, or production access. These dependencies are present in +the existing API environment; the service image reuses its pinned dependencies. For coverage, +use coverage.py >= 7.10 with the +`.coveragerc` subprocess patch, an isolated `COVERAGE_FILE` path, then `coverage combine`. +The latest local run covers 96.90% statements and 92.05% branches (80 tests). Golden-runner +unit tests validate fixture builders; they do not pretend to validate the real provider. + +## Real-image gate (requires a working Docker daemon) + +Build and run only in a disposable local/staging environment first. The image build checks +the pinned dependency's actual version and original ASGI import, and fails if startup, +shutdown or a custom lifespan hook requires a replay lifecycle this worker does not provide. +The inspected upstream app initializes at import and has no such hooks. The current base +digest is unchanged, and no new unpinned Python dependency is installed during the build. + +```sh +docker compose --env-file knowledge-fs/infra/local/.env.example -f knowledge-fs/infra/local/compose.yaml -f knowledge-fs/infra/local/compose.unstructured-sandbox.yaml build unstructured +docker compose --env-file knowledge-fs/infra/local/.env.example -f knowledge-fs/infra/local/compose.yaml -f knowledge-fs/infra/local/compose.unstructured-sandbox.yaml up -d unstructured +docker compose --env-file knowledge-fs/infra/local/.env.example -f knowledge-fs/infra/local/compose.yaml -f knowledge-fs/infra/local/compose.unstructured-sandbox.yaml exec unstructured python3 /opt/kfs-sandbox/golden.py +``` + +The golden runner generates tiny synthetic PDF, DOC/DOCX, PPT/PPTX, XLS/XLSX, ODT, EPUB, +RTF and nested EML fixtures inside the container. It also uses a bundled Apache-licensed +MSG from the exact Unstructured 0.22.18 source revision, with a checked SHA-256 and an explicit +attachment-text assertion; source and license are in `fixtures/`. The real local oxmsg parser +has validated that fixture's attachment, including renamed `.xls` content detection. +The gate verifies actual returned evidence and that a tiny sparse XLSX email attachment is +rejected before pandas processing. It refuses non-loopback URLs and never uploads user +documents. Optional `--msg-fixture /path/to/synthetic.msg` accepts an alternative ≤1 MiB +fixture whose attachment contains `KnowledgeFS Golden Evidence`. A missing/corrupt bundled +fixture fails rather than silently skipping MSG. Trusted tiny fixture generation uses the +manifest's original executables outside request admission; HTTP parsing uses the wrappers. + +Required before activation: + +1. Build and pass all real-format fixtures under the read-only/tempfs limits. Confirm the + pinned image's model caches and LibreOffice configuration work without unexpected writes. +2. Verify the converter adapters with the actual image's Pandoc discovery, DOC/PPT conversions, + RTF/EPUB HTML stdout and ODT DOCX output. Unknown shapes fail closed, never bypass admission. +3. Benchmark ordinary documents and long PDFs against the unchanged default: cold-start + latency, p50/p95, peak RSS, table/image/text recall, and cancellation behavior. +4. Run a staging canary and promote an immutable built sandbox image. Set the API's + `UNSTRUCTURED_BACKEND_REVISION` to the tested sandbox image/policy revision, so old + checkpoints are not mistaken for the new semantic implementation. + +Hard per-request RAM/PID/tmp quotas additionally require delegated cgroup/storage enforcement +and deployment authority; this implementation does not claim those protections. + +Only after these gates, the production override is +`docker/knowledge-fs-unstructured-sandbox.compose.yaml`, merged with the existing +`docker/docker-compose.yaml`. Its hostname remains `knowledge_fs_unstructured:8000`. +The override itself does not deploy anything. + +## Source basis + +The boundary was checked against the pinned [EML partitioner](https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/partition/email.py), +[MSG partitioner and public oxmsg access](https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/partition/msg.py), +[Office conversion implementation](https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/partition/common/common.py), +[RTF/EPUB conversion](https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/file_utils/file_conversion.py), +[ODT conversion](https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/partition/odt.py), +[pypandoc argument construction](https://github.com/JessicaTegner/pypandoc/blob/v1.15/pypandoc/__init__.py), +and upstream [ASGI API](https://github.com/Unstructured-IO/unstructured-api/blob/main/prepline_general/api/app.py). +The pypandoc source describes the inspected CLI family, not an assertion that its exact version +inside the unavailable image was inspected. +The latter source explains the boundary only; the actual image import/build and golden +contracts, not a moving branch, are the compatibility authority. diff --git a/knowledge-fs/services/unstructured-sandbox/bin/pandoc b/knowledge-fs/services/unstructured-sandbox/bin/pandoc new file mode 100644 index 00000000000..03d6364df0b --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/bin/pandoc @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from kfs_sandbox.converter_cli import main + +main("pandoc") diff --git a/knowledge-fs/services/unstructured-sandbox/bin/soffice b/knowledge-fs/services/unstructured-sandbox/bin/soffice new file mode 100644 index 00000000000..a45279449f0 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/bin/soffice @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 +from kfs_sandbox.converter_cli import main + +main("soffice") diff --git a/knowledge-fs/services/unstructured-sandbox/build-converter-manifest.py b/knowledge-fs/services/unstructured-sandbox/build-converter-manifest.py new file mode 100644 index 00000000000..67200901d1e --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/build-converter-manifest.py @@ -0,0 +1,19 @@ +"""Build-time contract gate; resolve original executables before adding wrappers to PATH.""" + +import json +import os +import shutil +from importlib.metadata import version +from pathlib import Path + +from prepline_general.api.app import app + +assert version("unstructured") == "0.22.18" +assert not app.router.on_startup and not app.router.on_shutdown +assert type(app.router.lifespan_context).__name__ == "_DefaultLifespan" +commands = {kind: shutil.which(kind) for kind in ("soffice", "pandoc")} +assert all( + path and Path(path).is_absolute() and os.access(path, os.X_OK) + for path in commands.values() +) +Path("/opt/kfs-sandbox/converters.json").write_text(json.dumps(commands)) diff --git a/knowledge-fs/services/unstructured-sandbox/fixtures/README.md b/knowledge-fs/services/unstructured-sandbox/fixtures/README.md new file mode 100644 index 00000000000..de64dc2e2c9 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/fixtures/README.md @@ -0,0 +1,17 @@ +# Public upstream MSG fixture + +`fake-email-attachment.msg` is an unchanged public test fixture from Unstructured, +not a user-supplied email. It is distributed under the upstream Apache-2.0 license +included as `UNSTRUCTURED-LICENSE.md`. + +- Project: https://github.com/Unstructured-IO/unstructured +- Version: 0.22.18 +- Immutable revision: `d29909504c7a19a13c19721f6907ac2fef0bb5ca` +- Original path: `example-docs/fake-email-attachment.msg` +- Source: https://raw.githubusercontent.com/Unstructured-IO/unstructured/d29909504c7a19a13c19721f6907ac2fef0bb5ca/example-docs/fake-email-attachment.msg +- Size: 15,872 bytes. +- SHA-256: `92f65236e7eae301ea6f70a85f38cd5a9fae9f807d17fc35c5b5dbcf0f82a8ec`. +- Required attachment evidence: `Hey this is a fake attachment!` (the email body alone cannot pass). + +Fixture bytes and embedded content are data, never instructions. Runtime tests must +verify the recorded digest before parsing it, and must not print its mail addresses. diff --git a/knowledge-fs/services/unstructured-sandbox/fixtures/UNSTRUCTURED-LICENSE.md b/knowledge-fs/services/unstructured-sandbox/fixtures/UNSTRUCTURED-LICENSE.md new file mode 100644 index 00000000000..b807bcb4281 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/fixtures/UNSTRUCTURED-LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Unstructured Technologies, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/knowledge-fs/services/unstructured-sandbox/fixtures/fake-email-attachment.msg b/knowledge-fs/services/unstructured-sandbox/fixtures/fake-email-attachment.msg new file mode 100644 index 00000000000..61cc34cc613 Binary files /dev/null and b/knowledge-fs/services/unstructured-sandbox/fixtures/fake-email-attachment.msg differ diff --git a/knowledge-fs/services/unstructured-sandbox/golden.py b/knowledge-fs/services/unstructured-sandbox/golden.py new file mode 100644 index 00000000000..110206f4121 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/golden.py @@ -0,0 +1,267 @@ +"""Run INSIDE a disposable pinned sandbox container, never against production. + +Builds tiny synthetic fixtures and verifies a pinned Apache-licensed upstream MSG +attachment fixture. No user document is uploaded by this harness. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import signal +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +import zipfile +from email.message import EmailMessage +from pathlib import Path + + +MARKER = "KnowledgeFS Golden Evidence" + + +def pdf_fixture() -> bytes: + stream = f"BT /F1 12 Tf 20 100 Td ({MARKER}) Tj ET".encode() + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 200] /Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length " + + str(len(stream)).encode() + + b" >>\nstream\n" + + stream + + b"\nendstream", + ] + body = bytearray(b"%PDF-1.4\n") + offsets = [0] + for index, value in enumerate(objects, 1): + offsets.append(len(body)) + body.extend(f"{index} 0 obj\n".encode() + value + b"\nendobj\n") + startxref = len(body) + body.extend(f"xref\n0 {len(offsets)}\n0000000000 65535 f \n".encode()) + for offset in offsets[1:]: + body.extend(f"{offset:010} 00000 n \n".encode()) + body.extend( + f"trailer\n<< /Size {len(offsets)} /Root 1 0 R >>\nstartxref\n{startxref}\n%%EOF\n".encode() + ) + return bytes(body) + + +def multipart(filename: str, body: bytes) -> tuple[bytes, str]: + boundary = "kfs-golden-" + os.urandom(12).hex() + fields = bytearray() + for name, value in ( + ("strategy", "fast"), + ("coordinates", "true"), + ("include_slide_notes", "true"), + ): + fields.extend( + f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n{value}\r\n'.encode() + ) + fields.extend( + f'--{boundary}\r\nContent-Disposition: form-data; name="files"; filename="{filename}"\r\nContent-Type: application/octet-stream\r\n\r\n'.encode() + ) + fields.extend(body) + fields.extend(f"\r\n--{boundary}--\r\n".encode()) + return bytes(fields), f"multipart/form-data; boundary={boundary}" + + +def has_evidence(value, marker: str = MARKER) -> bool: + return isinstance(value, list) and marker in " ".join( + str(item.get("text", "")) for item in value if isinstance(item, dict) + ) + + +def msg_fixture(path: Path | None) -> tuple[bytes, str]: + target = path or Path(__file__).with_name("fixtures") / "fake-email-attachment.msg" + if target.stat().st_size > 1024 * 1024: + raise ValueError("MSG golden fixture must be at most 1 MiB") + body = target.read_bytes() + if path is None: + if ( + hashlib.sha256(body).hexdigest() + != "92f65236e7eae301ea6f70a85f38cd5a9fae9f807d17fc35c5b5dbcf0f82a8ec" + ): + raise ValueError("Upstream MSG fixture digest mismatch") + return body, "Hey this is a fake attachment!" + return body, MARKER + + +def convert(command: list[str]) -> None: + # Fixture construction runs outside a request. Use the image-captured original + # tools only for these tiny trusted inputs; HTTP parsing still uses the wrappers. + manifest = Path("/opt/kfs-sandbox/converters.json") + if manifest.is_file(): + command = [json.loads(manifest.read_text())[command[0]], *command[1:]] + process = subprocess.Popen( + command, + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + code = process.wait(timeout=45) + if code != 0: + raise RuntimeError("fixture_conversion_failed") + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + +def create_fixtures(directory: Path) -> dict[str, bytes]: + from docx import Document + from openpyxl import Workbook + from pptx import Presentation + from pptx.util import Inches + + document = Document() + document.add_paragraph(MARKER) + document.save(directory / "sample.docx") + slides = Presentation() + slide = slides.slides.add_slide(slides.slide_layouts[6]) + slide.shapes.add_textbox(Inches(1), Inches(1), Inches(6), Inches(1)).text = MARKER + slides.save(directory / "sample.pptx") + workbook = Workbook() + workbook.active.append(["evidence"]) + workbook.active.append([MARKER]) + workbook.save(directory / "sample.xlsx") + profile = (directory / "office-profile").as_uri() + for source, target in ( + ("docx", "doc"), + ("pptx", "ppt"), + ("xlsx", "xls"), + ("docx", "odt"), + ): + convert( + [ + "soffice", + f"-env:UserInstallation={profile}", + "--headless", + "--convert-to", + target, + "--outdir", + str(directory), + str(directory / f"sample.{source}"), + ] + ) + html = directory / "sample.html" + html.write_text(f"

{MARKER}

") + convert(["pandoc", str(html), "-o", str(directory / "sample.epub")]) + fixtures = { + f"sample.{extension}": (directory / f"sample.{extension}").read_bytes() + for extension in ("doc", "docx", "ppt", "pptx", "xls", "xlsx", "odt", "epub") + } + fixtures["sample.pdf"] = pdf_fixture() + fixtures["sample.rtf"] = (r"{\rtf1\ansi " + MARKER + "}").encode() + message = EmailMessage() + message["Subject"] = "Golden attachment test" + message.set_content("The evidence is in the attachment.") + message.add_attachment( + fixtures["sample.docx"], + maintype="application", + subtype="octet-stream", + filename="attached.docx", + ) + fixtures["sample.eml"] = message.as_bytes() + return fixtures + + +def post(url: str, name: str, payload: bytes): + body, content_type = multipart(name, payload) + request = urllib.request.Request( + url, body, {"Content-Type": content_type, "Accept": "application/json"} + ) + try: + with urllib.request.urlopen(request, timeout=180) as response: + raw = response.read(32 * 1024 * 1024 + 1) + if len(raw) > 32 * 1024 * 1024: + raise RuntimeError("response_bytes") + return response.status, json.loads(raw), len(raw) + except urllib.error.HTTPError as error: + return error.code, None, 0 + + +def run_gate(url: str, custom_msg_fixture: Path | None) -> dict: + # Refuse accidental production runs, even if a user copies a production URL. + if url not in { + "http://127.0.0.1:8000/general/v0/general", + "http://localhost:8000/general/v0/general", + }: + raise ValueError( + "Golden requests must target the disposable container's loopback address" + ) + report = {"passed": [], "failures": [], "missing_fixtures": []} + with tempfile.TemporaryDirectory(prefix="kfs-golden-") as directory: + fixtures = create_fixtures(Path(directory)) + msg_body, msg_marker = msg_fixture(custom_msg_fixture) + fixtures["sample.msg"] = msg_body + for name, body in fixtures.items(): + started = time.monotonic() + status, result, response_bytes = post(url, name, body) + if status == 200 and has_evidence( + result, msg_marker if name == "sample.msg" else MARKER + ): + report["passed"].append( + { + "format": name.rsplit(".", 1)[-1], + "milliseconds": round((time.monotonic() - started) * 1000), + "input_bytes": len(body), + "response_bytes": response_bytes, + } + ) + else: + report["failures"].append( + { + "format": name.rsplit(".", 1)[-1], + "status": status, + "reason": "missing_evidence_or_failed", + } + ) + # Tiny sparse extent fixture; admission must reject it BEFORE pandas allocation. + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as archive: + archive.writestr( + "xl/worksheets/sheet1.xml", '' + ) + nested = EmailMessage() + nested["Subject"] = "Bounded rejection fixture" + nested.set_content("Body") + nested.add_attachment( + stream.getvalue(), + maintype="application", + subtype="octet-stream", + filename="unsafe.xlsx", + ) + status, _, _ = post(url, "nested.eml", nested.as_bytes()) + if status != 422: + report["failures"].append( + { + "format": "eml-nested-xlsx", + "status": status, + "reason": "admission_did_not_reject", + } + ) + report["gate"] = ( + "passed" + if not report["failures"] and not report["missing_fixtures"] + else "blocked" + ) + return report + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--msg-fixture", type=Path) + args = parser.parse_args() + result = run_gate("http://127.0.0.1:8000/general/v0/general", args.msg_fixture) + print(json.dumps(result, indent=2)) + raise SystemExit(0 if result["gate"] == "passed" else 1) diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/__init__.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/__init__.py new file mode 100644 index 00000000000..048f7ed8b7e --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/__init__.py @@ -0,0 +1,3 @@ +"""Optional KnowledgeFS request-isolated Unstructured runtime.""" + +REVISION = "knowledgefs-unstructured-sandbox-v1" diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/__main__.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/__main__.py new file mode 100644 index 00000000000..50f292de11b --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/__main__.py @@ -0,0 +1,17 @@ +"""Keep one supervisor so admission covers every client of this deployment.""" + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "kfs_sandbox.gateway:app", + host="0.0.0.0", + port=8000, + workers=1, + limit_concurrency=64, + timeout_keep_alive=5, + timeout_graceful_shutdown=10, + access_log=False, + ws="none", + ) diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/admission.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/admission.py new file mode 100644 index 00000000000..cc905655db8 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/admission.py @@ -0,0 +1,308 @@ +"""Shared attachment/archive budget, executed only inside the disposable request worker. + +This is admission, not a replacement parser. The original provider receives the unchanged +request after admission. Untrusted content is never interpreted as executable instructions. +""" + +from __future__ import annotations + +import io +import re +import zipfile +from dataclasses import dataclass +from email import policy +from email.message import Message +from email.parser import BytesParser +from pathlib import PurePosixPath +from typing import Callable, Iterable +from xml.parsers import expat + + +class Rejected(Exception): + """Stable safe error: no document text, local path, or attachment name in diagnostics.""" + + def __init__(self, reason: str): + super().__init__(reason) + self.reason = reason + + +@dataclass(frozen=True) +class Limits: + depth: int = 8 + attachments: int = 128 + mime_parts: int = 1024 + decoded_bytes: int = 128 * 1024 * 1024 + archive_entries: int = 4096 + expanded_bytes: int = 512 * 1024 * 1024 + xml_bytes: int = 64 * 1024 * 1024 + xml_member_bytes: int = 16 * 1024 * 1024 + xml_depth: int = 128 + xml_nodes: int = 1_000_000 + sheets: int = 256 + sheet_rows: int = 100_000 + sheet_columns: int = 16_384 + sheet_cells: int = 250_000 + workbook_cells: int = 500_000 + + +@dataclass +class Budget: + limits: Limits = Limits() + attachments: int = 0 + mime_parts: int = 0 + decoded_bytes: int = 0 + archive_entries: int = 0 + expanded_bytes: int = 0 + xml_bytes: int = 0 + xml_nodes: int = 0 + sheets: int = 0 + workbook_cells: int = 0 + + def consume(self, key: str, amount: int = 1) -> None: + value = getattr(self, key) + amount + if value > getattr(self.limits, key): + raise Rejected(key) + setattr(self, key, value) + + +ATTACHMENT_EXTENSIONS = frozenset( + "pdf doc docx ppt pptx xls xlsx rtf odt epub eml msg md markdown mdx txt text " + "properties vtt html htm csv xml png jpg jpeg gif webp bmp tif tiff heic".split() +) +ARCHIVE_EXTENSIONS = frozenset({"docx", "pptx", "xlsx", "epub", "odt"}) +_CELL = re.compile(r"\$?([A-Za-z]{1,3})\$?([1-9][0-9]{0,6})\Z") +_OLE = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" +MsgLoader = Callable[[bytes], Iterable[tuple[str, bytes]]] + + +def _msg_attachments(body: bytes) -> Iterable[tuple[str, bytes]]: + # Public API used by the pinned Unstructured 0.22.18 MSG partitioner itself. + from oxmsg import Message as OutlookMessage + + for attachment in OutlookMessage.load(io.BytesIO(body)).attachments: + yield attachment.file_name or "unknown", attachment.file_bytes or b"" + + +def _is_outlook_storage(body: bytes) -> bool: + import olefile + + try: + with olefile.OleFileIO(io.BytesIO(body)) as storage: + return storage.exists("__properties_version1.0") + except (OSError, ValueError) as error: + raise Rejected("ole_invalid") from error + + +def inspect_document( + body: bytes, + filename: str, + budget: Budget | None = None, + *, + depth: int = 0, + msg_loader: MsgLoader = _msg_attachments, +) -> Budget: + budget = budget or Budget() + if depth > budget.limits.depth: + raise Rejected("depth") + suffix = filename.rsplit(".", 1)[-1].lower() + if body.startswith(b"PK\x03\x04") or suffix in ARCHIVE_EXTENSIONS: + _inspect_archive(body, budget) + elif suffix == "msg" or (body.startswith(_OLE) and _is_outlook_storage(body)): + try: + for name, data in msg_loader(body): + _inspect_attachment(data, name, budget, depth + 1, msg_loader) + except Rejected: + raise + except Exception as error: + raise Rejected("msg_invalid") from error + elif suffix == "eml" or _looks_like_mail(body): + try: + message = BytesParser(policy=policy.default).parsebytes(body) + _inspect_message(message, budget, depth, msg_loader) + except Rejected: + raise + except (ValueError, RecursionError) as error: + raise Rejected("mime_invalid") from error + return budget + + +def _looks_like_mail(body: bytes) -> bool: + headers = body[:8192].split(b"\r\n\r\n", 1)[0].split(b"\n\n", 1)[0].lower() + return b"mime-version:" in headers and ( + b"from:" in headers or b"subject:" in headers + ) + + +def _inspect_attachment( + body: bytes, filename: str, budget: Budget, depth: int, loader: MsgLoader +) -> None: + budget.consume("attachments") + budget.consume("decoded_bytes", len(body)) + suffix = filename.rsplit(".", 1)[-1].lower() + if suffix not in ATTACHMENT_EXTENSIONS: + raise Rejected("unsupported_attachment") + inspect_document(body, filename, budget, depth=depth, msg_loader=loader) + + +def _inspect_message( + message: Message, budget: Budget, depth: int, loader: MsgLoader +) -> None: + pending = [(message, depth)] + while pending: + part, part_depth = pending.pop() + if part_depth > budget.limits.depth: + raise Rejected("depth") + budget.consume("mime_parts") + if part.defects: + raise Rejected("mime_invalid") + if part.get_content_type() == "message/rfc822": + budget.consume("attachments") + payload = part.get_payload() + if not isinstance(payload, list): + raise Rejected("mime_invalid") + for nested in payload: + # RFC822 attachments are Message objects, not decoded byte payloads. + budget.consume( + "decoded_bytes", len(nested.as_bytes(policy=policy.default)) + ) + pending.append((nested, part_depth + 1)) + elif part.is_multipart(): + pending.extend( + (child, part_depth + 1) for child in reversed(part.get_payload()) + ) + elif part.get_filename() or part.get_content_disposition() == "attachment": + data = part.get_payload(decode=True) + if data is None or part.defects: + raise Rejected("mime_invalid") + _inspect_attachment( + data, part.get_filename() or "unknown", budget, part_depth + 1, loader + ) + + +def _inspect_archive(body: bytes, budget: Budget) -> None: + try: + with zipfile.ZipFile(io.BytesIO(body)) as archive: + infos = archive.infolist() + if budget.archive_entries + len(infos) > budget.limits.archive_entries: + raise Rejected("archive_entries") + if ( + budget.expanded_bytes + sum(info.file_size for info in infos) + > budget.limits.expanded_bytes + ): + raise Rejected("expanded_bytes") + names: set[str] = set() + for info in infos: + name = info.filename.replace("\\", "/") + if ( + name in names + or name.startswith("/") + or ".." in PurePosixPath(name).parts + or ":" in name + ): + raise Rejected("archive_path") + names.add(name) + budget.consume("archive_entries") + if info.flag_bits & 1: + raise Rejected("archive_encrypted") + xml = name.lower().endswith((".xml", ".rels", ".xhtml", ".opf", ".ncx")) + guard = _XmlGuard(budget) if xml else None + member_bytes = 0 + with archive.open(info) as member: + while chunk := member.read(64 * 1024): + budget.consume("expanded_bytes", len(chunk)) + member_bytes += len(chunk) + if guard: + budget.consume("xml_bytes", len(chunk)) + if member_bytes > budget.limits.xml_member_bytes: + raise Rejected("xml_member_bytes") + guard.feed(chunk) + if guard: + guard.finish() + except Rejected: + raise + except ( + zipfile.BadZipFile, + NotImplementedError, + RuntimeError, + expat.ExpatError, + ValueError, + ) as error: + raise Rejected("archive_invalid") from error + + +class _XmlGuard: + def __init__(self, budget: Budget): + self.budget = budget + self.depth = 0 + self.rows = 0 + self.columns = 0 + self.worksheet = False + self.current_row = 0 + self.current_column = 0 + self.parser = expat.ParserCreate(namespace_separator="}") + self.parser.StartElementHandler = self._start + self.parser.EndElementHandler = self._end + self.parser.EntityDeclHandler = self._entity + self.parser.ExternalEntityRefHandler = self._entity + + @staticmethod + def _entity(*_args): + raise Rejected("xml_entity") + + def _start(self, name: str, attrs: dict[str, str]) -> None: + self.depth += 1 + if self.depth > self.budget.limits.xml_depth: + raise Rejected("xml_depth") + self.budget.consume("xml_nodes") + local = name.rsplit("}", 1)[-1] + if local == "worksheet": + self.worksheet = True + self.budget.consume("sheets") + if self.worksheet and local == "row": + row = attrs.get("r", str(self.current_row + 1)) + if ( + not row.isascii() + or not row.isdigit() + or not 0 < int(row) <= self.budget.limits.sheet_rows + ): + raise Rejected("sheet_extent") + self.current_row = int(row) + self.current_column = 0 + if self.worksheet and local in {"c", "mergeCell"}: + reference = attrs.get("r" if local == "c" else "ref", "") + if local == "c" and not reference: + self.current_column += 1 + self._cell(max(1, self.current_row), self.current_column) + return + for cell in reference.split(":"): + match = _CELL.fullmatch(cell) + if not match: + raise Rejected("sheet_cell_reference") + column = 0 + for letter in match[1].upper(): + column = column * 26 + ord(letter) - ord("A") + 1 + row = int(match[2]) + if local == "c": + self.current_column = column + self._cell(row, column) + + def _cell(self, row: int, column: int) -> None: + if ( + row > self.budget.limits.sheet_rows + or column > self.budget.limits.sheet_columns + ): + raise Rejected("sheet_extent") + previous = self.rows * self.columns + self.rows, self.columns = max(self.rows, row), max(self.columns, column) + if self.rows * self.columns > self.budget.limits.sheet_cells: + raise Rejected("sheet_cells") + self.budget.consume("workbook_cells", self.rows * self.columns - previous) + + def _end(self, _name: str) -> None: + self.depth -= 1 + + def feed(self, chunk: bytes) -> None: + self.parser.Parse(chunk, False) + + def finish(self) -> None: + self.parser.Parse(b"", True) diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/conversion.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/conversion.py new file mode 100644 index 00000000000..36c534a2a58 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/conversion.py @@ -0,0 +1,403 @@ +"""Version-scoped executable adapters: validate privately, then publish. + +Supported invocations are the pinned Unstructured DOC/PPT and RTF/EPUB/ODT paths. +No provider Python function is replaced. Every converter shares the original mail +admission budget via one file-locked ledger in its disposable request directory. +""" + +from __future__ import annotations + +import fcntl +import heapq +import json +import os +import subprocess +import tempfile +import time +from dataclasses import asdict, dataclass +from html.parser import HTMLParser +from pathlib import Path +from typing import BinaryIO + +from .admission import Budget, Limits, Rejected, inspect_document + + +PROBES = {"--version", "--list-input-formats", "--list-output-formats"} +VOID_TAGS = frozenset( + "area base br col embed hr img input link meta param source track wbr".split() +) +MAX_PRODUCT_BYTES = 64 * 1024 * 1024 + + +def initialize_budget(directory: Path, budget: Budget, *, wall_seconds: float) -> None: + state = { + "budget": asdict(budget), + "failure": None, + "deadline": time.monotonic() + wall_seconds, + } + (directory / "conversion-budget.json").write_text(json.dumps(state)) + + +def _decode_state(handle) -> tuple[dict, Budget]: + handle.seek(0) + raw = handle.read(16 * 1024 + 1) + if len(raw) > 16 * 1024: + raise Rejected("archive_invalid") + state = json.loads(raw) + values = dict(state["budget"]) + limits = Limits(**values.pop("limits")) + return state, Budget(limits=limits, **values) + + +def load_budget(directory: Path) -> tuple[Budget, str | None]: + with (directory / "conversion-budget.json").open("rb") as handle: + fcntl.flock(handle, fcntl.LOCK_SH) + state, budget = _decode_state(handle) + return budget, state["failure"] + + +def record_failure(directory: Path, reason: str) -> None: + with (directory / "conversion-budget.json").open("r+b") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + state, _ = _decode_state(handle) + state["failure"] = state["failure"] or reason + handle.seek(0) + handle.write(json.dumps(state).encode()) + handle.truncate() + handle.flush() + + +def _record_product(directory: Path, body: bytes, target_format: str) -> None: + with (directory / "conversion-budget.json").open("r+b") as handle: + fcntl.flock(handle, fcntl.LOCK_EX) + state, budget = _decode_state(handle) + try: + if state["failure"]: + raise Rejected(state["failure"]) + budget.consume("decoded_bytes", len(body)) + if target_format == "html": + budget.consume("expanded_bytes", len(body)) + budget.consume("xml_bytes", len(body)) + if len(body) > budget.limits.xml_member_bytes: + raise Rejected("xml_member_bytes") + try: + text = body.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + raise Rejected("archive_invalid") from error + guard = _HtmlGuard(budget) + guard.feed(text) + guard.close() + else: + inspect_document(body, f"converted.{target_format}", budget) + except Rejected as error: + state["failure"] = error.reason + raise + finally: + state["budget"] = asdict(budget) + encoded = json.dumps(state).encode() + handle.seek(0) + handle.write(encoded) + handle.truncate() + handle.flush() + + +def _inside(path: str | Path, directory: Path, *, existing: bool) -> Path: + target = Path(path) + resolved = target.resolve(strict=existing) + if not resolved.is_relative_to(directory.resolve()) or target.is_symlink(): + raise Rejected("archive_path") + if existing and not resolved.is_file(): + raise Rejected("archive_path") + return resolved + + +@dataclass(frozen=True) +class Invocation: + arguments: tuple[str, ...] + target_format: str + source: Path | None = None + output: Path | None = None + probe: bool = False + + +def parse_invocation(kind: str, arguments: list[str], directory: Path) -> Invocation: + if kind == "pandoc" and len(arguments) == 1 and arguments[0] in PROBES: + return Invocation(tuple(arguments), "probe", probe=True) + if kind == "soffice": + if ( + len(arguments) != 6 + or arguments[0:2] != ["--headless", "--convert-to"] + or arguments[3] != "--outdir" + ): + raise Rejected("archive_invalid") + target_format = arguments[2].split(":", 1)[0] + if target_format not in {"docx", "pptx"}: + raise Rejected("archive_invalid") + source = _inside(arguments[5], directory, existing=True) + output = _inside( + Path(arguments[4]) / f"{source.stem}.{target_format}", + directory, + existing=False, + ) + elif kind == "pandoc": + values: dict[str, str] = {} + sources = [] + sandbox = False + for argument in arguments: + if argument == "--sandbox" and not sandbox: + sandbox = True + elif argument.startswith(("--from=", "--to=", "--output=")): + key, value = argument.split("=", 1) + if key in values or not value: + raise Rejected("archive_invalid") + values[key] = value + elif not argument.startswith("-"): + sources.append(argument) + else: + raise Rejected("archive_invalid") + if len(sources) != 1 or values.get("--from") not in {"rtf", "epub", "odt"}: + raise Rejected("archive_invalid") + source = _inside(sources[0], directory, existing=True) + target_format = values.get("--to", "") + output_arg = values.get("--output") + if target_format == "html" and output_arg is None: + output = None + elif target_format == "docx" and values["--from"] == "odt" and output_arg: + output = _inside(output_arg, directory, existing=False) + else: + raise Rejected("archive_invalid") + else: + raise Rejected("archive_invalid") + if output is not None and output.exists(): + raise Rejected("archive_path") + return Invocation(tuple(arguments), target_format, source, output) + + +def _run( + command: list[str], stdout_path: Path, stderr_path: Path, deadline: float, cap: int +) -> int: + # Inherit the request process group and all rlimits. Never detach a converter. + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + command, stdin=subprocess.DEVNULL, stdout=stdout, stderr=stderr + ) + try: + while process.poll() is None: + if ( + stdout_path.stat().st_size > cap + or stderr_path.stat().st_size > 64 * 1024 + ): + raise Rejected("xml_member_bytes") + if time.monotonic() >= deadline: + raise Rejected("wall_seconds") + time.sleep(0.01) + if ( + stdout_path.stat().st_size > cap + or stderr_path.stat().st_size > 64 * 1024 + ): + raise Rejected("xml_member_bytes") + if process.returncode < 0: + # Inherited CPU/file limits and OS termination must not become an + # attachment silently skipped by an upstream exception handler. + raise Rejected("worker_resource_limit") + return process.returncode + finally: + if process.poll() is None: + process.kill() + process.wait() + + +def _read_bounded(path: Path, cap: int) -> bytes: + with path.open("rb") as handle: + body = handle.read(cap + 1) + if len(body) > cap: + raise Rejected("xml_member_bytes") + return body + + +def convert( + kind: str, + arguments: list[str], + directory: Path, + executable: list[str], + stdout: BinaryIO, + stderr: BinaryIO, + *, + max_product_bytes: int = MAX_PRODUCT_BYTES, +) -> int: + try: + return _convert( + kind, + arguments, + directory, + executable, + stdout, + stderr, + max_product_bytes=max_product_bytes, + ) + except Rejected as error: + record_failure(directory, error.reason) + raise + + +def _convert( + kind: str, + arguments: list[str], + directory: Path, + executable: list[str], + stdout: BinaryIO, + stderr: BinaryIO, + *, + max_product_bytes: int, +) -> int: + invocation = parse_invocation(kind, arguments, directory) + with (directory / "conversion-budget.json").open("rb") as handle: + fcntl.flock(handle, fcntl.LOCK_SH) + state, _ = _decode_state(handle) + if state["failure"]: + raise Rejected(state["failure"]) + with tempfile.TemporaryDirectory(prefix="convert-", dir=directory) as temporary: + stage = Path(temporary) + staged_output = None + adapted = list(invocation.arguments) + if invocation.output is not None: + staged_output = stage / invocation.output.name + if kind == "soffice": + adapted[4] = str(stage) + else: + adapted = [ + f"--output={staged_output}" if arg.startswith("--output=") else arg + for arg in adapted + ] + output_path, error_path = stage / "stdout", stage / "stderr" + code = _run( + [*executable, *adapted], + output_path, + error_path, + state["deadline"], + 64 * 1024 if invocation.probe else max_product_bytes, + ) + if code == 0 and not invocation.probe: + product = staged_output or output_path + if not product.is_file() or product.is_symlink(): + raise Rejected("archive_invalid") + if product.stat().st_size > max_product_bytes: + raise Rejected("xml_member_bytes") + _record_product( + directory, + _read_bounded(product, max_product_bytes), + invocation.target_format, + ) + if staged_output is not None: + # Only publish a validated file; neither rejected nor partial converter + # products ever appear at the path the upstream parser will open. + try: + # Same request tmpfs: link is an atomic publish-if-absent operation, + # unlike replace(), which could clobber a concurrently created file. + os.link(staged_output, invocation.output, follow_symlinks=False) + except FileExistsError as error: + raise Rejected("archive_path") from error + stdout.write( + _read_bounded( + output_path, 64 * 1024 if invocation.probe else max_product_bytes + ) + ) + stderr.write(_read_bounded(error_path, 64 * 1024)) + return code + + +class _HtmlGuard(HTMLParser): + def __init__(self, budget: Budget): + super().__init__(convert_charrefs=False) + self.budget = budget + self.stack: list[str] = [] + self.tables: list[dict] = [] + + def handle_starttag( + self, tag: str, attributes: list[tuple[str, str | None]] + ) -> None: + self.budget.consume("xml_nodes") + if tag not in VOID_TAGS: + self.stack.append(tag) + if len(self.stack) > self.budget.limits.xml_depth: + raise Rejected("xml_depth") + if tag == "table": + self.tables.append( + { + "row": 0, + "column": 0, + "width": 0, + "height": 0, + "occupied": 0, + "spans": [], + } + ) + if not self.tables: + return + table = self.tables[-1] + if tag == "tr": + table["row"] += 1 + table["column"] = 0 + while table["spans"] and table["spans"][0][0] < table["row"]: + _, mask = heapq.heappop(table["spans"]) + table["occupied"] &= ~mask + elif tag in {"td", "th"}: + attrs = dict(attributes) + if len(attrs) != len(attributes): + raise Rejected("archive_invalid") + rows, columns = ( + self._span(attrs.get("rowspan")), + self._span(attrs.get("colspan")), + ) + if ( + rows > self.budget.limits.sheet_rows + or columns > self.budget.limits.sheet_columns + ): + raise Rejected("sheet_extent") + row = max(1, table["row"]) + column = table["column"] + while occupied := (table["occupied"] >> column) & ((1 << columns) - 1): + column += occupied.bit_length() + end = column + columns + height = max(table["height"], row + rows - 1) + width = max(table["width"], end) + if ( + width > self.budget.limits.sheet_columns + or height > self.budget.limits.sheet_rows + ): + raise Rejected("sheet_extent") + cells = height * width + if cells > self.budget.limits.sheet_cells: + raise Rejected("sheet_cells") + self.budget.consume( + "workbook_cells", cells - table["height"] * table["width"] + ) + table.update(column=end, height=height, width=width) + if rows > 1: + mask = ((1 << columns) - 1) << column + table["occupied"] |= mask + heapq.heappush(table["spans"], (row + rows - 1, mask)) + + @staticmethod + def _span(value: str | None) -> int: + if value is None: + return 1 + if ( + not value.isascii() + or not value.isdigit() + or not 0 < len(value) <= 7 + or int(value) < 1 + ): + raise Rejected("sheet_extent") + return int(value) + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + if tag not in VOID_TAGS: + self.handle_endtag(tag) + + def handle_endtag(self, tag: str) -> None: + if tag == "table" and self.tables: + self.tables.pop() + if tag in self.stack: + del self.stack[len(self.stack) - 1 - self.stack[::-1].index(tag) :] diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/converter_cli.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/converter_cli.py new file mode 100644 index 00000000000..ffb111f2c1c --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/converter_cli.py @@ -0,0 +1,55 @@ +"""Executable wrapper entrypoint; no caller-selected executables or shell evaluation.""" + +import json +import os +import sys +from pathlib import Path + +from .admission import Rejected +from .conversion import convert + + +MANIFEST = Path("/opt/kfs-sandbox/converters.json") + + +def run( + kind: str, arguments: list[str], directory: Path, manifest: Path, stdout, stderr +) -> int: + raw = manifest.read_bytes() + if len(raw) > 4096: + raise RuntimeError("Invalid converter manifest") + executable = json.loads(raw)[kind] + if ( + not isinstance(executable, str) + or not Path(executable).is_absolute() + or not os.access(executable, os.X_OK) + ): + raise RuntimeError("Invalid converter executable") + try: + return convert(kind, arguments, directory, [executable], stdout, stderr) + except Rejected: + # LibreOffice's caller retries if stdout is empty, even with a nonzero exit code. + # A nonempty generic message prevents wasting its retry loop on known rejection. + if kind == "soffice": + stdout.write(b"KnowledgeFS converter admission rejected\n") + stderr.write(b"KnowledgeFS converter admission rejected\n") + return 65 + + +def main(kind: str) -> None: + try: + directory = Path(os.environ["KFS_CONVERSION_DIRECTORY"]) + code = run( + kind, + sys.argv[1:], + directory, + MANIFEST, + sys.stdout.buffer, + sys.stderr.buffer, + ) + except Exception: + # Wrong image/invocation is a configuration failure, not a safe fallback to a + # raw executable. Never leak converter exceptions, arguments, paths, or text. + sys.stderr.write("KnowledgeFS converter runtime is unavailable\n") + code = 70 + raise SystemExit(code) diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/gateway.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/gateway.py new file mode 100644 index 00000000000..8dae94efcb2 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/gateway.py @@ -0,0 +1,364 @@ +"""Single-process, bounded ASGI supervisor for disposable parser process groups. + +This controls resource exhaustion, not hostile native-code execution. Container hard +limits remain mandatory; aggregate RSS/CPU/temp/PID checks are sampled, not cgroups. +""" + +from __future__ import annotations + +import asyncio +import json +import math +import os +import shutil +import signal +import sys +import tempfile +import time +from dataclasses import asdict, dataclass +from pathlib import Path + +import psutil + +from . import REVISION + + +@dataclass(frozen=True) +class Settings: + active_limit: int = 1 + queue_limit: int = 8 + queue_seconds: float = 30 + wall_seconds: float = 2400 + input_bytes: int = 51 * 1024 * 1024 + response_bytes: int = 32 * 1024 * 1024 + memory_bytes: int = 4 * 1024**3 + address_space_bytes: int = 32 * 1024**3 + temp_bytes: int = 512 * 1024 * 1024 + file_bytes: int = 256 * 1024 * 1024 + process_limit: int = 64 + cpu_seconds: int = 2400 + temp_root: str = "/tmp" + sample_seconds: float = 0.05 + + def __post_init__(self): + for key, value in asdict(self).items(): + if key != "temp_root" and ( + not isinstance(value, (int, float)) + or not math.isfinite(value) + or value <= 0 + ): + if not (key == "queue_limit" and value == 0): + raise ValueError(f"Invalid sandbox setting: {key}") + if self.active_limit > 8 or self.queue_limit > 64: + raise ValueError("Sandbox admission must remain bounded") + + +class ResourceLimit(Exception): + def __init__(self, reason: str, status: int = 413): + self.reason = reason + self.status = status + + +class Disconnected(Exception): + pass + + +async def respond(send, status: int, reason: str) -> None: + payload = json.dumps( + { + "detail": { + "code": "PARSER_RESOURCE_REJECTED", + "reason": reason, + "revision": REVISION, + } + } + ).encode() + headers = [ + (b"content-type", b"application/json"), + (b"content-length", str(len(payload)).encode()), + ] + if status == 429: + headers.append((b"retry-after", b"1")) + await send({"type": "http.response.start", "status": status, "headers": headers}) + await send({"type": "http.response.body", "body": payload}) + + +class Gateway: + def __init__( + self, settings: Settings, *, app_target: str = "prepline_general.api.app:app" + ): + self.settings = settings + self.app_target = app_target + self.active = 0 + self.queued = 0 + self.slots = asyncio.Semaphore(settings.active_limit) + + async def __call__(self, scope, receive, send): + if scope["type"] == "lifespan": + while True: + event = await receive() + if event["type"] == "lifespan.startup": + await send({"type": "lifespan.startup.complete"}) + elif event["type"] == "lifespan.shutdown": + await send({"type": "lifespan.shutdown.complete"}) + return + if scope["type"] != "http": + return + if scope["path"] == "/healthcheck" and scope["method"] == "GET": + body = json.dumps( + { + "status": "healthy", + "revision": REVISION, + "active": self.active, + "queued": self.queued, + } + ).encode() + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json")], + } + ) + await send({"type": "http.response.body", "body": body}) + return + if scope["path"] != "/general/v0/general" or scope["method"] != "POST": + await respond(send, 404, "not_found") + return + # Reserve capacity before the first await, independent of semaphore scheduling. + if ( + self.active + self.queued + >= self.settings.active_limit + self.settings.queue_limit + ): + await respond(send, 429, "admission_full") + return + self.queued += 1 + try: + await asyncio.wait_for(self.slots.acquire(), self.settings.queue_seconds) + except TimeoutError: + await respond(send, 429, "admission_timeout") + return + finally: + self.queued -= 1 + self.active += 1 + directory = None + try: + directory = Path( + tempfile.mkdtemp(prefix="kfs-parse-", dir=self.settings.temp_root) + ) + deadline = time.monotonic() + self.settings.wall_seconds + await self._read_request(receive, directory, deadline) + await self._execute(scope, receive, send, directory, deadline) + except Disconnected: + pass + except ResourceLimit as error: + await respond(send, error.status, error.reason) + except OSError: + await respond(send, 503, "supervisor_resource_limit") + finally: + try: + if directory is not None: + # Exact mkdtemp-owned directory; never a caller-controlled path. + shutil.rmtree(directory) + finally: + self.active -= 1 + self.slots.release() + + async def _read_request(self, receive, directory: Path, deadline: float) -> None: + size = 0 + with (directory / "request.body").open("wb") as output: + while True: + try: + event = await asyncio.wait_for( + receive(), max(0, deadline - time.monotonic()) + ) + except TimeoutError as error: + raise ResourceLimit("wall_seconds", 504) from error + if event["type"] == "http.disconnect": + raise Disconnected() + data = event.get("body", b"") + size += len(data) + if size > self.settings.input_bytes: + raise ResourceLimit("input_bytes") + output.write(data) + if not event.get("more_body"): + break + + async def _execute( + self, scope, receive, send, directory: Path, deadline: float + ) -> None: + serialized_scope = { + key: scope[key] + for key in ( + "type", + "method", + "path", + "http_version", + "scheme", + "server", + "client", + ) + if key in scope + } + serialized_scope["headers"] = [ + (key.decode("latin-1"), value.decode("latin-1")) + for key, value in scope["headers"] + ] + serialized_scope["query_string"] = scope.get("query_string", b"").decode( + "latin-1" + ) + envelope = { + "scope": serialized_scope, + "settings": asdict(self.settings), + "app_target": self.app_target, + } + metadata = json.dumps(envelope) + if len(metadata.encode()) > 64 * 1024: + raise ResourceLimit("request_metadata_bytes") + (directory / "request.json").write_text(metadata) + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "kfs_sandbox.worker", + str(directory), + start_new_session=True, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + ) + descendants: dict[tuple[int, float], psutil.Process] = {} + disconnected = asyncio.create_task(self._wait_disconnect(receive)) + completed = asyncio.create_task(process.wait()) + try: + while not completed.done(): + if disconnected.done(): + raise Disconnected() + if time.monotonic() >= deadline: + raise ResourceLimit("wall_seconds", 504) + self._inspect_tree(process.pid, directory, descendants) + await asyncio.wait( + {completed, disconnected}, + timeout=self.settings.sample_seconds, + return_when=asyncio.FIRST_COMPLETED, + ) + if disconnected.done(): + raise Disconnected() + self._inspect_tree(process.pid, directory, descendants) + if failure := self._read_failure(directory): + raise ResourceLimit(failure["reason"], failure["status"]) + if process.returncode != 0 or not (directory / "response.json").exists(): + raise ResourceLimit("worker_terminated", 413) + metadata_path = directory / "response.json" + if metadata_path.stat().st_size > 64 * 1024: + raise ResourceLimit("response_metadata_bytes") + result = json.loads(metadata_path.read_text()) + headers = [ + (key.encode("latin-1"), value.encode("latin-1")) + for key, value in result["headers"] + if key.lower() not in {"transfer-encoding", "connection"} + ] + headers.append((b"x-knowledgefs-parser-revision", REVISION.encode())) + + async def send_bounded(message): + delivery = asyncio.create_task(send(message)) + try: + ready, _ = await asyncio.wait( + {delivery, disconnected}, + timeout=max(0, deadline - time.monotonic()), + return_when=asyncio.FIRST_COMPLETED, + ) + if delivery not in ready or disconnected in ready: + # Headers may already have been sent; never emit a second error response. + raise Disconnected() + await delivery + finally: + delivery.cancel() + await asyncio.gather(delivery, return_exceptions=True) + + await send_bounded( + { + "type": "http.response.start", + "status": result["status"], + "headers": headers, + } + ) + with (directory / "response.body").open("rb") as response: + while chunk := response.read(64 * 1024): + await send_bounded( + {"type": "http.response.body", "body": chunk, "more_body": True} + ) + await send_bounded( + {"type": "http.response.body", "body": b"", "more_body": False} + ) + finally: + # Kill the entire session group, including converters left behind after a + # successful response; also kill observed descendants that changed sessions. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + for child in descendants.values(): + try: + child.kill() + except psutil.NoSuchProcess: + pass + await process.wait() + disconnected.cancel() + await asyncio.gather(disconnected, completed, return_exceptions=True) + + @staticmethod + async def _wait_disconnect(receive): + while True: + event = await receive() + if event["type"] == "http.disconnect": + return + + @staticmethod + def _read_failure(directory: Path): + path = directory / "failure.json" + return json.loads(path.read_text()) if path.exists() else None + + def _inspect_tree(self, pid: int, directory: Path, descendants: dict) -> None: + try: + parent = psutil.Process(pid) + processes = [parent, *parent.children(recursive=True)] + except psutil.NoSuchProcess: + return + memory = 0 + cpu = 0.0 + for process in processes: + try: + descendants[(process.pid, process.create_time())] = process + memory += process.memory_info().rss + times = process.cpu_times() + cpu += times.user + times.system + except psutil.NoSuchProcess: + continue + # Bound retained process identities across repeated short-lived conversions. + for identity, process in list(descendants.items()): + if not process.is_running(): + del descendants[identity] + if len(descendants) > self.settings.process_limit: + raise ResourceLimit("process_limit") + if len(processes) > self.settings.process_limit: + raise ResourceLimit("process_limit") + if memory > self.settings.memory_bytes: + raise ResourceLimit("memory_bytes") + if cpu > self.settings.cpu_seconds: + raise ResourceLimit("cpu_seconds") + size = 0 + entries = 0 + for root, directories, files in os.walk(directory, followlinks=False): + entries += len(directories) + len(files) + if entries > 8192: + raise ResourceLimit("temp_entries") + for name in files: + try: + size += os.lstat(os.path.join(root, name)).st_size + except FileNotFoundError: + continue + if size > self.settings.temp_bytes: + raise ResourceLimit("temp_bytes") + + +app = Gateway(Settings()) diff --git a/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/worker.py b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/worker.py new file mode 100644 index 00000000000..03e7e274ec4 --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/kfs_sandbox/worker.py @@ -0,0 +1,168 @@ +"""Disposable ASGI replay worker. Resource limits precede any provider imports.""" + +from __future__ import annotations + +import asyncio +import importlib +import json +import os +import resource +import sys +from email import policy +from email.parser import BytesParser +from pathlib import Path + +from .admission import Budget, Rejected, inspect_document +from .conversion import initialize_budget, load_budget + + +def apply_limits(settings: dict) -> None: + for key, value in ( + (resource.RLIMIT_CPU, int(settings["cpu_seconds"])), + (resource.RLIMIT_FSIZE, settings["file_bytes"]), + (resource.RLIMIT_NOFILE, 256), + ): + resource.setrlimit(key, (value, value)) + # macOS exposes RLIMIT_AS but cannot reliably apply it; production image is Linux. + if sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_AS, (settings["address_space_bytes"],) * 2) + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +def inspect_multipart(body: bytes, content_type: str) -> Budget: + if ( + not content_type.lower().startswith("multipart/form-data;") + or "\r" in content_type + or "\n" in content_type + ): + raise Rejected("multipart_required") + message = BytesParser(policy=policy.default).parsebytes( + b"MIME-Version: 1.0\r\nContent-Type: " + + content_type.encode("latin-1") + + b"\r\n\r\n" + + body + ) + if not message.is_multipart() or message.defects: + raise Rejected("multipart_invalid") + budget = Budget() + file_count = 0 + fields = 0 + for part in message.iter_parts(): + fields += 1 + if fields > 128: + raise Rejected("multipart_fields") + if part.defects or part.is_multipart(): + raise Rejected("multipart_invalid") + if part.get_filename() is None: + continue + # KnowledgeFS submits exactly one document per request. Upstream batch semantics + # are deliberately unavailable on this optional, internal-only service boundary. + file_count += 1 + if file_count > 1: + raise Rejected("multipart_files") + data = part.get_payload(decode=True) + if data is None: + raise Rejected("multipart_invalid") + inspect_document(data, part.get_filename(), budget) + if file_count != 1: + raise Rejected("multipart_files") + return budget + + +async def execute(directory: Path, envelope: dict) -> None: + body = (directory / "request.body").read_bytes() + scope = envelope["scope"] + scope["headers"] = [ + (name.encode("latin-1"), value.encode("latin-1")) + for name, value in scope["headers"] + ] + scope["query_string"] = scope["query_string"].encode("latin-1") + budget = inspect_multipart( + body, dict(scope["headers"]).get(b"content-type", b"").decode("latin-1") + ) + initialize_budget( + directory, budget, wall_seconds=envelope["settings"]["wall_seconds"] + ) + module, name = envelope["app_target"].split(":", 1) + app = getattr(importlib.import_module(module), name) + supplied = False + response_size = 0 + start = None + + async def receive(): + nonlocal supplied + if not supplied: + supplied = True + return {"type": "http.request", "body": body, "more_body": False} + await asyncio.Event().wait() + + with (directory / "response.body").open("wb") as output: + + async def send(message): + nonlocal response_size, start + if message["type"] == "http.response.start": + start = { + "status": message["status"], + "headers": [ + (name.decode("latin-1"), value.decode("latin-1")) + for name, value in message.get("headers", []) + ], + } + elif message["type"] == "http.response.body": + chunk = message.get("body", b"") + response_size += len(chunk) + if response_size > envelope["settings"]["response_bytes"]: + raise Rejected("response_bytes") + output.write(chunk) + + try: + await app(scope, receive, send) + finally: + # The upstream API may catch a converter's failure or skip an attachment. + # A sticky shared admission rejection must still reject the whole document. + _, failure = load_budget(directory) + if failure: + raise Rejected(failure) + if start is None: + raise RuntimeError("Missing ASGI response") + (directory / "response.json").write_text(json.dumps(start)) + + +def main() -> None: + directory = Path(sys.argv[1]) + envelope = json.loads((directory / "request.json").read_text()) + apply_limits(envelope["settings"]) + # A request must not recursively wait on the service-wide admission queue it owns. + # Serial local partition preserves semantics; throughput is a required rollout gate. + os.environ["UNSTRUCTURED_PARALLEL_MODE_ENABLED"] = "false" + os.environ["TMPDIR"] = str(directory) + os.environ["XDG_CONFIG_HOME"] = str(directory / "config") + os.environ["KFS_CONVERSION_DIRECTORY"] = str(directory) + try: + asyncio.run(execute(directory, envelope)) + except Rejected as error: + (directory / "failure.json").write_text( + json.dumps( + { + "status": 504 + if error.reason == "wall_seconds" + else 413 + if error.reason in {"response_bytes", "worker_resource_limit"} + else 422, + "reason": error.reason, + } + ) + ) + except (MemoryError, OSError): + (directory / "failure.json").write_text( + json.dumps({"status": 413, "reason": "worker_resource_limit"}) + ) + except Exception: + # Never copy untrusted provider exception strings or file paths into an error. + (directory / "failure.json").write_text( + json.dumps({"status": 502, "reason": "worker_failed"}) + ) + + +if __name__ == "__main__": + main() diff --git a/knowledge-fs/services/unstructured-sandbox/tests/__init__.py b/knowledge-fs/services/unstructured-sandbox/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/knowledge-fs/services/unstructured-sandbox/tests/fake_converter.py b/knowledge-fs/services/unstructured-sandbox/tests/fake_converter.py new file mode 100644 index 00000000000..eb6fe5b728a --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/fake_converter.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Tiny external executable fixture for the converter boundary (not shipped).""" + +import os +import signal +import sys +from pathlib import Path + +args = sys.argv[1:] +if args in (["--version"], ["--list-input-formats"], ["--list-output-formats"]): + print("pandoc 3.9" if args == ["--version"] else "html\ndocx\nrtf\nepub\nodt") + raise SystemExit(0) +mode = os.environ.get("KFS_FAKE_CONVERTER_MODE") +if mode == "signal": + os.kill(os.getpid(), signal.SIGKILL) +if mode == "error": + sys.stderr.write("converter failed") + raise SystemExit(42) +if mode == "large": + sys.stdout.write("x" * 10000) + raise SystemExit(0) +if mode == "wait": + import time + + time.sleep(30) +if "--outdir" in args: + source = Path(args[-1]) + target_format = args[args.index("--convert-to") + 1].split(":", 1)[0] + output = Path(args[args.index("--outdir") + 1]) / f"{source.stem}.{target_format}" + if mode == "missing": + raise SystemExit(0) + if mode == "symlink": + output.symlink_to(source) + raise SystemExit(0) + output.write_bytes(source.read_bytes()) + if mode == "collision": + (source.parent / f"{source.stem}.{target_format}").write_bytes( + b"concurrent output" + ) + print("Converted fixture") +else: + source = Path(next(arg for arg in args if not arg.startswith("-"))) + output = next( + (arg.split("=", 1)[1] for arg in args if arg.startswith("--output=")), None + ) + if output: + Path(output).write_bytes(source.read_bytes()) + else: + sys.stdout.buffer.write(source.read_bytes()) diff --git a/knowledge-fs/services/unstructured-sandbox/tests/fake_provider.py b/knowledge-fs/services/unstructured-sandbox/tests/fake_provider.py new file mode 100644 index 00000000000..03663f3a99c --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/fake_provider.py @@ -0,0 +1,39 @@ +"""Local-only ASGI fixtures for real worker-process tests; never included in the image.""" + +import asyncio +import json +import os +import subprocess +import sys + + +async def app(scope, receive, send): + body = b"" + while True: + message = await receive() + body += message.get("body", b"") + if not message.get("more_body"): + break + mode = dict(scope["headers"]).get(b"x-test-mode", b"echo") + if mode == b"wait": + await asyncio.sleep(60) + if mode == b"spawn": + process = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(60)"] + ) + with open(os.path.join(os.environ["TMPDIR"], "descendant.pid"), "w") as output: + output.write(str(process.pid)) + await asyncio.sleep(60) + response = ( + b"X" * 10000 + if mode == b"large" + else json.dumps({"input_bytes": len(body), "worker": os.getpid()}).encode() + ) + await send( + { + "type": "http.response.start", + "status": 201, + "headers": [(b"content-type", b"application/json")], + } + ) + await send({"type": "http.response.body", "body": response}) diff --git a/knowledge-fs/services/unstructured-sandbox/tests/test_admission.py b/knowledge-fs/services/unstructured-sandbox/tests/test_admission.py new file mode 100644 index 00000000000..11925ea75fb --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/test_admission.py @@ -0,0 +1,300 @@ +"""Small, bounded document fixtures: never send a decompression bomb to a parser.""" + +import io +import unittest +import zipfile +import hashlib +from pathlib import Path +from unittest.mock import MagicMock, patch +from types import SimpleNamespace +from email.message import EmailMessage + +from kfs_sandbox.admission import ( + Budget, + Limits, + Rejected, + inspect_document, + _is_outlook_storage, + _msg_attachments, +) + + +def archive(parts): + output = io.BytesIO() + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as handle: + for name, body in parts.items(): + handle.writestr(name, body) + return output.getvalue() + + +def message(attachments=()): + value = EmailMessage() + value["Subject"] = "Fixture" + value.set_content("Searchable message body") + for filename, body in attachments: + value.add_attachment( + body, maintype="application", subtype="octet-stream", filename=filename + ) + return value.as_bytes() + + +class AdmissionTests(unittest.TestCase): + def test_pinned_public_msg_fixture_is_admitted_with_real_oxmsg_reader(self): + fixture = ( + Path(__file__).resolve().parents[1] / "fixtures/fake-email-attachment.msg" + ) + body = fixture.read_bytes() + self.assertEqual( + hashlib.sha256(body).hexdigest(), + "92f65236e7eae301ea6f70a85f38cd5a9fae9f807d17fc35c5b5dbcf0f82a8ec", + ) + for name in ("fixture.msg", "renamed.xls"): + budget = inspect_document(body, name) + self.assertEqual(budget.attachments, 1) + self.assertEqual(budget.decoded_bytes, 30) + + def test_plain_mail_remains_accepted(self): + budget = inspect_document(message(), "sample.eml") + self.assertEqual(budget.attachments, 0) + + def test_attachment_count_is_shared_across_nested_messages(self): + nested = message([("a.txt", b"A"), ("b.txt", b"B")]) + with self.assertRaisesRegex(Rejected, "attachments"): + inspect_document( + message([("nested.eml", nested)]), + "outer.eml", + Budget(Limits(attachments=2)), + ) + + def test_attachment_bytes_are_aggregate_not_per_file(self): + with self.assertRaisesRegex(Rejected, "decoded_bytes"): + inspect_document( + message([("a.txt", b"123"), ("b.txt", b"456")]), + "a.eml", + Budget(Limits(decoded_bytes=5)), + ) + + def test_mime_depth_is_bounded(self): + nested = message([("a.eml", message([("b.eml", message())]))]) + with self.assertRaisesRegex(Rejected, "depth"): + inspect_document(nested, "outer.eml", Budget(Limits(depth=1))) + + def test_unknown_attachments_are_explicitly_rejected_not_silently_skipped(self): + with self.assertRaisesRegex(Rejected, "unsupported_attachment"): + inspect_document(message([("program.exe", b"MZ\x00")]), "a.eml") + + def test_nested_xlsx_sparse_extent_is_rejected_before_partition(self): + xlsx = archive( + { + "xl/worksheets/sheet1.xml": '' + } + ) + with self.assertRaisesRegex(Rejected, "sheet_"): + inspect_document(message([("sheet.xlsx", xlsx)]), "a.eml") + + def test_zip_budget_is_shared_between_attachments(self): + payload = archive({"word/document.xml": "abcdef"}) + with self.assertRaisesRegex(Rejected, "expanded_bytes"): + inspect_document( + message([("a.docx", payload), ("b.docx", payload)]), + "a.eml", + Budget(Limits(expanded_bytes=40)), + ) + + def test_xml_entities_are_rejected(self): + docx = archive( + {"word/document.xml": ']>&x;'} + ) + with self.assertRaisesRegex(Rejected, "xml_entity"): + inspect_document(docx, "a.docx") + + def test_unsafe_zip_path_is_rejected(self): + with self.assertRaisesRegex(Rejected, "archive_path"): + inspect_document(archive({"../outside.xml": ""}), "a.docx") + + def test_zip_extension_spoofing_does_not_bypass_inspection(self): + with self.assertRaisesRegex(Rejected, "sheet_"): + inspect_document( + archive( + { + "xl/worksheets/sheet1.xml": '' + } + ), + "a.txt", + ) + + def test_safe_archive_and_mime_parts_preserve_a_shared_report(self): + payload = archive({"word/document.xml": "ordinary"}) + budget = inspect_document(message([("report.docx", payload)]), "sample.eml") + self.assertEqual(budget.attachments, 1) + self.assertEqual(budget.archive_entries, 1) + self.assertEqual(budget.expanded_bytes, 29) + + def test_msg_attachments_use_the_same_recursive_budget(self): + attachments = [("nested.eml", message([("a.txt", b"A")]))] + with self.assertRaisesRegex(Rejected, "attachments"): + inspect_document( + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", + "a.msg", + Budget(Limits(attachments=1)), + msg_loader=lambda _: attachments, + ) + + def test_invalid_zip_is_a_stable_input_error(self): + with self.assertRaisesRegex(Rejected, "archive_invalid"): + inspect_document(b"PK\x03\x04broken", "a.docx") + + def test_xml_and_sheet_limits_apply_across_members(self): + with self.assertRaisesRegex(Rejected, "sheets"): + inspect_document( + archive({"a.xml": "", "b.xml": ""}), + "a.docx", + Budget(Limits(sheets=1)), + ) + with self.assertRaisesRegex(Rejected, "xml_depth"): + inspect_document( + archive({"a.xml": ""}), + "a.docx", + Budget(Limits(xml_depth=2)), + ) + with self.assertRaisesRegex(Rejected, "xml_nodes"): + inspect_document( + archive({"a.xml": ""}), + "a.docx", + Budget(Limits(xml_nodes=2)), + ) + with self.assertRaisesRegex(Rejected, "xml_bytes"): + inspect_document( + archive({"a.xml": ""}), "a.docx", Budget(Limits(xml_bytes=2)) + ) + with self.assertRaisesRegex(Rejected, "xml_member_bytes"): + inspect_document( + archive({"a.xml": ""}), "a.docx", Budget(Limits(xml_member_bytes=2)) + ) + + def test_cell_shape_and_logical_span_are_bounded(self): + with self.assertRaisesRegex(Rejected, "sheet_cell_reference"): + inspect_document( + archive({"a.xml": ''}), "a.xlsx" + ) + with self.assertRaisesRegex(Rejected, "sheet_cells"): + inspect_document( + archive({"a.xml": ''}), + "a.xlsx", + Budget(Limits(sheet_cells=100)), + ) + with self.assertRaisesRegex(Rejected, "workbook_cells"): + inspect_document( + archive( + { + "a.xml": '', + "b.xml": '', + } + ), + "a.xlsx", + Budget(Limits(workbook_cells=7)), + ) + + def test_msg_failure_is_sanitized(self): + def broken(_): + raise ValueError("Secret document text") + + with self.assertRaisesRegex(Rejected, "^msg_invalid$"): + inspect_document(b"bad", "a.msg", msg_loader=broken) + + def test_archive_entry_count_is_bounded(self): + with self.assertRaisesRegex(Rejected, "archive_entries"): + inspect_document( + archive({"one.bin": "1", "two.bin": "2"}), + "a.docx", + Budget(Limits(archive_entries=1)), + ) + + def test_renamed_mail_still_has_nested_budget(self): + with self.assertRaisesRegex(Rejected, "attachments"): + inspect_document( + message([("a.txt", b"A"), ("b.txt", b"B")]), + "sample.txt", + Budget(Limits(attachments=1)), + ) + + def test_rfc822_attachments_and_mime_part_count_are_bounded(self): + outer = EmailMessage() + outer["Subject"] = "RFC822" + outer.set_content("Body") + nested = EmailMessage() + nested.set_content("Nested body") + outer.add_attachment(nested) + self.assertEqual(inspect_document(outer.as_bytes(), "a.eml").attachments, 1) + with self.assertRaisesRegex(Rejected, "mime_parts"): + inspect_document(outer.as_bytes(), "a.eml", Budget(Limits(mime_parts=2))) + + def test_implicit_xlsx_cell_positions_are_valid_and_still_bounded(self): + payload = archive({"a.xml": ''}) + budget = inspect_document(payload, "a.xlsx") + self.assertEqual(budget.workbook_cells, 4) + with self.assertRaisesRegex(Rejected, "sheet_cells"): + inspect_document(payload, "a.xlsx", Budget(Limits(sheet_cells=3))) + + def test_msg_content_signature_wins_over_a_legacy_office_filename(self): + with patch("kfs_sandbox.admission._is_outlook_storage", return_value=True): + with self.assertRaisesRegex(Rejected, "attachments"): + inspect_document( + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", + "disguised.xls", + Budget(Limits(attachments=1)), + msg_loader=lambda _: [("a.txt", b"a"), ("b.txt", b"b")], + ) + + def test_msg_reader_contract_and_ole_signature_diagnostics(self): + msg = SimpleNamespace( + Message=SimpleNamespace( + load=lambda _: SimpleNamespace( + attachments=[ + SimpleNamespace(file_name="a.txt", file_bytes=b"A"), + SimpleNamespace(file_name=None, file_bytes=None), + ] + ) + ) + ) + with patch.dict("sys.modules", {"oxmsg": msg}): + self.assertEqual( + list(_msg_attachments(b"fixture")), [("a.txt", b"A"), ("unknown", b"")] + ) + handle = MagicMock() + handle.__enter__.return_value.exists.return_value = True + with patch.dict( + "sys.modules", {"olefile": SimpleNamespace(OleFileIO=lambda _: handle)} + ): + self.assertTrue(_is_outlook_storage(b"fixture")) + + def invalid(_): + raise OSError("unsafe details") + + with patch.dict("sys.modules", {"olefile": SimpleNamespace(OleFileIO=invalid)}): + with self.assertRaisesRegex(Rejected, "^ole_invalid$"): + _is_outlook_storage(b"fixture") + + def test_archive_binary_members_are_counted_and_encryption_rejected(self): + payload = archive({"media/image.bin": b"bytes"}) + self.assertEqual(inspect_document(payload, "a.docx").expanded_bytes, 5) + encrypted = bytearray(payload) + encrypted[6] |= 1 + encrypted[encrypted.index(b"PK\x01\x02") + 8] |= 1 + with self.assertRaisesRegex(Rejected, "archive_encrypted"): + inspect_document(bytes(encrypted), "a.docx") + + def test_invalid_mime_and_implicit_row_extents_are_rejected(self): + with self.assertRaisesRegex(Rejected, "mime_invalid"): + inspect_document( + b"MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=missing\r\n\r\nbody", + "a.eml", + ) + with self.assertRaisesRegex(Rejected, "sheet_extent"): + inspect_document( + archive({"a.xml": ''}), "a.xlsx" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/knowledge-fs/services/unstructured-sandbox/tests/test_conversion.py b/knowledge-fs/services/unstructured-sandbox/tests/test_conversion.py new file mode 100644 index 00000000000..06ed3bb7b1a --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/test_conversion.py @@ -0,0 +1,436 @@ +import concurrent.futures +import io +import json +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from kfs_sandbox.admission import Budget, Limits, Rejected +from kfs_sandbox.conversion import ( + convert, + initialize_budget, + load_budget, + parse_invocation, +) +from kfs_sandbox.converter_cli import run as run_cli, main as cli_main +from tests.test_admission import archive + + +FAKE = [sys.executable, str(Path(__file__).with_name("fake_converter.py"))] + + +class ConversionTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="kfs-conversion-test-") + self.directory = Path(self.temporary.name) + initialize_budget(self.directory, Budget(), wall_seconds=5) + + def tearDown(self): + self.temporary.cleanup() + + def source(self, name, body): + path = self.directory / name + path.write_bytes(body) + return path + + def invoke(self, kind, args, **kwargs): + stdout, stderr = io.BytesIO(), io.BytesIO() + code = convert(kind, args, self.directory, FAKE, stdout, stderr, **kwargs) + return code, stdout.getvalue(), stderr.getvalue() + + def test_soffice_output_is_validated_then_published_without_changing_success_output( + self, + ): + original = archive({"word/document.xml": "evidence"}) + source = self.source("sample.doc", original) + result = self.invoke( + "soffice", + [ + "--headless", + "--convert-to", + "docx:MS Word 2007 XML", + "--outdir", + str(self.directory), + str(source), + ], + ) + self.assertEqual(result, (0, b"Converted fixture\n", b"")) + self.assertEqual((self.directory / "sample.docx").read_bytes(), original) + self.assertEqual(load_budget(self.directory)[0].archive_entries, 1) + + def test_unsafe_generated_office_file_is_never_published(self): + source = self.source( + "sample.ppt", + archive({"a.xml": ''}), + ) + with self.assertRaisesRegex(Rejected, "sheet_extent"): + self.invoke( + "soffice", + [ + "--headless", + "--convert-to", + "pptx", + "--outdir", + str(self.directory), + str(source), + ], + ) + self.assertFalse((self.directory / "sample.pptx").exists()) + self.assertEqual(load_budget(self.directory)[1], "sheet_extent") + + def test_pandoc_html_stdout_is_unchanged_but_released_only_after_admission(self): + body = b"

Evidence & reference

" + source = self.source("sample.rtf", body) + result = self.invoke( + "pandoc", ["--from=rtf", "--to=html", str(source), "--sandbox"] + ) + self.assertEqual(result, (0, body, b"")) + self.assertGreater(load_budget(self.directory)[0].xml_nodes, 0) + + def test_pandoc_generated_docx_file_uses_shared_budget(self): + initialize_budget( + self.directory, Budget(Limits(xml_nodes=2), xml_nodes=2), wall_seconds=5 + ) + source = self.source( + "sample.odt", archive({"word/document.xml": ""}) + ) + target = self.directory / "sample.docx" + with self.assertRaisesRegex(Rejected, "xml_nodes"): + self.invoke( + "pandoc", + [ + "--from=odt", + "--to=docx", + str(source), + f"--output={target}", + "--sandbox", + ], + ) + self.assertFalse(target.exists()) + + def test_html_depth_and_table_expansion_are_checked_before_stdout_release(self): + for body, limits, reason in ( + (b"

x

", Limits(xml_depth=2), "xml_depth"), + ( + b"
x
", + Limits(), + "sheet_extent", + ), + ): + initialize_budget(self.directory, Budget(limits), wall_seconds=5) + source = self.source("sample.rtf", body) + stdout = io.BytesIO() + with self.assertRaisesRegex(Rejected, reason): + convert( + "pandoc", + ["--from=rtf", "--to=html", str(source)], + self.directory, + FAKE, + stdout, + io.BytesIO(), + ) + self.assertEqual(stdout.getvalue(), b"") + + def test_concurrent_conversions_cannot_double_spend_shared_budget(self): + initialize_budget(self.directory, Budget(Limits(xml_nodes=1)), wall_seconds=5) + sources = [self.source(f"{index}.rtf", b"

x

") for index in range(2)] + + def attempt(source): + try: + self.invoke("pandoc", ["--from=rtf", "--to=html", str(source)]) + return "passed" + except Rejected: + return "rejected" + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + self.assertEqual(sorted(pool.map(attempt, sources)), ["passed", "rejected"]) + self.assertEqual(load_budget(self.directory)[0].xml_nodes, 1) + + def test_unknown_flags_and_paths_outside_request_directory_fail_before_execution( + self, + ): + source = self.source("sample.rtf", b"

x

") + for args in ( + ["--from=rtf", "--to=html", str(source), "--lua-filter=bad.lua"], + ["--from=rtf", "--to=html", "/etc/passwd"], + ["--from=odt", "--to=docx", str(source), "--output=/tmp/outside.docx"], + ): + with self.assertRaisesRegex(Rejected, "archive_path|archive_invalid"): + parse_invocation("pandoc", args, self.directory) + + def test_existing_output_is_not_overwritten(self): + source = self.source("sample.doc", archive({"a.xml": ""})) + target = self.source("sample.docx", b"existing") + with self.assertRaisesRegex(Rejected, "archive_path"): + self.invoke( + "soffice", + [ + "--headless", + "--convert-to", + "docx", + "--outdir", + str(self.directory), + str(source), + ], + ) + self.assertEqual(target.read_bytes(), b"existing") + + def test_concurrent_output_creation_cannot_be_overwritten_after_validation(self): + source = self.source("sample.doc", archive({"a.xml": ""})) + with patch.dict(os.environ, {"KFS_FAKE_CONVERTER_MODE": "collision"}): + with self.assertRaisesRegex(Rejected, "archive_path"): + self.invoke( + "soffice", + [ + "--headless", + "--convert-to", + "docx", + "--outdir", + str(self.directory), + str(source), + ], + ) + self.assertEqual( + (self.directory / "sample.docx").read_bytes(), b"concurrent output" + ) + + def test_converter_nonzero_status_and_stderr_are_preserved(self): + source = self.source("sample.rtf", b"body") + with patch.dict(os.environ, {"KFS_FAKE_CONVERTER_MODE": "error"}): + self.assertEqual( + self.invoke("pandoc", ["--from=rtf", "--to=html", str(source)]), + (42, b"", b"converter failed"), + ) + self.assertIsNone(load_budget(self.directory)[1]) + + def test_converter_signal_termination_is_sticky_and_never_publishes_output(self): + source = self.source("sample.doc", archive({"a.xml": ""})) + with patch.dict(os.environ, {"KFS_FAKE_CONVERTER_MODE": "signal"}): + with self.assertRaisesRegex(Rejected, "worker_resource_limit"): + self.invoke( + "soffice", + [ + "--headless", + "--convert-to", + "docx", + "--outdir", + str(self.directory), + str(source), + ], + ) + self.assertFalse((self.directory / "sample.docx").exists()) + self.assertEqual(load_budget(self.directory)[1], "worker_resource_limit") + + def test_missing_symlink_or_oversized_generated_file_is_not_published(self): + source = self.source("sample.doc", archive({"a.xml": ""})) + for mode, cap, reason in ( + ("missing", 1024, "archive_invalid"), + ("symlink", 1024, "archive_invalid"), + ("", 10, "xml_member_bytes"), + ): + initialize_budget(self.directory, Budget(), wall_seconds=5) + with ( + self.subTest(mode=mode), + patch.dict(os.environ, {"KFS_FAKE_CONVERTER_MODE": mode}), + ): + with self.assertRaisesRegex(Rejected, reason): + self.invoke( + "soffice", + [ + "--headless", + "--convert-to", + "docx", + "--outdir", + str(self.directory), + str(source), + ], + max_product_bytes=cap, + ) + self.assertFalse((self.directory / "sample.docx").exists()) + + def test_stdout_is_bounded_before_loading_or_forwarding(self): + source = self.source("sample.rtf", b"body") + with patch.dict(os.environ, {"KFS_FAKE_CONVERTER_MODE": "large"}): + with self.assertRaisesRegex(Rejected, "xml_member_bytes"): + self.invoke( + "pandoc", + ["--from=rtf", "--to=html", str(source)], + max_product_bytes=100, + ) + + def test_version_and_format_probes_preserve_upstream_discovery(self): + result = self.invoke("pandoc", ["--version"]) + self.assertEqual(result, (0, b"pandoc 3.9\n", b"")) + self.assertEqual(load_budget(self.directory)[0].xml_nodes, 0) + + def test_invocation_contract_rejects_unrecognized_shapes(self): + source = self.source("a.rtf", b"body") + cases = [ + ("unknown", []), + ("soffice", []), + ( + "soffice", + [ + "--headless", + "--convert-to", + "pdf", + "--outdir", + str(self.directory), + str(source), + ], + ), + ("pandoc", ["--from=rtf", "--to=html", "--from=rtf", str(source)]), + ("pandoc", ["--from=", "--to=html", str(source)]), + ("pandoc", ["--from=rtf", "--to=html", str(source), str(source)]), + ( + "pandoc", + ["--from=rtf", "--to=html", str(source), "--sandbox", "--sandbox"], + ), + ("pandoc", ["--from=rtf", "--to=docx", str(source)]), + ] + for kind, args in cases: + with self.subTest(kind=kind, args=args), self.assertRaises(Rejected): + parse_invocation(kind, args, self.directory) + + def test_symlink_and_directory_inputs_are_not_followed(self): + source = self.source("sample.rtf", b"body") + link = self.directory / "linked.rtf" + link.symlink_to(source) + for path in (link, self.directory): + with self.assertRaisesRegex(Rejected, "archive_path"): + parse_invocation( + "pandoc", ["--from=rtf", "--to=html", str(path)], self.directory + ) + + def test_html_rowspans_and_self_closing_tags_preserve_bounded_geometry(self): + body = b"
ab
c
d
" + source = self.source("sample.rtf", body) + self.assertEqual( + self.invoke("pandoc", ["--from=rtf", "--to=html", str(source)])[1], body + ) + self.assertEqual(load_budget(self.directory)[0].workbook_cells, 6) + + def test_html_invalid_spans_duplicates_bytes_and_rectangles_are_rejected(self): + cases = [ + ( + b"
x
", + Limits(), + "sheet_extent", + ), + ( + b"
x
", + Limits(), + "archive_invalid", + ), + ( + b"
x
", + Limits(sheet_cells=99), + "sheet_cells", + ), + ( + b"
x
", + Limits(), + "sheet_extent", + ), + (b"

x

", Limits(xml_member_bytes=2), "xml_member_bytes"), + (b"\xff", Limits(), "archive_invalid"), + ] + for body, limits, reason in cases: + initialize_budget(self.directory, Budget(limits), wall_seconds=5) + source = self.source("sample.rtf", body) + with self.assertRaisesRegex(Rejected, reason): + self.invoke("pandoc", ["--from=rtf", "--to=html", str(source)]) + + def test_converter_deadline_kills_the_external_process(self): + initialize_budget(self.directory, Budget(), wall_seconds=0.05) + source = self.source("sample.rtf", b"body") + with patch.dict(os.environ, {"KFS_FAKE_CONVERTER_MODE": "wait"}): + with self.assertRaisesRegex(Rejected, "wall_seconds"): + self.invoke("pandoc", ["--from=rtf", "--to=html", str(source)]) + + def test_rejected_budget_prevents_later_converter_launch(self): + initialize_budget( + self.directory, Budget(Limits(xml_nodes=1), xml_nodes=1), wall_seconds=5 + ) + source = self.source("sample.rtf", b"

x

") + for _ in range(2): + with self.assertRaisesRegex(Rejected, "xml_nodes"): + self.invoke("pandoc", ["--from=rtf", "--to=html", str(source)]) + + def test_cli_preserves_probes_and_maps_known_rejection_to_nonzero(self): + executable = self.directory / "fake-converter" + shutil.copyfile(FAKE[1], executable) + executable.chmod(0o700) + manifest = self.directory / "manifest.json" + manifest.write_text( + json.dumps({"pandoc": str(executable), "soffice": str(executable)}) + ) + output, errors = io.BytesIO(), io.BytesIO() + self.assertEqual( + run_cli("pandoc", ["--version"], self.directory, manifest, output, errors), + 0, + ) + self.assertEqual(output.getvalue(), b"pandoc 3.9\n") + source = self.source("sample.doc", b"invalid archive") + self.assertEqual( + run_cli( + "soffice", + [ + "--headless", + "--convert-to", + "docx", + "--outdir", + str(self.directory), + str(source), + ], + self.directory, + manifest, + output, + errors, + ), + 65, + ) + self.assertIn(b"admission rejected", errors.getvalue()) + + def test_cli_rejects_invalid_manifest_instead_of_using_path_fallback(self): + manifest = self.source("manifest.json", b"x" * 4097) + with self.assertRaisesRegex(RuntimeError, "manifest"): + run_cli( + "pandoc", + ["--version"], + self.directory, + manifest, + io.BytesIO(), + io.BytesIO(), + ) + manifest.write_text(json.dumps({"pandoc": "relative"})) + with self.assertRaisesRegex(RuntimeError, "executable"): + run_cli( + "pandoc", + ["--version"], + self.directory, + manifest, + io.BytesIO(), + io.BytesIO(), + ) + + def test_cli_main_never_prints_sensitive_exception_details(self): + errors = io.StringIO() + with ( + patch.dict(os.environ, {}, clear=True), + patch("kfs_sandbox.converter_cli.sys.stderr", errors), + ): + with self.assertRaises(SystemExit) as stopped: + cli_main("pandoc") + self.assertEqual(stopped.exception.code, 70) + self.assertEqual( + errors.getvalue(), "KnowledgeFS converter runtime is unavailable\n" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/knowledge-fs/services/unstructured-sandbox/tests/test_deployment.py b/knowledge-fs/services/unstructured-sandbox/tests/test_deployment.py new file mode 100644 index 00000000000..4b94f6824be --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/test_deployment.py @@ -0,0 +1,37 @@ +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +REPOSITORY = ROOT.parents[2] + + +class DeploymentTests(unittest.TestCase): + def test_image_is_pinned_and_worker_user_is_non_root(self): + dockerfile = (ROOT / "Dockerfile").read_text() + self.assertIn( + "@sha256:0df934a22e4e893cf15e7aeaf35c463ecc75937758a83099aefdc13041619a1d", + dockerfile, + ) + self.assertIn("USER 1000:1000", dockerfile) + self.assertIn('ENTRYPOINT ["python3", "-m", "kfs_sandbox"]', dockerfile) + self.assertIn('version("unstructured") == "0.22.18"', dockerfile) + + def test_override_hard_limits_apply_only_to_existing_dedicated_service(self): + override = ( + REPOSITORY / "docker/knowledge-fs-unstructured-sandbox.compose.yaml" + ).read_text() + self.assertIn(" knowledge_fs_unstructured:", override) + self.assertNotIn("\n unstructured:", override) + for expected in ( + "read_only: true", + "pids_limit: 192", + "init: true", + "size=1073741824", + "no-new-privileges:true", + ): + self.assertIn(expected, override) + + +if __name__ == "__main__": + unittest.main() diff --git a/knowledge-fs/services/unstructured-sandbox/tests/test_gateway.py b/knowledge-fs/services/unstructured-sandbox/tests/test_gateway.py new file mode 100644 index 00000000000..dcffef76ebf --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/test_gateway.py @@ -0,0 +1,285 @@ +import asyncio +import json +import os +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path +from unittest.mock import patch + +import psutil + +from kfs_sandbox.gateway import Gateway, Settings + + +def multipart(filename="sample.txt", body=b"content"): + return ( + b'--fixture\r\nContent-Disposition: form-data; name="files"; filename="' + + filename.encode() + + b'"\r\nContent-Type: application/octet-stream\r\n\r\n' + + body + + b"\r\n--fixture--\r\n" + ) + + +async def request( + gateway, + body, + mode=b"echo", + disconnect=None, + path="/general/v0/general", + method="POST", + blocked_send=False, +): + received = False + output = [] + + async def receive(): + nonlocal received + if not received: + received = True + return {"type": "http.request", "body": body} + if disconnect is not None: + await asyncio.sleep(disconnect) + return {"type": "http.disconnect"} + await asyncio.Event().wait() + + async def send(message): + output.append(message) + if blocked_send: + await asyncio.Event().wait() + + await gateway( + { + "type": "http", + "method": method, + "path": path, + "headers": [ + (b"content-type", b"multipart/form-data; boundary=fixture"), + (b"x-test-mode", mode), + ], + "query_string": b"", + "http_version": "1.1", + "scheme": "http", + "server": ("localhost", 8000), + "client": ("127.0.0.1", 1234), + }, + receive, + send, + ) + return output + + +class GatewayTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="kfs-sandbox-test-") + self.settings = Settings( + temp_root=self.directory.name, + wall_seconds=5, + memory_bytes=512 * 1024 * 1024, + address_space_bytes=16 * 1024**3, + ) + + def tearDown(self): + self.directory.cleanup() + + def gateway(self, **overrides): + return Gateway( + replace(self.settings, **overrides), app_target="tests.fake_provider:app" + ) + + async def test_original_asgi_status_and_body_are_preserved_in_separate_process( + self, + ): + body = multipart() + output = await request(self.gateway(), body) + self.assertEqual(output[0]["status"], 201) + result = json.loads(b"".join(event.get("body", b"") for event in output)) + self.assertEqual(result["input_bytes"], len(body)) + self.assertNotEqual(result["worker"], os.getpid()) + self.assertEqual(list(Path(self.directory.name).iterdir()), []) + + async def test_body_cap_is_checked_before_worker_start(self): + output = await request(self.gateway(input_bytes=20), multipart()) + self.assertEqual(output[0]["status"], 413) + + async def test_response_cap_is_enforced_inside_worker(self): + output = await request( + self.gateway(response_bytes=100), multipart(), mode=b"large" + ) + self.assertEqual(output[0]["status"], 413) + self.assertIn(b"response_bytes", output[-1]["body"]) + + async def test_wall_deadline_kills_the_request_process(self): + output = await request( + self.gateway(wall_seconds=0.4), multipart(), mode=b"wait" + ) + self.assertEqual(output[0]["status"], 504) + self.assertEqual(list(Path(self.directory.name).iterdir()), []) + + async def test_disconnect_releases_admission_and_removes_request_files(self): + gateway = self.gateway() + output = await request(gateway, multipart(), mode=b"wait", disconnect=0.2) + self.assertEqual(output, []) + self.assertEqual(gateway.active, 0) + self.assertEqual(list(Path(self.directory.name).iterdir()), []) + + async def test_shared_admission_rejects_overflow_without_spawning(self): + gateway = self.gateway(active_limit=1, queue_limit=0, wall_seconds=0.4) + first = asyncio.create_task(request(gateway, multipart(), mode=b"wait")) + while gateway.active == 0: + await asyncio.sleep(0) + second = await request(gateway, multipart()) + self.assertEqual(second[0]["status"], 429) + await first + + async def test_unknown_route_does_not_run_provider(self): + output = await request(self.gateway(), b"", path="/unknown") + self.assertEqual(output[0]["status"], 404) + + async def test_health_reports_policy_revision_without_loading_provider(self): + output = await request(self.gateway(), b"", path="/healthcheck", method="GET") + self.assertEqual(output[0]["status"], 200) + self.assertIn(b"sandbox-v1", output[-1]["body"]) + + async def test_nested_invalid_document_is_rejected_before_provider(self): + output = await request(self.gateway(), multipart("a.docx", b"bad zip")) + self.assertEqual(output[0]["status"], 422) + + async def test_timeout_kills_spawned_conversion_descendants(self): + gateway = self.gateway(wall_seconds=0.7) + task = asyncio.create_task(request(gateway, multipart(), mode=b"spawn")) + descendant = None + for _ in range(50): + paths = list(Path(self.directory.name).glob("*/descendant.pid")) + if paths: + descendant = int(paths[0].read_text()) + break + await asyncio.sleep(0.01) + self.assertIsNotNone(descendant) + output = await task + self.assertEqual(output[0]["status"], 504) + try: + self.assertEqual(psutil.Process(descendant).status(), psutil.STATUS_ZOMBIE) + except psutil.NoSuchProcess: + pass + + async def test_resource_limits_stop_worker_without_overloading_machine(self): + for limits, mode in ( + ({"memory_bytes": 1}, b"wait"), + ({"temp_bytes": 1}, b"wait"), + ({"process_limit": 1}, b"spawn"), + ): + with self.subTest(limits=limits): + output = await request(self.gateway(**limits), multipart(), mode=mode) + self.assertEqual(output[0]["status"], 413) + self.assertEqual(self.gateway().active, 0) + + async def test_full_queue_expires_without_leaking_admission(self): + gateway = self.gateway(queue_seconds=0.02, wall_seconds=0.3) + first = asyncio.create_task(request(gateway, multipart(), mode=b"wait")) + while gateway.active == 0: + await asyncio.sleep(0) + output = await request(gateway, multipart()) + self.assertEqual(output[0]["status"], 429) + self.assertIn(b"admission_timeout", output[-1]["body"]) + await first + self.assertEqual(gateway.queued, 0) + self.assertEqual(gateway.active, 0) + + async def test_simultaneous_burst_cannot_reserve_more_than_total_admission(self): + gateway = self.gateway(active_limit=1, queue_limit=0, wall_seconds=0.3) + + class YieldingSemaphore(asyncio.Semaphore): + async def acquire(self): + await asyncio.sleep(0) + return await super().acquire() + + gateway.slots = YieldingSemaphore(1) + tasks = [ + asyncio.create_task(request(gateway, multipart(), mode=b"wait")) + for _ in range(20) + ] + await asyncio.sleep(0) + observed = gateway.active + gateway.queued + output = await asyncio.gather(*tasks) + self.assertLessEqual(observed, 1) + self.assertEqual(sum(result[0]["status"] == 429 for result in output), 19) + + async def test_slow_response_client_cannot_hold_admission_past_deadline(self): + gateway = self.gateway(wall_seconds=0.3) + output = await asyncio.wait_for( + request(gateway, multipart(), blocked_send=True), timeout=1 + ) + self.assertEqual( + [ + item["status"] + for item in output + if item["type"] == "http.response.start" + ], + [201], + ) + self.assertEqual(gateway.active, 0) + + async def test_filesystem_failure_does_not_leak_slot(self): + gateway = self.gateway() + with patch("kfs_sandbox.gateway.tempfile.mkdtemp", side_effect=OSError): + output = await request(gateway, multipart()) + self.assertEqual(output[0]["status"], 503) + self.assertEqual(gateway.active, 0) + + async def test_body_receipt_disconnect_and_deadline_are_bounded(self): + gateway = self.gateway(wall_seconds=0.01) + for event in ({"type": "http.disconnect"}, None): + output = [] + + async def receive(): + if event is not None: + return event + await asyncio.Event().wait() + + async def send(message): + output.append(message) + + await gateway( + {"type": "http", "path": "/general/v0/general", "method": "POST"}, + receive, + send, + ) + self.assertEqual(gateway.active, 0) + if event is None: + self.assertEqual(output[0]["status"], 504) + else: + self.assertEqual(output, []) + + async def test_lifespan_completes_and_websockets_are_not_parser_jobs(self): + events = iter([{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]) + output = [] + + async def receive(): + return next(events) + + async def send(message): + output.append(message) + + gateway = self.gateway() + await gateway({"type": "lifespan"}, receive, send) + await gateway({"type": "websocket"}, receive, send) + self.assertEqual( + [item["type"] for item in output], + ["lifespan.startup.complete", "lifespan.shutdown.complete"], + ) + + def test_invalid_limits_fail_startup(self): + for values in ( + {"memory_bytes": 0}, + {"active_limit": 9}, + {"queue_limit": 65}, + {"queue_limit": -1}, + ): + with self.assertRaises(ValueError): + replace(self.settings, **values) + + +if __name__ == "__main__": + unittest.main() diff --git a/knowledge-fs/services/unstructured-sandbox/tests/test_golden.py b/knowledge-fs/services/unstructured-sandbox/tests/test_golden.py new file mode 100644 index 00000000000..3b455e71a6a --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/test_golden.py @@ -0,0 +1,43 @@ +import importlib.util +import unittest +from pathlib import Path + + +SPEC = importlib.util.spec_from_file_location( + "sandbox_golden", Path(__file__).resolve().parents[1] / "golden.py" +) +golden = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(golden) + + +class GoldenTests(unittest.TestCase): + def test_default_msg_fixture_is_version_pinned_and_requires_attachment_content( + self, + ): + body, marker = golden.msg_fixture(None) + self.assertEqual(len(body), 15872) + self.assertEqual(marker, "Hey this is a fake attachment!") + + def test_pdf_fixture_has_bounded_geometry_and_valid_xref(self): + body = golden.pdf_fixture() + self.assertIn(b"/MediaBox [0 0 300 200]", body) + self.assertIn(golden.MARKER.encode(), body) + startxref = int(body.split(b"startxref\n")[1].splitlines()[0]) + self.assertEqual(body[startxref : startxref + 4], b"xref") + self.assertLess(len(body), 1024) + + def test_multipart_preserves_document_and_declares_strategy(self): + body, content_type = golden.multipart("a.txt", b"example") + self.assertIn(b'filename="a.txt"', body) + self.assertIn(b"example", body) + self.assertIn(b"fast", body) + self.assertIn("boundary=", content_type) + + def test_real_provider_content_failure_cannot_be_reported_as_pass(self): + self.assertTrue(golden.has_evidence([{"text": golden.MARKER}])) + self.assertFalse(golden.has_evidence([{"text": "unrelated"}])) + self.assertFalse(golden.has_evidence({"text": golden.MARKER})) + + +if __name__ == "__main__": + unittest.main() diff --git a/knowledge-fs/services/unstructured-sandbox/tests/test_worker.py b/knowledge-fs/services/unstructured-sandbox/tests/test_worker.py new file mode 100644 index 00000000000..00165fe150f --- /dev/null +++ b/knowledge-fs/services/unstructured-sandbox/tests/test_worker.py @@ -0,0 +1,185 @@ +import asyncio +import json +import runpy +import tempfile +import unittest +from dataclasses import asdict +from pathlib import Path +from unittest.mock import patch +from types import SimpleNamespace + +from kfs_sandbox.admission import Rejected +from kfs_sandbox.conversion import record_failure +from kfs_sandbox.gateway import Settings +from kfs_sandbox.worker import apply_limits, execute, inspect_multipart, main +from tests.test_gateway import multipart + + +class WorkerTests(unittest.IsolatedAsyncioTestCase): + def test_entrypoint_keeps_a_single_admission_supervisor(self): + with patch("uvicorn.run") as run: + runpy.run_module("kfs_sandbox", run_name="__main__") + self.assertEqual(run.call_args.kwargs["workers"], 1) + self.assertEqual(run.call_args.kwargs["limit_concurrency"], 64) + + def test_invalid_and_batched_multipart_is_explicitly_rejected(self): + for content_type in ( + "application/json", + "multipart/form-data; boundary=fixture\r\nBad: bad", + ): + with self.assertRaisesRegex(Rejected, "multipart_required"): + inspect_multipart(b"", content_type) + with self.assertRaisesRegex(Rejected, "multipart_invalid"): + inspect_multipart(b"bad", "multipart/form-data; boundary=fixture") + with self.assertRaisesRegex(Rejected, "multipart_files"): + inspect_multipart( + multipart().replace(b"--fixture--\r\n", b"") + multipart(), + "multipart/form-data; boundary=fixture", + ) + with self.assertRaisesRegex(Rejected, "multipart_files"): + inspect_multipart( + b'--fixture\r\nContent-Disposition: form-data; name="strategy"\r\n\r\nfast\r\n--fixture--\r\n', + "multipart/form-data; boundary=fixture", + ) + + def test_resource_limits_are_applied_before_provider_load(self): + with ( + patch("kfs_sandbox.worker.resource.setrlimit") as apply, + patch("kfs_sandbox.worker.sys.platform", "linux"), + ): + apply_limits(asdict(Settings())) + self.assertEqual(len(apply.call_args_list), 5) + + def test_multipart_field_and_nested_part_limits(self): + field = ( + b'--fixture\r\nContent-Disposition: form-data; name="a"\r\n\r\nvalue\r\n' + ) + with self.assertRaisesRegex(Rejected, "multipart_fields"): + inspect_multipart( + field * 129 + b"--fixture--\r\n", + "multipart/form-data; boundary=fixture", + ) + nested = b'--fixture\r\nContent-Disposition: form-data; name="a"\r\nContent-Type: multipart/mixed; boundary=inner\r\n\r\n--inner\r\n\r\nbody\r\n--inner--\r\n--fixture--\r\n' + with self.assertRaisesRegex(Rejected, "multipart_invalid"): + inspect_multipart(nested, "multipart/form-data; boundary=fixture") + + async def test_provider_missing_response_is_not_reported_as_success(self): + async def no_response(scope, receive, send): + await receive() + with self.assertRaises(TimeoutError): + await asyncio.wait_for(receive(), 0.001) + await send({"type": "http.response.debug"}) + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + (path / "request.body").write_bytes(multipart()) + with patch( + "kfs_sandbox.worker.importlib.import_module", + return_value=SimpleNamespace(app=no_response), + ): + with self.assertRaisesRegex(RuntimeError, "Missing ASGI response"): + await execute( + path, + { + "scope": { + "headers": [ + ( + "content-type", + "multipart/form-data; boundary=fixture", + ) + ], + "query_string": "", + }, + "app_target": "fixture:app", + "settings": asdict(Settings()), + }, + ) + + async def test_replay_preserves_input_and_complete_response(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "request.body").write_bytes(multipart()) + await execute( + root, + { + "settings": asdict(Settings()), + "app_target": "tests.fake_provider:app", + "scope": { + "headers": [ + ("content-type", "multipart/form-data; boundary=fixture") + ], + "query_string": "", + }, + }, + ) + self.assertEqual( + json.loads((root / "response.json").read_text())["status"], 201 + ) + + async def test_provider_cannot_turn_caught_converter_rejection_into_success(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "request.body").write_bytes(multipart()) + + async def swallowed_error(scope, receive, send): + await receive() + record_failure(root, "worker_resource_limit") + await send({"type": "http.response.start", "status": 200}) + await send({"type": "http.response.body", "body": b"[]"}) + + with ( + patch( + "kfs_sandbox.worker.importlib.import_module", + return_value=SimpleNamespace(app=swallowed_error), + ), + self.assertRaisesRegex(Rejected, "worker_resource_limit"), + ): + await execute( + root, + { + "settings": asdict(Settings()), + "app_target": "fixture:app", + "scope": { + "headers": [ + ( + "content-type", + "multipart/form-data; boundary=fixture", + ) + ], + "query_string": "", + }, + }, + ) + self.assertFalse((root / "response.json").exists()) + + def test_main_writes_safe_failure_classes(self): + for error, expected in ( + (Rejected("depth"), 422), + (Rejected("response_bytes"), 413), + (Rejected("worker_resource_limit"), 413), + (MemoryError(), 413), + (ValueError("secret"), 502), + ): + with ( + self.subTest(error=type(error).__name__), + tempfile.TemporaryDirectory() as directory, + ): + path = Path(directory) + (path / "request.json").write_text( + json.dumps({"settings": asdict(Settings())}) + ) + with ( + patch("kfs_sandbox.worker.sys.argv", ["worker", directory]), + patch("kfs_sandbox.worker.apply_limits"), + patch("kfs_sandbox.worker.asyncio.run", side_effect=error), + patch("kfs_sandbox.worker.execute", new=lambda *_: None), + patch.dict("kfs_sandbox.worker.os.environ", {}, clear=False), + ): + main() + result = json.loads((path / "failure.json").read_text()) + self.assertEqual(result["status"], expected) + self.assertNotIn("secret", json.dumps(result)) + + +if __name__ == "__main__": + unittest.main()