opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-08 02:43:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(knowledge-fs): bound document parsing and isolate resource-heavy work

Browse Source
This commit is contained in:
Jyong 2026-09-05 23:40:21 -04:00
parent ad43bf7eda
commit 24db4f5af9
157 changed files with 14546 additions and 889 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/knowledge-fs-ci.yml vendored
View File

@ -106,6 +106,7 @@ jobs:
- 'docker/envs/core-services/knowledge-fs.env.example'
- 'docker/envs/core-services/knowledge-fs-unstructured-service.defaults'
- 'docker/envs/core-services/knowledge-fs-unstructured.env.example'
- 'docker/knowledge-fs-unstructured-sandbox.compose.yaml'
- 'docker/generate_docker_compose'
- 'docs/design/knowledge-fs*'
- '.github/dependabot.yml'
@ -195,6 +196,12 @@ jobs:
- name: Install KnowledgeFS dependencies
run: pnpm install --frozen-lockfile
- name: Install PDF metadata inspector
run: |
sudo apt-get update
sudo apt-get install --yes --no-install-recommends poppler-utils
pdfinfo -v
- name: Scan KnowledgeFS secrets
run: pnpm security:secrets
@ -227,6 +234,12 @@ jobs:
working-directory: .
run: uv sync --project api --locked --dev
- name: Test dedicated parser sandbox
working-directory: .
env:
PYTHONPATH: knowledge-fs/services/unstructured-sandbox
run: uv run --project api python -m unittest discover -s knowledge-fs/services/unstructured-sandbox/tests -t knowledge-fs/services/unstructured-sandbox -v
- name: Collect Dify KnowledgeFS gate targets
working-directory: .
run: |

4
api/knowledge-fs-contract.lock.json
View File

@ -1,7 +1,7 @@
{
"schemaVersion": 5,
"subtreeTree": "d8b34a54bfe2744c76c84fc04ddc137d98a886d5",
"openapiSha256": "eba6f0e32eb27fd68ac20021c46b5f647217005fdb85522e7e4de6f0a1afc9a8",
"subtreeTree": "b7565ce4a39cd86563d41848bfb4d0672ca0a67b",
"openapiSha256": "f8cd6ad1e8ca2e62ceea2fc8a6f80f241135e23a2064d3a5f4ab45e25baf00b1",
"capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
"productOperationManifestSha256": "0b472db40cfc89ca16127db9a334891bb291b4f07e77128e5ec7521ae581a3a7",

8
docker/README.md
View File

@ -69,7 +69,13 @@ defaults.
The optional `knowledge-fs-unstructured` profile starts one isolated parser service named
`knowledge_fs_unstructured` for every KnowledgeFS remote format. Its tracked
`knowledge-fs-unstructured-service.defaults` additionally enables bounded page parallelism for
PDFs; an optional copied `knowledge-fs-unstructured.env` can override those service values. The
PDFs and a 25-million-pixel pre-allocation limit per page at the pinned 350 DPI; an optional copied
`knowledge-fs-unstructured.env` can override those service values. Do not raise the DPI or pixel
limit, or disable the guard: the KnowledgeFS client estimates page rasters at 350 DPI and rejects
oversized or unverifiable page geometry before sending it to the parser. Oversized pages need
smaller page dimensions or tiling before import; reducing compressed file size alone does not
help. Existing installations need the new KnowledgeFS image and a recreated parser service to
activate both guards. The
existing `unstructured` profile remains unchanged for Dify's legacy ETL, so KnowledgeFS tuning
cannot alter its PDF or Office parsing behavior. The copied
`knowledge-fs.env` pairs every PDF and structurally/byte-heavy remote document with the longer

4
docker/envs/core-services/knowledge-fs-unstructured-service.defaults
View File

@ -2,6 +2,10 @@
# PDFs; the same service also handles standard and heavy Office, mail, EPUB, ODT, and RTF inputs.
# Operator values in knowledge-fs-unstructured.env are loaded afterwards and take precedence.
# The non-.env suffix is intentional so this required file remains tracked in clean checkouts.
# Keep the pinned image's 350 DPI output and align its pre-allocation guard with the client.
# Raising DPI above 350 invalidates the client's raster-size estimate; do not disable the guard.
PDF_RENDER_DPI=350
PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000
UNSTRUCTURED_PARALLEL_MODE_ENABLED=true
UNSTRUCTURED_PARALLEL_MODE_URL=http://127.0.0.1:8000/general/v0/general
UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE=6

6
docker/envs/core-services/knowledge-fs-unstructured.env.example
View File

@ -8,7 +8,13 @@
#
# KnowledgeFS owns durable attempt retries. Keep the child retry loop disabled so one parser
# failure cannot multiply retries at both layers.
# PDF geometry, not compressed bytes, determines raster memory. The client preflights at 350 DPI;
# do not raise PDF_RENDER_DPI above 350 or raise/disable the 25-million-pixel per-page guard.
# These are provider process settings: this pinned HTTP API does not accept a pdf_image_dpi
# request parameter. Oversized pages must be resized or tiled before import.
# ------------------------------------------------------------------
PDF_RENDER_DPI=350
PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000
UNSTRUCTURED_PARALLEL_MODE_ENABLED=true
UNSTRUCTURED_PARALLEL_MODE_URL=http://127.0.0.1:8000/general/v0/general
UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE=6

3
docker/envs/core-services/knowledge-fs.env.example
View File

@ -104,6 +104,9 @@ KNOWLEDGE_QUERY_IMAGE_EXPANSION_TIMEOUT_MS=8000
# The bundled profile uses this internal service name. Override it with an external
# Unstructured-compatible endpoint when the deployment owns that provider instead.
UNSTRUCTURED_API_URL=http://knowledge_fs_unstructured:8000
# Change with parser image/model/output-policy upgrades to invalidate raw parse checkpoints.
# Leave empty for an unknown/custom backend; do not reuse a revision across semantic changes.
UNSTRUCTURED_BACKEND_REVISION=
UNSTRUCTURED_API_KEY=
# This is the process-wide ceiling shared by all remote parser requests. The nested heavy limit
# covers every PDF plus structurally/byte-heavy Office, email, EPUB, ODT, and RTF request.

35
docker/knowledge-fs-unstructured-sandbox.compose.yaml Normal file
View File

@ -0,0 +1,35 @@
# Opt-in only. Run the pinned-image golden gate before applying to production.
# Merge with docker-compose.yaml; this replaces the EXISTING dedicated service.
services:
knowledge_fs_unstructured:
build:
context: ../knowledge-fs/services/unstructured-sandbox
dockerfile: Dockerfile
image: knowledge-fs-unstructured-sandbox:local
init: true
read_only: true
pids_limit: 192
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
tmpfs:
- /tmp:rw,nosuid,nodev,size=1073741824,mode=1777
# Override the original mount by target, keeping parser outputs off host storage.
volumes:
- type: bind
source: ./volumes/knowledge-fs-unstructured
target: /app/data
read_only: true
environment:
UNSTRUCTURED_PARALLEL_MODE_ENABLED: "false"
healthcheck:
test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthcheck', timeout=3).read(4096)"]
interval: 30s
timeout: 5s
retries: 3
deploy:
resources:
limits:
cpus: "4.0"
memory: 6G
pids: 192

18
knowledge-fs/.harness/changes/2026-09-05-incremental-structured-admission.md Normal file
View File

@ -0,0 +1,18 @@
# Incremental structured-text admission
## Changes and rationale
- JSONL scans one line slice at a time instead of allocating a document-wide `split()` array. Its existing record bound now stops the scan before the tail is materialized; scalar/null/array records, exact numeric lexemes, blank lines and CRLF behavior are preserved.
- Native XML receives a SAX-only structure admission pass before the authoritative `fast-xml-parser` object projection. Tag depth, element/attribute/text nodes and processing instructions are bounded without building a DOM, with 64 KiB cooperative scan chunks. Existing post-projection byte/node limits remain defense in depth. No new dependency or XML semantic projection change is introduced.
- CSV already used `csv-parse`'s `on_record` callback, kept no duplicate parser result array, and stopped when `maxRows` was exceeded. A regression verifies that this limit wins before an invalid later record; no unnecessary CSV rewrite was made.
## Verification
- New helper tests were introduced before implementation. Integration verifies over-deep XML never invokes `XMLParser.parse`; line/row guards preserve values and prevent later invalid records being decoded. Comments, CDATA, namespaces, self-closing elements and chunk-boundary text are covered.
- Full parser package before the final indentation regression: 520 passed (including contemporaneous sandbox protocol tests). Coverage: 97.13% statements/lines, 91.48% branches, 97.97% functions; new SAX/line helper is 100% lines/functions and 92.3% branches.
- The final XML regression verifies indentation-only whitespace does not consume semantic nodes, matching the authoritative parser's trimming behavior. RED→GREEN confirmed; the focused structural-admission suite passed all 16 tests afterward.
- Parser typecheck and Biome formatting passed.
## Limits
Input bytes are still loaded through the existing explicit upload/parser byte caps; this change is bounded structural/record admission, not a claim of arbitrary-size file streaming. Process isolation provides external interruption of synchronous library work. SAX admission is not a substitute for the authoritative projection or the post-projection expansion limits.

23
knowledge-fs/.harness/changes/2026-09-05-isolated-parser-boundary-review.md Normal file
View File

@ -0,0 +1,23 @@
# Independent isolated parser lifecycle review
Security, resource accounting and failure handling were reviewed separately from the initial process-isolation implementation.
## Fixes
- Shared executor validates supplied byte counts as finite, non-negative safe integers before reserving input memory; `NaN` or negative counts cannot corrupt admission accounting.
- A worker message followed by nonzero exit or signal termination is rejected. Receiving a result is not proof that the isolated operation completed cleanly.
- Cancellation preserves all legal abort reasons, including falsy `null`, `false` and `0`, instead of confusing them with absence of failure.
- Native IPC responses validate the success artifact against the existing core contract and validate error packet shape/bounds, producing a typed terminal response failure on malformed output.
- The internal native-isolation wrapper accepts only serializable options; function-valued callback options fail up front rather than throwing during IPC. Standalone native parser factories retain their original callback API.
## Verification
- Fourteen new regression tests were written and failed before these fixes (one existing Infinity-over-limit path already rejected correctly).
- New boundary tests plus existing real-child and lifecycle tests: 28 passed.
- API app TypeScript check passed. Biome check/format passed for the four touched implementation/test files.
## Reviewed trade-offs
- Response contract validation happens after the child has enforced its 32 MiB serialized output limit; this costs a bounded parent-side validation/copy, accepted for boundary correctness.
- A 256 MiB V8 heap limit is not an OS RSS sandbox. Child process termination, input/output byte caps, and image pixel limits are distinct protections and must be described as such. No claim of a hard per-child native-memory ceiling is made.
- Worker queue slots are still held until actual child close. No early abort release or remote retry behavior was introduced.

27
knowledge-fs/.harness/changes/2026-09-05-multimodal-budget-completeness.md Normal file
View File

@ -0,0 +1,27 @@
# Multimodal compilation budgets, capability parity, and isolated image analysis
## Changes and rationale
- Capture an immutable `document-media-v1` execution plan. Explicit text-only capabilities now disable extraction, rasterization, and visual indexing while preserving original element/source references. Both synchronous upload/source compilation and durable jobs use the same parser-hint builder. The synchronous path resolves two tenant/space-scoped profile heads concurrently and freezes their capability snapshots before parsing; durable jobs retain attempt-frozen profiles. The synchronous PDF path now preserves the same external-raster/provider-fallback behavior as durable compilation.
- Remote media fetches are sequential and document-bounded: 100 unique URL attempts, 32 MiB aggregate delivered/reserved network bytes, 60-second absolute deadline, 10 MiB per image. Failed/null attempts count; unknown consumed bytes remain conservatively reserved. Identical URL fetches are deduplicated without collapsing element ids, captions, or source-URI hashes. Parent cancellation propagates; transient transport/storage failures preserve existing retry semantics instead of silently completing a document.
- All newly materialized original and generated image bytes share a 64 MiB document budget. Image decoding reserves at most 20 MP per image and 100 MP per document (unknown headers reserve the per-image ceiling); variant generation also has a 60-second document deadline. Optional media omitted by a limit or terminal decoder failure leave originals available and persist bounded `parseCoverage.media` reasons and `multimodalAssets` counters. Existing provider/archive completeness is preserved, never promoted to complete without evidence.
- Allowlisted local images require both lexical and canonical (`realpath`) containment. Roots resolve once per document. Opens use `O_NOFOLLOW`, reads are bounded through the same file descriptor, and size changes during reading are rejected. The default empty allowlist still prohibits local reads.
- Production image variants run in a one-request Node child: two concurrent children, eight queued requests, 64 MiB reserved input, 50 MiB per input (matching the existing local-file ceiling), 128 MiB V8 heap, 384 MiB sampled Linux RSS guard, and 30-second worker deadline. Parent cancellation/resource termination releases admission only after process close through the shared executor. IPC responses validate shape, names, image bytes, dimensions, two-variant maximum, and 16 MiB aggregate output. Worker execution diagnostics (input/output bytes, elapsed time, peak RSS) persist on generated variant references.
- The default preview remains 320 px; a separate bounded 2048 px `analysis` variant supplies vision embedding, enrichment/OCR, and answer models. Legacy objects without an analysis variant fall back to the original, not the preview. New assets rejected by count/byte/pixel/decode/deadline limits carry a bounded `analysisUnavailable.reason` marker: all image-model consumers skip them without dropping original download/preview references or text indexing. The optional marker survives manifest, candidate, node, and descriptor projection, and enrichment cannot clear it or claim successful visual embedding. Overlong skipped inline data URIs remain recoverable in the parse artifact but become a hash and parse-element reference in manifests, preventing a secondary URI-schema failure or duplicate inline payload. Explicit preferred-variant overrides are preserved. Analysis is generated first; the thumbnail is derived from that bounded bitmap. No extra environment variables or database migration are required.
- Malformed image-worker IPC and unknown worker/dependency failures remain operational errors; only known input/pixel/output-limit rejections become partial media. This prevents a broken deployment from silently publishing documents with omitted images.
## Verification
- TDD reproduced missing failed-attempt limits, duplicate downloads, aggregate-byte bypass, hung fetches, ignored cancellation, local symlink escape, ignored profile flags, missing synchronous PDF fallback, thumbnail-as-model-input, derived-byte budget bypass, missing worker execution telemetry, and malformed IPC acceptance before their corresponding fixes.
- Targeted API verification: 115 tests passed across media budgets/extraction, immutable plans, image variants, synchronous/durable compilation, and answer-provider tests. Four focused behavior modules reached 99.78% lines/statements, 92.08% branches, and 100% functions.
- Targeted API-app verification: 62 tests passed across actual child-process decoding, IPC validation, multimodal configuration, embedding/enrichment/answer wiring. Two final validation guard regressions were separately rerun after implementation.
- API and API-app TypeScript checks passed. Repository-wide check/build/lint and Docker bundle verification are owned by the parent completion pass; no production deploy or commit was performed here.
- Final safety follow-up: TDD reproduced rejected-original model fallback, marker loss across manifest/candidate projection, malformed IPC misclassification, and overlong skipped inline-URI manifest failure. The final focused API pass includes 113 tests across 12 files (99.06% lines, 92.7% branches, 100% functions for extractor/manifest/candidate modules); image protocol, actual child, and enrichment checks include 46 passing app tests. Existing unmarked artifacts remain compatible.
## Risks and scope
- A sampled RSS guard is not a kernel per-process memory cgroup; whole-container limits remain the final memory boundary. Image inputs retain byte/pixel guards before native decoding. PDF crop/thumbnail work on bounded Poppler-generated PNGs remains in the existing rasterizer, not the untrusted-codec child.
- One child per image adds startup overhead (the local two-process decoding regression took roughly 0.3 seconds). It avoids decoder state retention; do not enable an unbounded worker pool to reduce this overhead. Real representative throughput/golden-set evaluation remains part of the parent evaluation pass.
- Custom isolated preview variant names must be 1–64 ASCII letters, digits, hyphens, or underscores and are rejected at startup otherwise. Native generator-only callers retain their existing API. Analysis dimensions are bounded to 4096; normal defaults remain 320/2048.
- Preserved references and explicit partial status do not make unsupported SVG/EMF/WMF/SmartArt resources newly renderable. Archive diagnostics report those separately. No claim is made that an unknown provider text/table extraction is complete.
- Legacy synchronous ingestion freezes only the media capability decision here; its later embedding resolver can still observe a changed manifest. Atomic model/profile publication and retry fencing remain guarantees of the durable path, not the legacy synchronous compiler. The upload HTTP abort signal now reaches parsing/materialization; legacy source materializer calls have no request signal in their existing interface.

55
knowledge-fs/.harness/changes/2026-09-05-native-markup-content-fidelity.md Normal file
View File

@ -0,0 +1,55 @@
# Preserve native Markdown and HTML content while bounding traversal
## Problem
Markdown blockquotes and ordinary HTML blocks were omitted. A paragraph beginning
with an image lost its remaining text, while other mixed paragraphs duplicated
the image syntax and placed all images before the paragraph. HTML traversal
ignored bare container text, returned before finding images in paragraphs/lists,
and selected only the first figure image. Deep HTML could overflow recursive
traversal before a useful input error was returned.
## Changes
- Visit Markdown blockquotes in document order, sharing section context. Keep
ordinary paragraph/list/table projections; mixed Markdown image/HTML blocks use
an inert HTML tree to preserve surrounding text and image placement.
- Keep static HTML text for both Markdown and MDX. No JSX evaluation, script
execution or URL fetch occurs in this parser. Exclude script/style/noscript
subtrees from text and image discovery, including nested table/heading paths.
- Preserve bare HTML text and ordered inline runs, paragraph/list images, all
figure images and captions, plus images associated with headings/tables.
Tables retain the existing normalized matrix element followed by their image
assets; this is not new cell-level image-anchor support.
- Preserve per-placement captions/titles when the same Markdown URI is repeated.
- Apply the shared maximum DOM depth (128) and node count (250,000) before
projection; classify violations as non-retryable `ParserResourceLimitError`.
Budget inspection, title search, image search and text collection use explicit
stacks. Block projection runs only after that bounded-depth inspection.
- Update the existing regression that deliberately expected ordinary Markdown
HTML content to be discarded. The parent change owns final parser version and
fingerprint updates.
## Verification
- TDD: all initial ten regression tests failed before implementation and passed
after it. Additional tests first exposed and then verified heading/table image
extraction, repeated-URI placement metadata and excluded-subtree image handling.
- Fifteen new tests pass, including 5,000 nested divs and a shallow 250,001-node
document; no network/provider was used. Fourteen selected existing markup/image
regressions also pass. Parser package typecheck passes.
- A broader run of the existing parser file plus the first ten new tests had
111 passes and one concurrent structured-XML expectation failure (score now
remains a string); the parent structured-data fix owns that expectation.
- The parent task owns full package coverage/build/lint and contract-lock
regeneration. Shared index formatting was deliberately left for the parent to
avoid colliding with concurrent structured/provider changes. The dedicated test
file is formatted. No commit, deployment or production mutation was performed.
## Remaining boundaries
The DOM budget is enforced after htmlparser2 constructs the DOM, before our
traversal. It prevents projection/stack blowups but does not claim a streaming,
pre-allocation DOM parser; input-byte admission still bounds the input. Marked
lexing precedes token traversal checks. A terminable CPU worker and streaming
parser-admission limits remain part of the broader resource-isolation work.

40
knowledge-fs/.harness/changes/2026-09-05-office-admission-regression-fixtures.md Normal file
View File

@ -0,0 +1,40 @@
# Office admission regression fixtures and compatibility boundary
## Changes
- Replace successful remote DOCX/XLSX transport test inputs that were arbitrary
bytes or fabricated ZIP headers with small actual deflated OOXML packages.
The API app uses a fixed, fflate-generated DOCX ZIP encoded as base64 to avoid
adding a production or test dependency solely for a fixture.
- Update default parser-version expectations and the Markdown artifact digest
golden for the explicitly bumped versions. Custom parser versions and
coordinator fingerprints are not changed.
- Separate successful archive-image extraction from unsafe archive admission.
DOCX/PPTX/XLSX/ODT/EPUB archives with `../` member paths must fail with a
terminal input error before the provider is called.
- Preserve malformed ZIP/XML inputs as rejection regressions. The prior
expectation that provider text remains usable for damaged spreadsheet archives
is intentionally superseded by preflight admission.
- Keep the original unsafe worksheet metadata fixture in an explicit rejection
case. A second case uses valid worksheet declarations and coordinates while
retaining malformed/external optional image relationships and verifies the
original fail-soft image anchoring and fallback behavior.
## Compatibility boundary
Damaged archives/XML, unsafe member paths, or missing/external worksheet
relationships no longer enter remote Office parsing. These affect safe resource
inspection and are not treated as optional image metadata. Optional image/drawing
relationship failures still leave admitted document text usable; no file is
rewritten before being sent to the provider.
## Verification
The first integrated run exposed 22 failures from obsolete version assertions,
fake archives, and the explicitly tightened admission policy. Both relevant files
now pass all 124 tests. An existing deadline mock hung only in the full suite; it
now retains its Request objects for the lifetime of the simulated transport and
checks an already-aborted signal, matching real fetch behavior. No timeout was
extended, assertion removed, or production deadline logic changed for this test.
Both parser and API-app typechecks pass. Scoped Biome and git whitespace checks
pass. The parent task owns final broader-suite validation and integration notes.

37
knowledge-fs/.harness/changes/2026-09-05-parser-format-contracts-and-coverage.md Normal file
View File

@ -0,0 +1,37 @@
# Parser format contracts, encoding, provenance and coverage
## What changed
- A shared document format registry now drives native/remote routing, upload MIME aliases and source filename MIME inference. The existing 25 upload extensions are unchanged; internal YAML/NDJSON support is not added to uploads. Inherited object keys such as `constructor` and `__proto__` cannot become formats or crash admission.
- Native text decoding is strict UTF-8, or UTF-16LE/BE with a BOM. Malformed sequences, unsupported UTF-32 BOMs and unmarked binary NUL text fail with terminal `provider_input` rather than indexing replacement characters.
- Properties documents have a dedicated semantic projection: comments omitted, continued lines joined, separators and escapes decoded, Unicode surrogate pairs validated, and original source line/key/value retained. This follows Java's character-reader properties grammar; there is deliberately no heuristic legacy code-page detection.
- WebVTT produces cue text with identifier, start/end milliseconds, settings and raw cue payload; header/NOTE/STYLE/REGION blocks are not indexed as speech. Malformed cues fail explicitly instead of silently losing content.
- Native structured syntax failures, row limits and element limits retain terminal input classification. Oversized native markup no longer bypasses native admission by silently being sent to the remote service; explicit OCR/layout/language routes remain available.
- Unstructured accepts a deployment-owned `backendRevision` (bounded to 256 characters). It participates in checkpoint fingerprint, artifact hash and metadata. Client semantic versions advance to native Markdown/MDX 4, HTML 5, structured 4, Unstructured 12. An external provider without a configured revision is explicitly `external-unversioned`, not falsely attested as the pinned image.
- `requiresImages` remains tri-state in fingerprints/hashes: legacy-auto (`undefined`) and explicit text-only (`false`) cannot coalesce or reuse one another's archive-media output. A concurrent parse regression verifies distinct artifacts, not just distinct policy strings.
- Archive fallback honors `requiresImages=false`. Unsupported media (including SVG), count/byte omissions, Office chart/diagram visuals and inspection failures produce bounded `archiveMediaReport` references and `parseCoverage.media` reasons. Reports preserve up to 4,096 resource paths (1,024 characters each), explicitly reporting truncation. Unknown provider text/table/media completeness remains `unknown`; it is never guessed complete.
- Artifact-level coverage/provenance metadata is included in structural output admission, not just element arrays.
## Why
Prevent silent encoding/content corruption, format-policy drift, generic misleading compilation failures, cache reuse across provider semantic upgrades, and silent partial multimodal extraction. Keep byte/pixel/normalization protections from the first hardening phase unchanged.
## Verification
- TDD reproductions failed first for UTF-16, malformed UTF-8, properties, VTT, backend revision hashes, native overflow routing, unsupported/archive image gating, omitted charts/images, missing source MIME aliases, and prototype-key filename admission.
- `pnpm --filter @knowledge/parsers test:coverage`: 489 passed; statements/lines 97.04%, branches 91.34%, functions 97.91%.
- `pnpm --filter @knowledge/parsers typecheck`: passed.
- Correctly scoped upload/source tests: 80 passed. Initial root-directory Vitest invocation omitted the API package's Dify object-storage setup and consequently failed two source imports. The correctly scoped API command loads `packages/api/vitest.config.ts`; this was a test-invocation issue, not a compilation regression. Full API verification belongs to the integration pass.
- Biome formatting/check applied only to owned modified files; full workspace verification belongs to the integration pass.
## Scope and risks
- No live provider, production deploy, migration, commit or push was performed by this slice. Pinned-provider real-file golden execution requires an available Docker/service runtime.
- Parse coverage is a truthful report, not support for rasterizing previously unsupported SVG/EMF/WMF/Office charts. Original document bytes and archive-path provenance remain available; omitted visual content is explicitly partial.
- Existing raw checkpoints correctly invalidate on the new semantic versions. Transport/admission-only limit changes are not misrepresented as provider content revisions.
- Strict malformed text/VTT handling intentionally rejects content that previously appeared to import successfully while indexing corrupt/control text. Normal UTF-8 and existing native format content stay covered by the parser regression suite.
## Grammar references
- [Java Properties character-reader grammar](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/Properties.html#load(java.io.Reader))
- [WebVTT specification](https://www.w3.org/TR/webvtt1/)

48
knowledge-fs/.harness/changes/2026-09-05-parser-hardening-phase-one.md Normal file
View File

@ -0,0 +1,48 @@
# Parser hardening — first implementation phase
## Scope and intent
Implement the first phase of the 2026-09-05 parser audit: bound structural amplification, preserve native document content, and prevent incompatible size-based routing. This is not a claim that every parser or every resource-exhaustion vector is now sandboxed. Existing PDF safeguards and the MediaBox consistency follow-up remain in their dedicated change notes.
## Changes
- Office/ODF/EPUB admission streams actual ZIP member expansion before contacting Unstructured, within the existing shared single-flight/admission lifetime. It checks local/central entry agreement, duplicate/unsafe paths, XML structure, worksheet coordinates, shared strings, and logical worksheet references. A repeated worksheet relationship consumes its repeated dense-cell cost. Input bytes and provider partition options are unchanged for admitted files.
- Inspection has explicit defaults: 4,096 members, 512 MiB total actual expansion, 64 MiB total XML / 16 MiB per XML part, depth 128, 1 million XML nodes, 256 logical worksheets, 100,000 rows / 16,384 columns, 250,000 dense cells per sheet / 500,000 per workbook, 500,000 actual cells, 200,000 shared strings, 30-second inspection deadline. Column formatting ranges do not inflate occupied cell span. Standard EPUB 2 XHTML PUBLIC identifiers are allowed without loading DTDs; internal subsets and arbitrary declarations are not.
- Shared native budgets bound decoded structure (250,000 nodes / depth 128), projected output (32 MiB), table width (4,096) and cells (500,000). HTML expansion and table text bytes share a parse-local document-wide counter, including separate Markdown HTML blocks and separate provider table elements; independent parses do not share counters. Final/provider metadata uses a separate 1 million-node budget to accommodate the existing 20,000-element limit and ordinary coordinate metadata. Limits fail explicitly; they never silently truncate content.
- JSON/JSONL retain numeric source tokens, including large integers, precise decimals, exponent notation and negative zero. Node 22's native `JSON.parse` reviver source preserves own `__proto__` fields safely. No new runtime dependency is added. JSONL scalar/null/array records retain their values and source order.
- CSV uses array decoding and own-property record construction; duplicate headers receive collision-free stable names in amortized linear naming work, special property names are preserved, header-only documents retain headers, and absent object fields never read inherited properties.
- Sparse heterogeneous records stay sparse instead of expanding into the union-key Cartesian product. Ordinary homogeneous record tables retain their representation. Table text is byte-budgeted before joins, including header-only output and repeated labels.
- XML attributes and lexical tag values are retained. Untyped XML numeric-looking text is deliberately represented as a string, preventing loss of leading-zero identifiers.
- Structured files stay on the native structured parser when configured. The API uses the same admitted input-byte limit for structured and remote paths (15 MiB default; existing override capped at 50 MiB), so 10–15 MiB JSON/CSV uploads no longer switch to an incompatible remote format.
- Native parsing checks an already-aborted request before decoding. Shared remote normalization uses the coordinator-owned signal, not the first caller's signal, preserving remaining consumers of identical work.
- Content/layout changes advance default parser identities: Markdown/MDX v3, HTML v4, structured v3, Unstructured v11. Existing indexes are not rewritten automatically; subsequent parse/re-index operations use the new policy identity.
The markup fidelity, CJK spatial-index/heading budget, raw provider-response guard, synchronous deadline detection, HTML span budgets, and PDF geometry work are detailed in adjacent change notes.
## Compatibility and security boundaries
- Normal-file content and transport contracts are regression-tested, but newly imposed cost limits intentionally reject exceptionally large or ambiguous structures even when the file format itself permits them. Error classification is terminal input failure before remote work; no retry storm and no remote-acceptance ambiguity is introduced for admission failures.
- Previously permissive test fixtures containing path traversal, malformed ZIP/XML, invalid coordinates or unsafe worksheet relationships are now explicit rejection cases. Safe media-anchor failures remain fail-soft inside a valid admitted archive.
- Standard EPUB declaration support was checked against upstream Unstructured/Pandoc source and archive-based tests; this is not a pinned-Pandoc production round trip.
- Native numeric lexeme support was verified directly on Node **22.0.0**, the minimum major version used by the deployment images, as well as the development runtime.
## Verification
Tests were added and observed failing before behavioral fixes. Focused tests cover ordinary and adversarial inputs, source-byte preservation, special JSON/CSV property keys, cancellation/single-flight behavior, size-routing boundaries, and real Poppler page/crop geometry.
- Final parser suite: **442/442**, coverage **96.81% statements/lines, 91.09% branches, 97.8% functions**; no coverage threshold was lowered.
- Node **22.0.0**: 50 JSON/structured integration tests pass on the deployed major's initial release, not just the newer development runtime.
- API package regression suite: **5,061 passed / 3 existing skipped**, 426 passed test files / 1 existing skipped file. Focused real Poppler rendering/crop tests: **60/60**. API parser configuration/preflight tests: **55/55**.
- Full **`pnpm check` passes**, including all workspace tests, configured CI coverage gates, retrieval/phase-4 regressions, OpenAPI export, migration artifact checks and Compose/static smoke checks. The repository's existing CI coverage command excludes the API package; its complete test suite and focused PDF tests still run as recorded above.
- Full workspace typecheck and **`pnpm build` pass**; backend-scoped Biome passes. `git diff --check` passes. Contract pin/export validation uses an isolated temporary Git index so the user's staging area is not changed.
- Full `pnpm lint` was attempted and remains blocked by pre-existing, untouched Admin/test/generated OpenAPI formatting/file-size findings. Those unrelated files were not reformatted as part of parser work.
- Local diagnostic: the audit's 4,881-byte, 300-record sparse JSON now produces **3,979 bytes** of text (about 4 ms in one bounded run), instead of the audited 1,137,189-character dense projection. This is a regression probe, not a production benchmark.
- Production provider/worker redeployment and live ingestion E2E were not run. Real-pinned-provider corpus benchmarks and server verification remain required before calling the entire optimization roadmap complete.
## Remaining roadmap (not completed by this phase)
- Nested EML/MSG attachments do not yet share recursive admission/depth/budget accounting; attachment behavior is not disabled as a shortcut.
- Legacy DOC/PPT/XLS conversion and native library image decoding still require provider-side process isolation, hard memory/CPU limits and killable work. Format preflight cannot prove arbitrary binary parser safety.
- Native JSON/HTML/XML and Markdown tokenization still allocate in the API process within byte/structure bounds. The new deadline check detects synchronous overruns; it cannot preempt JavaScript execution. XML's structural check is post-decoding; move recursive native work to bounded workers in a later phase.
- Unified encoding policy, complete/partial visual-enrichment reporting, aggregate remote-image budgets, provider-version fingerprinting, cross-process resource admission and production corpus benchmarks remain later phases.
- No database migration, model configuration, production deployment, commit or push is performed by this implementation turn.

145
knowledge-fs/.harness/changes/2026-09-05-parser-hardening-remaining-phases.md Normal file
View File

@ -0,0 +1,145 @@
# Parser hardening: remaining phases
## Scope and sequence
Follow-up to `2026-09-05-parser-hardening-phase-one.md` and the all-format parser audit.
Existing uncommitted phase-one changes are preserved. No production deployment or migration
is part of this implementation turn.
1. Execution contracts: shared format registry, strict text decoding, properties/VTT semantics;
bounded cancellable native child processes; opt-in dedicated Unstructured request isolation
and nested mail/archive admission.
2. Completeness/reuse: immutable media execution plan, sync/durable capability parity,
aggregate remote media budgets, explicit skipped/partial coverage, backend semantic revision.
3. Evaluation: real native golden/resource benchmark, separate preview/analysis variants,
dedicated provider image golden gate and resource/cancellation tests.
## Native execution boundary
The Node API wraps native parsers in one shared admission controller (2 active processes,
32 queued requests, 128 MiB total reserved input, 50 MiB absolute input ceiling further
restricted by the existing API input setting). Each request receives a fresh process with a
256 MiB V8 old-generation ceiling and a 600-second wall deadline including admission.
Linux additionally samples resident memory every 50 ms and terminates the child above
512 MiB (this sampled check is not a hard RSS quota). The same lifecycle executor isolates
Sharp image variant work with separate 128 MiB V8 / 384 MiB sampled RSS limits, a 30-second
task deadline, 2 active / 8 queued tasks and 64 MiB reserved input. Each document also has
an aggregate image/variant work budget; independent task limits cannot reset that budget.
Cancellation/timeout sends SIGKILL; admission is released only on process close. The process
receives no provider credentials or inherited NODE_OPTIONS. Serialized output is capped at
32 MiB; peak RSS, parser elapsed time and input/output bytes are added to artifact metadata,
not semantic artifact identity. This is process isolation plus bounded allocation, **not** a
claim that V8 heap size is a hard OS RSS limit or that Node processes form a security sandbox.
Production builds include a compiled `native-parser-worker.mjs` (no production tsx). The image
bundle smoke now executes a native parsing request in the child, in addition to Sharp/Poppler.
It also executes the compiled image-variant worker. Worker responses are validated at the
parent boundary. Unexplained process exits and malformed IPC are operational response
failures, not falsely labelled malformed user documents. Known format rejection remains
an input failure; cancellation retains the original reason, including falsy reasons.
Unsafe or budget-rejected original images are retained with a bounded optional
`analysisUnavailable` marker. This marker survives manifest/node/candidate projection and
prevents embedding, enrichment and answer providers from bypassing rejection via an original
image fallback. Legacy artifacts without this optional field retain their previous behavior.
## Verification
- RED: missing isolated adapter; API native route had no process metadata; backend revision
did not affect API policy fingerprint; image bundle did not include/probe the worker.
- Native process regression tests: real parser content/hash equivalence, real CPU-bound child
killed on abort, shared slot lifetime, queue/byte admission, deadline, crash and typed errors.
- API parser options tests include configured backend semantic identity and >10 MiB JSON.
- Final native lifecycle/protocol/memory focused gate: 43 tests, 100% statements/lines/functions
and 96.35% branches across the four implementation modules. Individual module branches
are also at least 90%; no threshold was relaxed.
- Real native golden benchmark: 10 format families × 2 bounded fixture sizes × 3 repetitions;
all required Unicode markers preserved. Measures p50/p95, child peak RSS and output amplification.
Markers are **not** a proxy for OCR accuracy or retrieval recall. Development tsx startup is
included in latency; do not compare these values to production compiled worker latency.
## Release gates / environment limits
Local Docker CLI exists but the daemon socket is unavailable. Actual pinned Unstructured
image compatibility, legacy binary golden fixtures and multi-replica production capacity are
not validated by local mocked/unit tests. The new provider sandbox must remain opt-in until
its real-image contract gate passes. Do not mark this deployment/quality gate complete merely
because source tests pass. Final verification and remaining external gates will be recorded below.
## Measured local evaluation
- Node 22.23.2 / macOS arm64 native adapter: 10 format families × 10/1,000 rows × 3 runs;
60 actual isolated parses, required Unicode markers all preserved. Across the 20 cases,
p50 140–175 ms, p95 142–197 ms and peak child RSS approximately 99–112 MiB. These are
development tsx-startup-inclusive measurements, not compiled production throughput or OCR
quality estimates.
- Both compiled workers were separately exercised on Node 22.0.0, including exact large
JSON integer preservation and real Sharp analysis/thumbnail output. The full API dependency
graph already contains undici 8.10.0, whose installed engine requires Node >=22.19.0;
full API compatibility is therefore verified on 22.23.2, not asserted for 22.0.0. No runtime
dependency was downgraded or updated during this compatibility check.
- Bounded synthetic PDF thumbnail comparison: repeated two-Poppler rendering measured
95–103 ms, versus 73–83 ms for a shared main bitmap plus Sharp thumbnail (296 ms cold
run). The output was **not pixel equivalent** (12.54 dB PSNR), and the derived thumbnail
was larger (7,152 vs 6,115 bytes). The production two-render policy remains unchanged;
broader OCR, non-Latin, chart and memory measurements are required before replacement.
## Final local gates
- Full `pnpm check` **passed** after final fixes, including workspace typechecking/tests,
configured CI coverage gates, retrieval and phase-4 evaluations, hermetic OpenAPI export,
migration artifacts, Compose configuration and static smoke/workflow contracts.
- Parser package: **521 tests passed**, 97.13% statements/lines, 91.50% branches,
97.97% functions. API package: **5,124 passed, 3 pre-existing skipped** (430 passed files,
one skipped). API application: **432 tests passed** (57 files). Existing CI aggregate
coverage excludes the API package; focused changed-module coverage is recorded separately,
without lowering thresholds or treating unrelated baseline coverage as newly verified.
- Media focused gate: 113 tests, 99.06% lines, 92.70% branches, 100% functions before final
boundary refinements. Final marker/manifest/enhancer regression gate: 25 passed, followed
by the complete API suite above. It covers count/byte-skipped large inline sources,
whitespace-prefixed data URIs, recovery provenance and truthful missing/unsupported states.
- Dedicated provider runtime: **80 tests**, including actual disposable process/descendant
cancellation, file-locked conversion budgets, sticky signal-death rejection, atomic
publication and real oxmsg fixture admission; 96.90% statements and 92.05% branches.
Ruff passes. Both opt-in Compose overlays were configuration-validated.
- `pnpm build`, production esbuild with both worker entry points, backend Biome and
`git diff --check` pass. Final compiled-worker smoke on Node 22.23.2 preserves a large
integer, returns a typed invalid-JSON error, and generates both actual Sharp variants.
- Full `pnpm lint` still reports the known unrelated Admin/test/generated-artifact
formatting and generated OpenAPI size baseline; backend-scoped lint is clean. Those
unrelated files were not reformatted to manufacture a green global result.
- OpenAPI was regenerated for the additive optional `analysisUnavailable` contract. Contract
lock generation/validation uses a temporary Git index, preserving the user's staging area.
## Delivery status and outstanding acceptance
The remaining implementation slices are delivered: shared format/native admission contracts,
bounded child execution, nested mail and converted-product admission, capability-aware media
plans, aggregate image work limits, explicit completeness/provenance, backend revision identity,
incremental JSONL/XML admission, preview/analysis separation and reproducible local benchmarks.
Details live in the adjacent format, media and provider change notes.
Not yet accepted: actual pinned Unstructured image build/HTTP golden corpus, read-only image
compatibility, real OCR/table/image recall and localization, production cost-weighted/multi-replica
capacity and a staging canary. Docker is unavailable locally, and no staging deployment was
authorized or performed. Hard per-request cgroup/storage quotas require deployment-level
delegation; process sampling is not equivalent. The legacy synchronous path still has its
documented non-atomic model-head decision; durable production jobs use frozen profiles.
The provider override replaces the existing dedicated service only, never a second per-format
service, and stays opt-in. Existing documents are not automatically re-indexed. No database
migration, production environment mutation, service restart, commit or push occurred.
## Submission follow-up
The subsequent user request authorizes committing and pushing this change. The remote branch
had advanced to `ad43bf7eda`, so the local parser change was replayed on that tip without
rewriting any published commit. Only the generated contract lock conflicted; it was regenerated
against the combined tree. Independent review confirmed that the remote namespace-preview
authentication, website-import recovery, scheduler diagnostics and CI changes remain intact.
After integration, full `pnpm check`, `pnpm build`, production esbuild, backend Biome and secret
scanning pass. API regression now has 5,135 passing tests and the same 3 existing skips; parser
and API-app totals remain 521 and 432. The unmodified provider process suite was not needlessly
rerun. Root `.turbo/`, `artifacts/` and `tmp/` are excluded from the commit. The real-image and
deployment acceptance gates above remain open; committing does not enable the sandbox override.

54
knowledge-fs/.harness/changes/2026-09-05-pdf-mediabox-budget-consistency.md Normal file
View File

@ -0,0 +1,54 @@
# Use the actual Poppler MediaBox for local raster admission
## Why
The local rasterizer estimated allocation from `pdfinfo`'s `Page size`, which is
the CropBox, while its existing `pdftoppm` command renders the MediaBox. A tiny
CropBox therefore did not bound a larger backing bitmap. A safe 144 x 144 point
MediaBox with a 72 x 72 CropBox reproduced this: with a 10,000-pixel budget the
old implementation rendered 20,736 pixels and only then failed in Sharp.
## Changes
- Read the requested page's resolved MediaBox endpoints, subtracting nonzero
origins, and reject missing, duplicate, incomplete, non-finite or reversed
geometry before starting `pdftoppm`.
- Preserve MediaBox rendering and the established displayed-page coordinate
contract; do not switch to `-cropbox`. Page rotation only swaps the dimensions
used for area/long-edge admission. Actual rendered dimensions still drive
provider pixel, displayed-point and relative crop mapping.
- Account for `pdfinfo` rounding both box endpoints to two decimal places: use
conservative dimension bounds and the worst-case aspect ratio. The second
real regression (100.004 x 100 points) otherwise passed the rounded estimate
and exceeded its allocation budget by one pixel column.
- Preserve the earlier page-pixel/long-edge caps, shared page bitmap/session,
cancellation, cleanup and provider-fallback semantics. Ordinary uncapped pages
retain their rendering parameters. A page exactly at a rounded budget boundary
can be downscaled slightly more conservatively (typically one pixel).
## Verification
- TDD red: five real small-PDF tests failed before the MediaBox fix, including
the original mismatch and four rotations. They passed after the fix.
- TDD red: the fractional 100.004-point regression then failed against the first
MediaBox implementation; conservative rounding fixed it.
- 70 focused rasterizer tests pass, including eight real Poppler tests, nine
rejected geometry cases, existing admission/cancellation/cleanup tests, and
content-colour checks for nonzero origins with 0/90/180/270-degree rotation.
Both relative and provider-style pixel/displayed-point crops preserve colour
and position after downscaling. No unsafe-size raster was generated.
- Focused production-file coverage: statements/lines 95.46%, branches 92.98%,
functions 98.61%. API package typecheck passes. Scoped Biome and diff checks
pass after formatting.
- The parent implementation task owns full-workspace build/lint/test and contract
lock regeneration. They were not repeated for this isolated parallel slice.
No commit, deployment, database migration or production request was performed.
## Remaining limits
This closes the CropBox/MediaBox allocation mismatch; it is not an arbitrary PDF
object/embedded-image memory sandbox. The API preflight and provider-native pixel
guard remain complementary. Crops retain the pre-existing displayed-page frame;
raw bottom-left PDF user-space coordinates without frame metadata are not newly
supported. Changes to provider page-frame conventions require explicit contract
tests rather than silently switching rendering boxes.

99
knowledge-fs/.harness/changes/2026-09-05-pdf-page-raster-safety.md Normal file
View File

@ -0,0 +1,99 @@
# Bound PDF page rasters before parser allocation
## Problem and evidence
A user-provided 247,902-byte, single-page PDF has a 2267.72 × 5102.36 point
MediaBox/CropBox (80 × 180 cm). File-byte and document-concurrency admission did
not bound its bitmap allocation. The pinned Unstructured image contains
unstructured 0.22.18 and unstructured-inference 1.6.11, defaults to 350 DPI, and
would render this page to 11024 × 24804 = 273,439,296 pixels. A single RGB buffer
alone is approximately 820 MB before image copies and inference allocations.
Its existing renderer ceiling was one billion pixels per page.
The pinned API does not expose `pdf_image_dpi` through its form/partition
whitelist, so sending a per-request DPI field cannot fix this issue.
## Changes
- The production Node parser injects a PDF preflight into the existing shared
admission and single-flight operation, before transport starts. A single
`pdfinfo` invocation checks all pages and their resolved size, MediaBox and
CropBox. It admits at most 10,000 pages, 25 million pixels per page, and 10,000
pixels per edge at 350 DPI. Unknown, incomplete or unsafe geometry fails closed.
- Inspection has a 10-second subprocess deadline, 4 MiB output limit, private
temporary directory and 0600 source file. Cancellation kills and joins the
child before cleanup. Non-PDF inputs do not incur filesystem/subprocess work.
- Dedicated parser defaults and local Compose retain 350 DPI and set
`PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000`. The pinned PDFium renderer enforces
this before bitmap allocation across hi_res, OCR/auto fallback and image-block
extraction. This also covers geometry differences between PDF engines.
- The pinned service exposes its raster guard as HTTP 500. An exact bounded
JSON-detail classifier turns only that known rejection into a non-retryable
provider input error before inline retries. Error inspection is limited to
4 KiB/one second; other HTTP failures retain their retry semantics.
- Local Poppler crop generation now applies both edge and pixel budgets before
rendering, including short-edge rounding. Coordinate mapping still uses the
actual scaled page dimensions. Ordinary pages retain their original rendering
parameters.
- Docker build/runtime checks verify `pdfinfo`; CI installs Poppler for actual
PDF geometry integration tests. Deployment documentation covers both guards.
## Compatibility and operations
This is resource admission, not an output transformation. Original files, parser
strategies, text extraction and multimodal behavior of admitted files are
unchanged. Existing safe raw checkpoints remain reusable; budget-only checks do
not change parser output fingerprints. Platform-neutral callers constructing the
low-level parser client must inject an appropriate `requestPreflight`; the Node
API does so by default, and the dedicated service has its own allocation guard.
Oversized pages are rejected rather than silently converted to text-only parsing.
To import the sample banner, export it with smaller physical page dimensions or
tile its canvas into smaller pages. Splitting only between existing giant pages
or compressing the file further does not reduce this risk. Automatic layout-
preserving tiling/normalization is follow-up work, not implemented here.
Existing deployments need the new API image and a recreated dedicated parser with
the updated defaults. Operator env files load afterwards: remove any conflicting
DPI/pixel-limit overrides. Kubernetes users must apply the same limits to their
external Unstructured service. No database migration is required.
Geometry limits do not bound every embedded bitmap decoder, pathological PDF
object graph or model allocation. Container memory limits and workload admission
remain necessary. Poppler metadata inspection is bounded in duration/output, not
an OS-level memory sandbox.
## Verification
- TDD: unsafe input previously reached fetch; new tests fail before preflight
integration and pass afterwards. Provider raster rejection previously retried
as HTTP 500; the regression now observes exactly one request and no retry.
- Actual supplied PDF through `createApiDocumentParser`: rejected as
`provider_input`, retryable=false, providerCalls=0, about 13 ms locally. No
request was sent to an external parser. Bounded local rendering at 1 MP
completed in about 314 ms and produced a 163,694-byte PNG.
- Preflight: 39 tests, including real Poppler inherited boxes and an oversized
non-first page; lines/statements 99%, branches 94.89%, functions 100%.
- Parser package: 182 tests; lines/statements 96.23%, branches 90.30%, functions
96.86%. Local rasterizer: 53 tests including scaled crop-coordinate fidelity.
- API full suite: 5,044 passed, 3 existing skips; API-app full suite: 349 passed.
Local rasterizer focused coverage: lines/statements 94.86%, branches 92.61%,
functions 98.61%. Workspace build/type checks (12 packages) and API production
bundle passed. Backend lint (1,122 files), workflow/secret tests (28), Compose
tests (14), image artifact tests (3), OpenAPI export tests (2), actual secret
scan and local/Dify Compose configuration validation all passed.
- Full Docker image builds/live unsafe-file execution are intentionally not used
for this diagnosis. No production configuration was changed. Read-only server
inspection confirmed the two PDF limits are not configured yet; current
OOMKilled=false with RestartCount=10 is not proof of the historical restart cause.
- The omnibus `pnpm check` (including Docker builds and live stack smoke tests)
and whole-workspace frontend lint were not run for this backend-only change;
the affected full package suites, coverage, backend lint, build and deployment
artifact gates above were run instead. Dify backend logic and database schema
are unchanged; only the KnowledgeFS subtree contract lock needs regeneration.
## Upstream evidence
- https://github.com/Unstructured-IO/unstructured/blob/0.22.18/unstructured/partition/utils/config.py
- https://github.com/Unstructured-IO/unstructured-inference/blob/v1.6.11/unstructured_inference/config.py
- https://github.com/Unstructured-IO/unstructured-inference/blob/v1.6.11/unstructured_inference/inference/pdf_image.py

57
knowledge-fs/.harness/changes/2026-09-05-unstructured-normalization-safety.md Normal file
View File

@ -0,0 +1,57 @@
# Bounded Unstructured response normalization
## What changed
- Admit raw provider JSON before Zod cloning and layout normalization: scan JSON depth before decoding, then enforce raw element count, tree nodes, and response-byte budgets. Invalid or excessive provider output remains a non-retryable `provider_response_invalid` error.
- Replace repeated full-document CJK glyph scans with a conservative page-local spatial grid. Only neighboring buckets are examined, consumed candidates are removed, and exact adjacency plus the existing deterministic distance/tie ordering are preserved.
- Apply an explicit comparison budget to dense/adversarial buckets instead of silently omitting evidence or allowing quadratic work to continue.
- Bound heading ancestry depth and cumulative emitted section-path items before their repeated expansion.
- Detect request deadline overruns using monotonic elapsed time as well as the timeout callback. Synchronous work cannot report success merely because it delayed the event-loop timer.
- Bound HTML `colspan` / `rowspan` expansion before each cell assignment, including empty cells and the logical width of sparse carried rows. A parse-local budget spans all native HTML tables, HTML tokens within Markdown/MDX, and provider `text_as_html` tables. It also charges the extra padding needed to scan ragged rows as rectangular tables. Independent parses never share this counter. Bound dense header/headerless scans and joined header bytes before projection, and compute maximum width without variadic argument expansion.
- Charge projected table text bytes against the same document-local state before joining header-only labels or constructing data-row strings. Ordinary Markdown tables and HTML tables share this byte counter, preventing repeated individually-admitted tables from allocating many output-sized strings before the final artifact guard. Temporary flattened-header bytes remain a separate admission check and are not double charged.
Defaults:
| Resource | Limit |
|---|---:|
| Raw provider elements | 50,000 |
| Raw provider / final artifact tree nodes | 1,000,000 |
| JSON structural depth | 128 |
| Raw response bytes | Existing configured `maxResponseBytes` (default 32 MiB) |
| Heading depth | 64 |
| Cumulative emitted section-path items | 100,000 |
| Vertical candidate comparisons | 1,000,000 |
| Expanded HTML table columns | 4,096 |
| Cumulative expanded HTML table cells / dense traversal cells, per document | 500,000 |
| Flattened HTML header bytes | 32 MiB |
| Cumulative projected table text bytes, per document | 32 MiB |
The existing final `maxElements` remains unchanged (default 20,000). Native structured-record budgets remain separate. All over-budget cases fail explicitly; no truncation or empty-artifact fallback was introduced.
## Why
The all-format audit reproduced two distinct post-response amplification paths: repeated glyph scans across unrelated pages, and quadratic ancestry-array expansion. An HTTP byte cap alone did not constrain either path. Element filtering and final limits were also too late to protect schema and normalization work.
The implementation preserves transport-owned cancellation and single-flight. Shared operations must not inspect the first caller's cancelled signal while serving another active caller.
## Verification
- TDD: five existing-behavior tests failed before the normalization/deadline implementation (parent-chain depth, category depth, cumulative path expansion, dense candidate work budget, delayed-timer deadline); three raw-response tests failed before raw admission (filtered raw elements, hidden metadata tree nodes, excessive JSON depth).
- Dedicated tests cover local candidate counts, page separation, negative/spatial bucket boundaries, large finite coordinates, tie ordering, coordinate-system compatibility, geometry overflow, boundary admission, and rejection classification.
- The 17 normalization/grid tests plus 5 raw-admission tests passed.
- HTML table TDD: four existing-behavior regressions failed before implementation (empty overwide colspan, cumulative empty cells, carried rowspans, and sparse carried logical width). Four follow-up regressions confirmed separate HTML / Markdown / MDX / provider tables could otherwise reset this counter, and one further test caught aggregate ragged-row padding. Nineteen dedicated tests now pass, including independent-parse isolation, exact-boundary admission, unchanged multirow-header/carry output, pre-projection traversal limits, and pre-join header bytes. These tests use small mocked budgets to exercise the real parser paths without large allocations.
- Document table-byte TDD: five regressions failed before the shared byte counter (HTML headers/data rows, Markdown and MDX tables, and provider tables). Ten dedicated tests now pass, including ordinary Markdown mixed with HTML tokens, exact cumulative byte admission, and independent native/provider parses.
- Independent JSON/resource helper tests also exposed and verified root-agent fixes for prototype-shaped JSON fields and invalid numeric budget options. Together these five dedicated test files contain 90 passing tests.
- Four focused helper modules measured 100% statement/line/function coverage and 97.79% branch coverage; each exceeded 90% branch coverage. Parser TypeScript checking passed.
- The original parser suite passed after the localized changes; full workspace verification remains the responsibility of the parent task's integrated verification after concurrent parser edits finish.
- After the final parse-local table cell/byte budget changes, all 442 parser tests, parser TypeScript checking, and `git diff --check` passed. Broader workspace checks remain with the parent task.
- Bounded diagnostic only: a simulated 2,352,895-byte response containing 12,000 cross-page CJK glyphs retained all 12,000 output elements in 221 ms locally, compared with about 2.2 seconds in the audit's previous implementation. These timings are not a production benchmark or SLA, and no OOM-scale inputs or network provider requests were run.
## Remaining boundaries
- Monotonic checks recognize overruns but cannot forcibly interrupt JavaScript currently executing on the same event loop. Hard CPU/RSS cancellation still requires worker/subprocess isolation.
- Raw response buffering and native `JSON.parse` allocation still occur in the API process, within the existing byte/depth bounds. The structural pass runs after decoding but before schema cloning and normalization.
- The spatial grid is conservative, not a claim of constant time for all inputs. Dense or extreme-geometry inputs can exhaust the explicit comparison budget and be rejected.
- New finite structural limits can reject unusually deep or complex otherwise-readable documents. This is explicit resource admission, not a promise that all previously accepted files remain accepted.
- The raw JSON tree budget does not inspect the internal semantics of strings. HTML table expansion now has its own pre-expansion budget; future nested content decoders must also enforce their own structural limits.
- No commit, push, deployment, or database migration was performed by this subtask.

61
knowledge-fs/.harness/changes/2026-09-05-unstructured-request-isolation-gate.md Normal file
View File

@ -0,0 +1,61 @@
# Optional Unstructured request isolation and release gates
## What / why
- Added an opt-in, single-service supervisor derived from the existing pinned Unstructured
image, with bounded shared admission, temporary request spooling, disposable process groups,
disconnect/deadline cleanup, inherited Linux limits, and sampled aggregate resource checks.
- Added shared nested EML/MSG and archive/XML/Excel admission. Implicit XLSX coordinates are
supported; renamed MSG content is detected through OLE properties rather than trusting `.xls`.
- Added Docker overrides for the existing dedicated service only. Base deployment files and
legacy Dify Unstructured are unchanged; unverified runtime is not enabled automatically.
- Added a tiny real-image golden runner and a licensed, hash-checked upstream MSG fixture. No fixtures are actual
user documents; no dangerous OOM payload is sent to an unbounded parser.
- Added client classification for the exact versioned supervisor contract. Confirmed 413/422
resource/input rejections become nonretryable `provider_input`; confirmed killed-worker 504
becomes nonretryable, non-ambiguous `provider_timeout`. Unknown errors and 429/503 preserve
existing behavior. Shared bounded error-body reading also preserves the existing PDF guard.
- Added version-scoped executable adapters for DOC/PPT LibreOffice and RTF/EPUB/ODT Pandoc
conversions. Products are admitted against the root/mail shared locked budget before
atomic publication or stdout release. No upstream Python monkeypatch or raw fallback exists.
- Inspected original ASGI lifecycle requirements; image build asserts there are no startup,
shutdown or custom lifespan hooks before permitting direct request replay.
## TDD / verification
- Initial admission and worker tests failed for absent implementation; targeted sparse Excel,
nested attachment budget, implicit coordinate, and renamed MSG regressions were then fixed.
- A deterministic yielding semaphore reproduced burst over-admission (20 reservations with
capacity 1); an atomic pre-await capacity reservation fixes it.
- A blocked response-send test reproduced indefinite admission retention; delivery is now
bounded by disconnect and the same document deadline, without emitting a second response.
- Conversion TDD covered byte/structure rejection, concurrent shared-budget spend, exact CLI
shapes, HTML spans, lifecycle deadlines and bounded output. A fake converter creating the
destination during execution exposed an overwrite race; publish now uses atomic link-if-absent.
- Independent review found signal-killed converters could leave the sticky ledger clear.
Regression failed for both missing rejection and wrong 422 mapping; negative statuses now
become sticky `worker_resource_limit` 413. A provider that catches the rejection cannot
publish an apparent success. Ordinary positive converter error codes remain unchanged.
- 80 Python tests pass, including real child/descendant process cancellation and cleanup.
Subprocess-aware coverage: 96.90% statements, 92.05% branches, 95.61% combined.
- 37 TypeScript response-guard tests pass; parser package typecheck passes.
- `ruff check` and formatting pass for the new service. Local merged Compose config passes.
- Full workspace verification is performed by the coordinating change; these focused commands
were not repeatedly rerun without relevant edits.
## Remaining gates / honest limits
Docker daemon is unavailable on this machine (`/Users/jyong/.orbstack/run/docker.sock` missing).
The real pinned image has therefore **not been built, started, or golden-tested**. The pinned
public MSG fixture is now bundled with its full Apache license and SHA-256 attribution; actual
local oxmsg validates its 30-byte attachment, while full HTTP output remains an image gate.
Converted DOCX/PPTX/HTML structural re-admission is implemented and tested with real child
processes plus generated products. Its image-specific discovery/CLI compatibility still needs
the real golden gate. Hard aggregate request-specific cgroups are not present;
RSS/CPU/PID/tmp sampling can overshoot, and a fast-detaching unobserved descendant is not proven
captured. HTTP PDF self-parallelism is disabled inside this optional worker to avoid admission
deadlock; throughput must be benchmarked before promotion. These are release gates, not completed
production protections. See `services/unstructured-sandbox/README.md`.
No production deployment, database change, user-file upload, commit, or push occurred.

10
knowledge-fs/apps/api/Dockerfile
View File

@ -63,16 +63,20 @@ ENV NODE_ENV=production \
WORKDIR /workspace
# PDF image elements are rasterized outside Unstructured. Keep Poppler in the
# final image and fail the build if the executable is not usable on the target
# platform. Deployments can set KNOWLEDGE_PDF_RASTERIZER=off as a kill switch.
# final image and verify both the renderer and metadata inspector. pdfinfo is also
# required by remote PDF admission, even when local image rasterization is off.
RUN apt-get update \
&& apt-get install --yes --no-install-recommends poppler-utils \
&& rm -rf /var/lib/apt/lists/* \
&& command -v pdftoppm >/dev/null \
&& pdftoppm -v >/dev/null 2>&1
&& pdftoppm -v >/dev/null 2>&1 \
&& command -v pdfinfo >/dev/null \
&& pdfinfo -v >/dev/null 2>&1
COPY --from=builder /workspace/apps/api/dist/server.mjs ./server.mjs
COPY --from=builder /workspace/apps/api/dist/migrate.mjs ./migrate.mjs
COPY --from=builder /workspace/apps/api/dist/native-parser-worker.mjs ./native-parser-worker.mjs
COPY --from=builder /workspace/apps/api/dist/image-variant-worker.mjs ./image-variant-worker.mjs
COPY --from=builder /runtime/node_modules ./node_modules
# Fail the image build if the target platform's sharp binary or libvips payload

2
knowledge-fs/apps/api/package.json
View File

@ -4,7 +4,7 @@
"type": "module",
"scripts": {
"build": "tsc --noEmit",
"build:prod": "esbuild src/server.ts src/migrate.ts --bundle --platform=node --format=esm --target=node22 --external:sharp --banner:js=\"import { createRequire as __knowledgeCreateRequire } from 'node:module';const require = __knowledgeCreateRequire(import.meta.url);\" --outdir=dist --out-extension:.js=.mjs",
"build:prod": "esbuild src/server.ts src/migrate.ts src/native-parser-worker.ts src/image-variant-worker.ts --bundle --platform=node --format=esm --target=node22 --external:sharp --banner:js=\"import { createRequire as __knowledgeCreateRequire } from 'node:module';const require = __knowledgeCreateRequire(import.meta.url);\" --outdir=dist --out-extension:.js=.mjs",
"dev": "NODE_ENV=development node --env-file-if-exists=../../infra/local/.env --import tsx --watch src/server.ts",
"start": "node dist/server.mjs",
"test": "vitest run --passWithNoTests",

100
knowledge-fs/apps/api/src/image-variant-protocol.test.ts Normal file
View File

@ -0,0 +1,100 @@
import { describe, expect, it } from "vitest";
import { isImageInputRejection, validateImageVariantResponse } from "./image-variant-protocol";
import { imageVariantsFromResponse } from "./isolated-image-variant-generator";
const variant = () => ({
body: new Uint8Array([1]),
contentType: "image/png",
name: "analysis",
height: 12,
width: 24,
});
describe("image worker IPC response validation", () => {
it("classifies malformed worker replies as operational errors, not rejected documents", () => {
expect(() => imageVariantsFromResponse({ ok: true, variants: [{}] })).toThrow(
expect.objectContaining({ code: "provider_response_invalid" }),
);
expect(() =>
imageVariantsFromResponse({ ok: false, message: "Input image exceeds pixel limit" }),
).toThrow(expect.objectContaining({ code: "provider_input" }));
expect(imageVariantsFromResponse({ ok: true, variants: [variant()] })).toHaveLength(1);
});
it.each([
"Input image exceeds pixel limit",
"Input buffer contains unsupported image format",
"Sharp image thumbnail output exceeds maxOutputBytes",
"Image variants exceed the output byte budget",
])("recognizes bounded image rejection: %s", (message) => {
expect(isImageInputRejection(new Error(message))).toBe(true);
});
it.each([
new Error("Cannot find package sharp"),
new Error("Unexpected worker state"),
"Input image exceeds pixel limit",
])("does not silently downgrade operational decoder failures", (error) => {
expect(isImageInputRejection(error)).toBe(false);
});
it("rejects unbounded or malformed optional execution diagnostics", () => {
expect(
validateImageVariantResponse({
ok: true,
variants: [{ ...variant(), execution: { wallMs: Number.POSITIVE_INFINITY } }],
}),
).toBe(false);
});
it("accepts bounded success and failure messages", () => {
expect(validateImageVariantResponse({ ok: true, variants: [variant()] })).toBe(true);
expect(validateImageVariantResponse({ ok: true, variants: [] })).toBe(true);
expect(validateImageVariantResponse({ ok: false, message: "Image rejected" })).toBe(true);
});
it.each([
undefined,
null,
[],
{},
{ ok: 1 },
{ ok: true },
{ ok: true, variants: null },
{ ok: true, variants: [null] },
{ ok: true, variants: [variant(), variant(), variant()] },
{ ok: false, message: 1 },
{ ok: false, message: "" },
{ ok: false, message: "x".repeat(1025) },
])("rejects malformed message %j", (message) => {
expect(validateImageVariantResponse(message)).toBe(false);
});
it.each([
{ body: [1] },
{ body: new Uint8Array() },
{ body: new Uint8Array(16 * 1024 * 1024 + 1) },
{ contentType: "text/plain" },
{ name: "../escape" },
{ name: "x".repeat(65) },
{ width: 0 },
{ height: -1 },
{ width: Number.NaN },
{ height: 1.5 },
])("rejects malformed variant field", (patch) => {
expect(validateImageVariantResponse({ ok: true, variants: [{ ...variant(), ...patch }] })).toBe(
false,
);
});
it("bounds aggregate variant bytes and permits absent optional dimensions", () => {
expect(
validateImageVariantResponse({
ok: true,
variants: [
{ ...variant(), body: new Uint8Array(8 * 1024 * 1024 + 1) },
{ ...variant(), body: new Uint8Array(8 * 1024 * 1024) },
],
}),
).toBe(false);
expect(
validateImageVariantResponse({
ok: true,
variants: [{ body: new Uint8Array([1]), contentType: "image/png", name: "thumbnail" }],
}),
).toBe(true);
});
});

89
knowledge-fs/apps/api/src/image-variant-protocol.ts Normal file
View File

@ -0,0 +1,89 @@
import type {
GenerateDocumentImageVariantsInput,
GeneratedDocumentImageVariant,
SharpImageThumbnailVariantGeneratorOptions,
} from "../../../packages/api/src/document-image-variant-generator";
export interface ImageVariantRequest {
readonly input: Omit<GenerateDocumentImageVariantsInput, "signal">;
readonly options: SharpImageThumbnailVariantGeneratorOptions;
}
export type ImageVariantResponse =
| { readonly ok: true; readonly variants: readonly GeneratedDocumentImageVariant[] }
| { readonly ok: false; readonly message: string };
export const imageVariantMaxInputBytes = 50 * 1024 * 1024;
export const imageVariantMaxOutputBytes = 16 * 1024 * 1024;
/** Unknown/native initialization errors remain operational failures instead of silently omitting media. */
export function isImageInputRejection(error: unknown): error is Error {
return (
error instanceof Error &&
/^(?:Input (?:image exceeds pixel limit|buffer (?:contains unsupported image format|has corrupt header))|Sharp image thumbnail output exceeds maxOutputBytes|Image variants exceed the output byte budget)/u.test(
error.message,
)
);
}
export function validateImageVariantResponse(value: unknown): value is ImageVariantResponse {
if (typeof value !== "object" || value === null || Array.isArray(value) || !("ok" in value))
return false;
if (value.ok === false)
return (
"message" in value &&
typeof value.message === "string" &&
value.message.length > 0 &&
value.message.length <= 1024
);
if (
value.ok !== true ||
!("variants" in value) ||
!Array.isArray(value.variants) ||
value.variants.length > 2
)
return false;
let bytes = 0;
const names = new Set<string>();
for (const variant of value.variants) {
if (
typeof variant !== "object" ||
variant === null ||
!(variant.body instanceof Uint8Array) ||
variant.body.byteLength === 0 ||
variant.contentType !== "image/png" ||
typeof variant.name !== "string" ||
!/^[a-zA-Z0-9_-]{1,64}$/u.test(variant.name) ||
names.has(variant.name)
)
return false;
if (variant.execution !== undefined) {
const execution = variant.execution;
if (
typeof execution !== "object" ||
execution === null ||
execution.isolation !== "child-process" ||
!Number.isSafeInteger(execution.inputBytes) ||
execution.inputBytes < 0 ||
execution.inputBytes > imageVariantMaxInputBytes ||
!Number.isSafeInteger(execution.outputBytes) ||
execution.outputBytes < 0 ||
execution.outputBytes > imageVariantMaxOutputBytes ||
!Number.isFinite(execution.wallMs) ||
execution.wallMs < 0 ||
!Number.isSafeInteger(execution.peakRssKiB) ||
execution.peakRssKiB < 0
)
return false;
}
if (
(variant.width !== undefined &&
(!Number.isSafeInteger(variant.width) || variant.width < 1)) ||
(variant.height !== undefined &&
(!Number.isSafeInteger(variant.height) || variant.height < 1))
)
return false;
names.add(variant.name);
bytes += variant.body.byteLength;
if (bytes > imageVariantMaxOutputBytes) return false;
}
return true;
}

44
knowledge-fs/apps/api/src/image-variant-worker.ts Normal file
View File

@ -0,0 +1,44 @@
import { createSharpImageThumbnailVariantGenerator } from "../../../packages/api/src/document-image-variant-generator";
import {
type ImageVariantRequest,
type ImageVariantResponse,
imageVariantMaxInputBytes,
imageVariantMaxOutputBytes,
isImageInputRejection,
} from "./image-variant-protocol";
process.once("message", async (request: ImageVariantRequest) => {
const started = performance.now();
let response: ImageVariantResponse;
try {
if (
!(request.input.body instanceof Uint8Array) ||
request.input.body.byteLength > imageVariantMaxInputBytes
)
throw new Error("Image worker input exceeds its byte budget");
const variants = await createSharpImageThumbnailVariantGenerator(request.options).generate(
request.input,
);
if (
variants.reduce((sum, variant) => sum + variant.body.byteLength, 0) >
imageVariantMaxOutputBytes
)
throw new Error("Image variants exceed the output byte budget");
const execution = {
isolation: "child-process" as const,
inputBytes: request.input.body.byteLength,
outputBytes: variants.reduce((sum, variant) => sum + variant.body.byteLength, 0),
wallMs: performance.now() - started,
peakRssKiB: process.resourceUsage().maxRSS,
};
response = { ok: true, variants: variants.map((variant) => ({ ...variant, execution })) };
} catch (error) {
if (!isImageInputRejection(error)) throw error;
response = {
ok: false,
message:
error instanceof Error ? error.message.slice(0, 1024) : "Image variant generation failed",
};
}
process.send?.(response, () => process.disconnect());
});

59
knowledge-fs/apps/api/src/isolated-image-variant-generator.test.ts Normal file
View File

@ -0,0 +1,59 @@
import { describe, expect, it } from "vitest";
import { createIsolatedImageVariantGenerator } from "./isolated-image-variant-generator";
describe("isolated image variants", () => {
it("rejects incompatible variant names before starting any image work", () => {
expect(() => createIsolatedImageVariantGenerator({ variantName: "../escape" })).toThrow(
"Image variant name",
);
});
it("decodes in a child and returns separate preview and analysis variants", async () => {
const sharp = (await import("sharp")).default;
const body = await sharp({
create: { width: 640, height: 320, channels: 3, background: "white" },
})
.png()
.toBuffer();
const generator = createIsolatedImageVariantGenerator({ analysisMaxDimension: 512 });
const variants = await generator.generate({
body,
contentType: "image/png",
elementId: "figure",
});
expect(variants.map(({ name, width, height }) => ({ name, width, height }))).toEqual([
{ name: "thumbnail", width: 320, height: 160 },
{ name: "analysis", width: 512, height: 256 },
]);
expect(variants[0]?.execution).toMatchObject({
isolation: "child-process",
inputBytes: body.byteLength,
outputBytes: expect.any(Number),
wallMs: expect.any(Number),
peakRssKiB: expect.any(Number),
});
});
it("returns a terminal input error when the decoder rejects pixels", async () => {
const generator = createIsolatedImageVariantGenerator({ maxInputPixels: 100 });
await expect(
generator.generate({
body: new TextEncoder().encode(
'<svg xmlns="http://www.w3.org/2000/svg" width="100" height="100"/>',
),
contentType: "image/svg+xml",
elementId: "too-large",
}),
).rejects.toMatchObject({ code: "provider_input", retryable: false });
});
it("does not start a decoder after the execution lease was cancelled", async () => {
const controller = new AbortController();
controller.abort(new Error("lease lost"));
await expect(
createIsolatedImageVariantGenerator().generate({
body: new Uint8Array([1]),
contentType: "image/png",
elementId: "cancelled",
signal: controller.signal,
}),
).rejects.toThrow("lease lost");
});
});

75
knowledge-fs/apps/api/src/isolated-image-variant-generator.ts Normal file
View File

@ -0,0 +1,75 @@
import { fork } from "node:child_process";
import { ProviderInputError, ProviderResponseError } from "@knowledge/parsers";
import {
type DocumentImageVariantGenerator,
type GeneratedDocumentImageVariant,
type SharpImageThumbnailVariantGeneratorOptions,
createSharpImageThumbnailVariantGenerator,
} from "../../../packages/api/src/document-image-variant-generator";
import {
type ImageVariantRequest,
type ImageVariantResponse,
imageVariantMaxInputBytes,
validateImageVariantResponse,
} from "./image-variant-protocol";
import { createIsolatedProcessExecutor } from "./isolated-process-executor";
const executor = createIsolatedProcessExecutor<ImageVariantRequest, ImageVariantResponse>({
maxConcurrency: 2,
maxInputBytes: imageVariantMaxInputBytes,
maxReservedBytes: 64 * 1024 * 1024,
maxRssBytes: 384 * 1024 * 1024,
maxQueued: 8,
timeoutMs: 30_000,
spawn: () => {
const compiled = import.meta.url.endsWith(".mjs");
return fork(
new URL(
compiled ? "./image-variant-worker.mjs" : "./image-variant-worker.ts",
import.meta.url,
),
[],
{
execArgv: ["--max-old-space-size=128", ...(compiled ? [] : ["--import", "tsx"])],
env: {
PATH: process.env.PATH,
SYSTEMROOT: process.env.SYSTEMROOT,
VIPS_CONCURRENCY: "1",
MALLOC_ARENA_MAX: "2",
},
serialization: "advanced",
stdio: ["ignore", "ignore", "ignore", "ipc"],
},
);
},
});
export function imageVariantsFromResponse(
value: unknown,
): readonly GeneratedDocumentImageVariant[] {
if (!validateImageVariantResponse(value))
throw new ProviderResponseError("Image worker returned an invalid or over-budget response");
if (!value.ok) throw new ProviderInputError(value.message);
return value.variants;
}
export function createIsolatedImageVariantGenerator(
options: SharpImageThumbnailVariantGeneratorOptions = {},
): DocumentImageVariantGenerator {
createSharpImageThumbnailVariantGenerator(options); // Validate configuration before accepting jobs.
if (options.variantName !== undefined && !/^[a-zA-Z0-9_-]{1,64}$/u.test(options.variantName))
throw new Error(
"Image variant name must contain 1..64 ASCII letters, digits, hyphens or underscores",
);
const frozenOptions = Object.freeze({ ...options });
return {
generate: async ({ signal, ...input }) => {
const response = await executor.execute(
{ input, options: frozenOptions },
input.body.byteLength,
signal,
);
return imageVariantsFromResponse(response);
},
};
}

186
knowledge-fs/apps/api/src/isolated-process-executor.ts Normal file
View File

@ -0,0 +1,186 @@
import type { ChildProcess } from "node:child_process";
import { ProviderError, ProviderInputError, ProviderResponseError } from "@knowledge/parsers";
import { readProcessRssBytes, superviseProcessMemory } from "./isolated-process-memory";
export interface IsolatedProcessOptions {
readonly maxConcurrency?: number;
readonly maxQueued?: number;
readonly maxInputBytes?: number;
readonly maxReservedBytes?: number;
readonly timeoutMs?: number;
readonly maxRssBytes?: number;
readonly readRssBytes?: (pid: number) => Promise<number | undefined>;
readonly spawn: () => ChildProcess;
}
/** Shared bounded process lifecycle; resolve/release only after the worker actually exits. */
export function createIsolatedProcessExecutor<TRequest, TResponse>({
maxConcurrency = 2,
maxQueued = 32,
maxInputBytes = 50 * 1024 * 1024,
maxReservedBytes = 128 * 1024 * 1024,
timeoutMs = 600_000,
maxRssBytes,
readRssBytes = readProcessRssBytes,
spawn,
}: IsolatedProcessOptions) {
for (const value of [maxConcurrency, maxQueued, maxInputBytes, maxReservedBytes, timeoutMs]) {
if (!Number.isSafeInteger(value) || value < 1)
throw new Error("Invalid native parser isolation budget");
}
if (maxRssBytes !== undefined && (!Number.isSafeInteger(maxRssBytes) || maxRssBytes < 1))
throw new Error("Invalid worker RSS budget");
let active = 0;
let reservedBytes = 0;
const queue: Array<() => void> = [];
const acquire = (signal: AbortSignal): Promise<() => void> =>
new Promise((resolve, reject) => {
signal.throwIfAborted();
const enter = () => {
signal.removeEventListener("abort", cancel);
active++;
let released = false;
resolve(() => {
if (released) return;
released = true;
active--;
queue.shift()?.();
});
};
const cancel = () => {
const index = queue.indexOf(enter);
if (index >= 0) queue.splice(index, 1);
reject(signal.reason);
};
if (active < maxConcurrency) enter();
else if (queue.length >= maxQueued)
reject(
new ProviderError("Native parser admission queue is full", {
code: "provider_rate_limited",
retryable: true,
}),
);
else {
queue.push(enter);
signal.addEventListener("abort", cancel, { once: true });
}
});
return {
async execute(request: TRequest, inputBytes: number, signal?: AbortSignal): Promise<TResponse> {
signal?.throwIfAborted();
if (!Number.isSafeInteger(inputBytes) || inputBytes < 0)
throw new ProviderInputError(
"Isolated process input byte count must be a non-negative safe integer",
);
if (inputBytes > maxInputBytes)
throw new ProviderInputError(`Native parser input exceeds maxInputBytes=${maxInputBytes}`);
if (reservedBytes + inputBytes > maxReservedBytes)
throw new ProviderError("Native parser reserved-input byte budget is full", {
code: "provider_rate_limited",
retryable: true,
});
reservedBytes += inputBytes;
const controller = new AbortController();
const cancel = () => controller.abort(signal?.reason);
signal?.addEventListener("abort", cancel, { once: true });
const timer = setTimeout(
() =>
controller.abort(
new ProviderError("Native parser wall-clock budget exceeded", {
code: "provider_timeout",
retryable: false,
requestOutcomeAmbiguous: false,
}),
),
timeoutMs,
);
let release: (() => void) | undefined;
try {
release = await acquire(controller.signal);
controller.signal.throwIfAborted();
return await runChild<TRequest, TResponse>(
spawn,
request,
controller.signal,
maxRssBytes,
readRssBytes,
);
} finally {
release?.();
reservedBytes -= inputBytes;
clearTimeout(timer);
signal?.removeEventListener("abort", cancel);
}
},
};
}
function runChild<TRequest, TResponse>(
spawn: () => ChildProcess,
request: TRequest,
signal: AbortSignal,
maxRssBytes: number | undefined,
readRssBytes: (pid: number) => Promise<number | undefined>,
): Promise<TResponse> {
return new Promise((resolve, reject) => {
const child = spawn();
let response: TResponse | undefined;
let failure: unknown;
let failed = false;
const stopMemory =
child.pid !== undefined && maxRssBytes !== undefined
? superviseProcessMemory({
pid: child.pid,
maxRssBytes,
readRssBytes,
stop: (reason) => {
if (failed || signal.aborted) return;
failed = true;
failure = reason;
child.kill("SIGKILL");
},
})
: () => {};
const cancel = () => {
failed = true;
failure = signal.reason;
child.kill("SIGKILL");
};
signal.addEventListener("abort", cancel, { once: true });
child.once("error", (error) => {
failed = true;
failure = error;
child.kill("SIGKILL");
});
child.once("message", (message: TResponse) => {
response = message;
});
child.once("close", (code: number | null, exitSignal: NodeJS.Signals | null) => {
stopMemory();
signal.removeEventListener("abort", cancel);
if (failed) reject(failure);
else if (response === undefined || code !== 0 || exitSignal !== null)
reject(
new ProviderResponseError("Isolated worker exited without a valid completed response"),
);
else resolve(response);
});
if (signal.aborted) cancel();
else {
try {
child.send(request as Parameters<ChildProcess["send"]>[0], (error) => {
if (error) {
failed = true;
failure = error;
child.kill("SIGKILL");
}
});
} catch (error) {
failed = true;
failure = error;
child.kill("SIGKILL");
}
}
});
}

84
knowledge-fs/apps/api/src/isolated-process-memory-reader.test.ts Normal file
View File

@ -0,0 +1,84 @@
import { open } from "node:fs/promises";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
parseProcStatusRss,
readProcessRssBytes,
superviseProcessMemory,
} from "./isolated-process-memory";
vi.mock("node:fs/promises", () => ({ open: vi.fn() }));
const originalPlatform = process.platform;
afterEach(() => {
Object.defineProperty(process, "platform", { value: originalPlatform });
vi.useRealTimers();
vi.clearAllMocks();
});
describe("bounded Linux RSS reader", () => {
it("parses KiB and handles zombie status without invented values", () => {
expect(parseProcStatusRss("Name: node\nVmRSS:\t1200 kB\n")).toBe(1200 * 1024);
expect(parseProcStatusRss("Name: node\nState: Z\n")).toBeUndefined();
expect(() => parseProcStatusRss("VmRSS: 99999999999999999 kB\n")).toThrow("counter");
});
it("does not attempt procfs on non-Linux adapters", async () => {
Object.defineProperty(process, "platform", { value: "darwin" });
expect(await readProcessRssBytes(1)).toBeUndefined();
expect(open).not.toHaveBeenCalled();
});
it("uses a bounded read and closes the descriptor", async () => {
Object.defineProperty(process, "platform", { value: "linux" });
const close = vi.fn();
const read = vi.fn(async (buffer: Buffer) => ({ bytesRead: buffer.write("VmRSS: 10 kB\n") }));
vi.mocked(open).mockResolvedValue({ read, close } as never);
expect(await readProcessRssBytes(123)).toBe(10240);
expect(open).toHaveBeenCalledWith("/proc/123/status", "r");
expect(read.mock.calls[0]?.[0].length).toBe(16 * 1024);
expect(close).toHaveBeenCalledOnce();
});
it("handles exit races and rejects unavailable or oversized metadata", async () => {
Object.defineProperty(process, "platform", { value: "linux" });
await expect(readProcessRssBytes(-1)).rejects.toThrow("PID");
vi.mocked(open).mockRejectedValueOnce(Object.assign(new Error(), { code: "ENOENT" }));
expect(await readProcessRssBytes(123)).toBeUndefined();
vi.mocked(open).mockRejectedValueOnce(new Error("permission"));
await expect(readProcessRssBytes(123)).rejects.toThrow("permission");
const close = vi.fn();
vi.mocked(open).mockResolvedValueOnce({
read: vi.fn(async () => ({ bytesRead: 16 * 1024 })),
close,
} as never);
await expect(readProcessRssBytes(123)).rejects.toThrow("metadata budget");
expect(close).toHaveBeenCalledOnce();
});
it("never overlaps probes and ignores a late sample after disposal", async () => {
vi.useFakeTimers();
let resolve: (bytes: number) => void = () => {};
const readRssBytes = vi.fn(
() =>
new Promise<number>((done) => {
resolve = done;
}),
);
const stop = vi.fn();
const dispose = superviseProcessMemory({ pid: 1, maxRssBytes: 1, readRssBytes, stop });
await vi.advanceTimersByTimeAsync(200);
expect(readRssBytes).toHaveBeenCalledOnce();
dispose();
resolve(2000);
await Promise.resolve();
expect(stop).not.toHaveBeenCalled();
});
it("fails closed on a live probe error and accepts values below the budget", async () => {
vi.useFakeTimers();
const stop = vi.fn();
const readRssBytes = vi
.fn()
.mockResolvedValueOnce(undefined)
.mockResolvedValueOnce(10)
.mockRejectedValueOnce(new Error());
const dispose = superviseProcessMemory({ pid: 1, maxRssBytes: 100, readRssBytes, stop });
await vi.advanceTimersByTimeAsync(100);
expect(stop).toHaveBeenCalledWith(expect.objectContaining({ code: "provider_request_failed" }));
dispose();
});
});

24
knowledge-fs/apps/api/src/isolated-process-memory.test.ts Normal file
View File

@ -0,0 +1,24 @@
import type { ChildProcess } from "node:child_process";
import { EventEmitter } from "node:events";
import { describe, expect, it, vi } from "vitest";
import { createIsolatedProcessExecutor } from "./isolated-process-executor";
describe("isolated parser resident memory supervision", () => {
it("kills the worker when native allocations exceed the sampled RSS budget", async () => {
const child = new EventEmitter() as ChildProcess;
Object.defineProperty(child, "pid", { value: 123 });
child.send = vi.fn(() => true) as unknown as ChildProcess["send"];
child.kill = vi.fn(() => {
queueMicrotask(() => child.emit("close", null, "SIGKILL"));
return true;
});
const executor = createIsolatedProcessExecutor({
spawn: () => child,
timeoutMs: 500,
maxRssBytes: 1000,
readRssBytes: async () => 2000,
});
await expect(executor.execute({}, 0)).rejects.toThrow("resident-memory");
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
});
});

71
knowledge-fs/apps/api/src/isolated-process-memory.ts Normal file
View File

@ -0,0 +1,71 @@
import { open } from "node:fs/promises";
import { ProviderError, ProviderInputError } from "@knowledge/parsers";
export function parseProcStatusRss(status: string): number | undefined {
const value = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1];
if (value === undefined) return undefined; // Zombie processes have no VmRSS.
const bytes = Number(value) * 1024;
if (!Number.isSafeInteger(bytes)) throw new Error("Invalid process RSS counter");
return bytes;
}
export async function readProcessRssBytes(pid: number): Promise<number | undefined> {
if (process.platform !== "linux") return undefined;
if (!Number.isSafeInteger(pid) || pid < 1) throw new Error("Invalid worker PID");
try {
const file = await open(`/proc/${pid}/status`, "r");
try {
const buffer = Buffer.alloc(16 * 1024);
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
if (bytesRead === buffer.length) throw new Error("Process status exceeds metadata budget");
return parseProcStatusRss(buffer.toString("utf8", 0, bytesRead));
} finally {
await file.close();
}
} catch (error) {
if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT")
return undefined;
throw error;
}
}
/** Sampled stop for native allocations; not a hard cgroup or hostile-code sandbox. */
export function superviseProcessMemory(input: {
readonly pid: number;
readonly maxRssBytes: number;
readonly readRssBytes: (pid: number) => Promise<number | undefined>;
readonly stop: (reason: Error) => void;
}): () => void {
let stopped = false;
let sampling = false;
const sample = async () => {
if (stopped || sampling) return;
sampling = true;
try {
const bytes = await input.readRssBytes(input.pid);
if (!stopped && bytes !== undefined && bytes > input.maxRssBytes)
input.stop(
new ProviderInputError("Document parser worker resident-memory budget exceeded"),
);
} catch {
if (!stopped)
input.stop(
new ProviderError("Document parser worker memory supervision unavailable", {
code: "provider_request_failed",
retryable: false,
}),
);
} finally {
sampling = false;
}
};
const timer = setInterval(() => {
void sample();
}, 50);
timer.unref();
void sample();
return () => {
stopped = true;
clearInterval(timer);
};
}

111
knowledge-fs/apps/api/src/isolated-process-review.test.ts Normal file
View File

@ -0,0 +1,111 @@
import type { ChildProcess } from "node:child_process";
import { EventEmitter } from "node:events";
import { createNativeStructuredDataParser } from "@knowledge/parsers";
import { describe, expect, it, vi } from "vitest";
import { createIsolatedProcessExecutor } from "./isolated-process-executor";
import { createNativeParserIsolation } from "./native-parser-isolation";
function fakeChild() {
const child = new EventEmitter() as ChildProcess;
child.send = vi.fn(() => true) as unknown as ChildProcess["send"];
child.kill = vi.fn(() => {
queueMicrotask(() => child.emit("close", null, "SIGKILL"));
return true;
});
return child;
}
const input = {
body: new TextEncoder().encode("{}"),
documentAssetId: "00000000-0000-4000-8000-000000000001",
filename: "x.json",
mimeType: "application/json",
version: 1,
};
describe("isolated process boundary regressions", () => {
it.each([-1, Number.NaN, Number.POSITIVE_INFINITY, 0.5])(
"rejects invalid input byte accounting %s before spawning",
async (inputBytes) => {
const spawn = vi.fn(() => fakeChild());
const executor = createIsolatedProcessExecutor({ spawn, timeoutMs: 5 });
await expect(executor.execute({}, inputBytes)).rejects.toMatchObject({
code: "provider_input",
});
expect(spawn).not.toHaveBeenCalled();
},
);
it.each([
[1, null],
[null, "SIGKILL"],
])("rejects a response followed by abnormal exit %s/%s", async (code, signal) => {
const child = fakeChild();
const executor = createIsolatedProcessExecutor({ spawn: () => child });
const result = expect(executor.execute({}, 1)).rejects.toMatchObject({
code: "provider_response_invalid",
retryable: false,
});
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
child.emit("message", { result: "not publishable" });
child.emit("close", code, signal);
await result;
});
it("classifies a clean exit without a result as a worker protocol failure", async () => {
const child = fakeChild();
const executor = createIsolatedProcessExecutor({ spawn: () => child });
const result = expect(executor.execute({}, 1)).rejects.toMatchObject({
code: "provider_response_invalid",
retryable: false,
});
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
child.emit("close", 0, null);
await result;
});
it.each([null, {}, { ok: true, artifact: {} }, { ok: false, message: null }])(
"classifies malformed worker output %#",
async (response) => {
const child = fakeChild();
const parser = createNativeParserIsolation({ spawn: () => child }).wrap(
createNativeStructuredDataParser(),
{},
);
const result = expect(parser.parse(input)).rejects.toMatchObject({
code: "provider_response_invalid",
retryable: false,
});
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
child.emit("message", response);
child.emit("close", 0, null);
await result;
},
);
it.each(["provider_input", "provider_response_invalid"])(
"preserves the worker's typed %s failure",
async (errorCode) => {
const child = fakeChild();
const parser = createNativeParserIsolation({ spawn: () => child }).wrap(
createNativeStructuredDataParser(),
{},
);
const result = expect(parser.parse(input)).rejects.toMatchObject({ code: errorCode });
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
child.emit("message", { ok: false, message: "failure", errorCode });
child.emit("close", 0, null);
await result;
},
);
it("rejects non-serializable callback options before accepting work", () => {
const runtimeOptions = { maxInputBytes: 15, now: () => "2026-01-01T00:00:00Z" };
expect(() =>
createNativeParserIsolation().wrap(createNativeStructuredDataParser(), runtimeOptions),
).toThrow("serializable");
});
it.each([null, false, 0])("preserves a falsy abort reason %s", async (reason) => {
const child = fakeChild();
const executor = createIsolatedProcessExecutor({ spawn: () => child });
const controller = new AbortController();
const result = expect(executor.execute({}, 1, controller.signal)).rejects.toBe(reason);
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
controller.abort(reason);
await result;
});
});

2
knowledge-fs/apps/api/src/multimodal-answer-options.test.ts
View File

@ -119,7 +119,7 @@ describe("createApiMultimodalAnswerOptions", () => {
await objectStorage.putObject({
body: new Uint8Array([1, 2, 3]),
contentType: "image/png",
key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png",
key: "tenant/spaces/space/documents/doc/assets/chart.png",
});
globalThis.fetch = (async (input, init) => {
const request = new Request(input, init);

6
knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts
View File

@ -37,7 +37,7 @@ describe("createApiMultimodalEnrichmentOptions", () => {
await objectStorage.putObject({
body: new Uint8Array([1, 2, 3]),
contentType: "image/png",
key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png",
key: "tenant/spaces/space/documents/doc/assets/chart-analysis.png",
});
globalThis.fetch = (async (input, init) => {
const request = new Request(input, init);
@ -88,6 +88,10 @@ describe("createApiMultimodalEnrichmentOptions", () => {
contentType: "image/png",
objectKey: "tenant/spaces/space/documents/doc/assets/chart.png",
variants: {
analysis: {
contentType: "image/png",
objectKey: "tenant/spaces/space/documents/doc/assets/chart-analysis.png",
},
thumbnail: {
contentType: "image/png",
objectKey: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png",

5
knowledge-fs/apps/api/src/multimodal-enrichment-options.ts
View File

@ -450,8 +450,9 @@ async function objectBackedImageSource({
readonly objectStorage: KnowledgeGatewayOptions["adapter"]["objectStorage"];
}): Promise<{ base64Data: string; mimeType: string } | undefined> {
const assetRef = input.item.assetRef;
const thumbnail = assetRef?.variants?.thumbnail;
const candidate = thumbnail?.objectKey ? thumbnail : assetRef;
if (assetRef?.analysisUnavailable !== undefined) return undefined;
const analysis = assetRef?.variants?.analysis;
const candidate = analysis?.objectKey ? analysis : assetRef;
const objectKey = candidate?.objectKey;
const contentType = candidate?.contentType;

11
knowledge-fs/apps/api/src/multimodal-options.ts
View File

@ -2,8 +2,8 @@ import {
type DocumentImageVariantGenerator,
type DocumentPdfRasterizer,
createPopplerPdfRasterizer,
createSharpImageThumbnailVariantGenerator,
} from "@knowledge/api";
import { createIsolatedImageVariantGenerator } from "./isolated-image-variant-generator";
export interface ApiMultimodalEnv {
readonly DIFY_ROOT_KNOWLEDGE_DOCUMENT_MATERIALIZATION_MAX_CONCURRENCY_OVERRIDE?:
@ -149,7 +149,14 @@ function imageThumbnailOptions(
}
return {
documentMultimodalImageVariantGenerator: createSharpImageThumbnailVariantGenerator({
documentMultimodalImageVariantGenerator: createIsolatedImageVariantGenerator({
analysisMaxDimension: Math.max(
2048,
positiveIntegerEnv(
env.KNOWLEDGE_IMAGE_THUMBNAIL_MAX_DIMENSION ?? "320",
"KNOWLEDGE_IMAGE_THUMBNAIL_MAX_DIMENSION",
),
),
...(env.KNOWLEDGE_IMAGE_THUMBNAIL_MAX_DIMENSION !== undefined
? {
maxDimension: positiveIntegerEnv(

5
knowledge-fs/apps/api/src/native-parser-busy.fixture.mjs Normal file
View File

@ -0,0 +1,5 @@
// Test-only bounded by the parent process deadline; deliberately cannot service its event loop.
process.once("message", () => {
process.send({ started: true });
while (true) Math.sqrt(2);
});

244
knowledge-fs/apps/api/src/native-parser-isolation.test.ts Normal file
View File

@ -0,0 +1,244 @@
import { type ChildProcess, fork } from "node:child_process";
import { EventEmitter } from "node:events";
import { createNativeStructuredDataParser } from "@knowledge/parsers";
import { describe, expect, it, vi } from "vitest";
import { createNativeParserIsolation } from "./native-parser-isolation";
const input = {
body: new TextEncoder().encode('{"value":"preserved"}'),
documentAssetId: "00000000-0000-4000-8000-000000000001",
filename: "example.json",
mimeType: "application/json",
version: 1,
};
function childFixture() {
const child = new EventEmitter() as ChildProcess;
child.send = vi.fn(() => true) as unknown as ChildProcess["send"];
child.kill = vi.fn(() => {
queueMicrotask(() => child.emit("close", null, "SIGKILL"));
return true;
});
return child;
}
describe("native parser process isolation", () => {
it("runs the real parser outside the API process and preserves its semantic identity", async () => {
const native = createNativeStructuredDataParser();
const isolated = createNativeParserIsolation().wrap(native, {});
const expected = await native.parse(input);
const actual = await isolated.parse(input);
expect(actual.elements.map(({ id: _id, ...element }) => element)).toEqual(
expected.elements.map(({ id: _id, ...element }) => element),
);
expect(actual.artifactHash).toEqual(expected.artifactHash);
expect(isolated.policyFingerprint?.(input)).toBe(native.policyFingerprint?.(input));
expect(actual.metadata.parserExecution).toMatchObject({ isolation: "child-process" });
expect(actual.metadata.parserExecution).toHaveProperty("peakRssKiB");
}, 15_000);
it("terminates synchronous work on cancellation before releasing its shared slot", async () => {
const first = childFixture();
const second = childFixture();
const spawn = vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second);
const isolation = createNativeParserIsolation({ spawn, maxConcurrency: 1 });
const parser = isolation.wrap(createNativeStructuredDataParser(), {});
const controller = new AbortController();
const aborted = expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow(
"cancelled",
);
await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce());
const queuedController = new AbortController();
const queued = expect(
parser.parse({ ...input, signal: queuedController.signal }),
).rejects.toThrow("queued");
queuedController.abort(new Error("queued"));
await queued;
controller.abort(new Error("cancelled"));
await aborted;
expect(first.kill).toHaveBeenCalledWith("SIGKILL");
expect(spawn).toHaveBeenCalledOnce();
});
it("enforces a real wall deadline on a non-responsive child", async () => {
const child = childFixture();
const parser = createNativeParserIsolation({ spawn: () => child, timeoutMs: 20 }).wrap(
createNativeStructuredDataParser(),
{},
);
await expect(parser.parse(input)).rejects.toMatchObject({
code: "provider_timeout",
requestOutcomeAmbiguous: false,
retryable: false,
});
expect(child.kill).toHaveBeenCalledWith("SIGKILL");
});
it("never spawns for cancelled or oversized input", async () => {
const spawn = vi.fn();
const parser = createNativeParserIsolation({ spawn, maxInputBytes: 1 }).wrap(
createNativeStructuredDataParser(),
{},
);
await expect(parser.parse(input)).rejects.toMatchObject({ code: "provider_input" });
await expect(
parser.parse({ ...input, signal: AbortSignal.abort(new Error("cancelled")) }),
).rejects.toThrow("cancelled");
expect(spawn).not.toHaveBeenCalled();
});
it("kills an actual CPU-bound process, not just its request promise", async () => {
const controller = new AbortController();
let pid: number | undefined;
const parser = createNativeParserIsolation({
timeoutMs: 5_000,
spawn: () => {
const child = fork(new URL("./native-parser-busy.fixture.mjs", import.meta.url), [], {
execArgv: [],
stdio: ["ignore", "ignore", "ignore", "ipc"],
});
pid = child.pid;
child.once("message", () => controller.abort(new Error("actual CPU cancellation")));
return child;
},
}).wrap(createNativeStructuredDataParser(), {});
await expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow(
"actual CPU cancellation",
);
expect(pid).toBeTypeOf("number");
expect(() => process.kill(pid as number, 0)).toThrow();
});
it("bounds aggregate input retention before joining the queue", async () => {
const child = childFixture();
const spawn = vi.fn(() => child);
const parser = createNativeParserIsolation({
spawn,
maxReservedBytes: input.body.byteLength,
}).wrap(createNativeStructuredDataParser(), {});
const controller = new AbortController();
const first = expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow(
"done",
);
await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce());
await expect(parser.parse(input)).rejects.toMatchObject({
code: "provider_rate_limited",
retryable: true,
});
controller.abort(new Error("done"));
await first;
});
it("bounds queue length and frees the slot on process failure", async () => {
const child = childFixture();
const spawn = vi.fn(() => child);
const parser = createNativeParserIsolation({ spawn, maxConcurrency: 1, maxQueued: 1 }).wrap(
createNativeStructuredDataParser(),
{},
);
const first = expect(parser.parse(input)).rejects.toMatchObject({
code: "provider_response_invalid",
});
await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce());
const controller = new AbortController();
const second = expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow(
"queue cancellation",
);
await expect(parser.parse(input)).rejects.toMatchObject({ code: "provider_rate_limited" });
controller.abort(new Error("queue cancellation"));
await second;
child.emit("close", 1, null);
await first;
});
it("preserves typed format errors and rejects invalid worker budgets", async () => {
for (const maxConcurrency of [0, Number.NaN, 1.1, Number.POSITIVE_INFINITY])
expect(() => createNativeParserIsolation({ maxConcurrency })).toThrow("Invalid");
const child = childFixture();
const parser = createNativeParserIsolation({ spawn: () => child }).wrap(
createNativeStructuredDataParser(),
{},
);
const result = expect(parser.parse(input)).rejects.toMatchObject({
code: "document_parser_unsupported_type",
});
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
child.emit("message", {
ok: false,
message: "unsupported",
errorCode: "document_parser_unsupported_type",
});
child.emit("close", 0, null);
await result;
});
it.each(["event", "callback", "throw", "input"])(
"cleans up %s worker failures without leaking admission",
async (mode) => {
const child = childFixture();
if (mode === "callback" || mode === "throw") {
child.send = vi.fn((_message, callback) => {
if (mode === "throw") throw new Error("IPC failed");
callback(new Error("IPC failed"));
return false;
}) as unknown as ChildProcess["send"];
}
const parser = createNativeParserIsolation({ spawn: () => child }).wrap(
createNativeStructuredDataParser(),
{},
);
const result = expect(parser.parse(input)).rejects.toThrow(
mode === "input" ? "bad input" : "IPC failed",
);
if (mode === "event" || mode === "input") {
await vi.waitFor(() => expect(child.send).toHaveBeenCalledOnce());
if (mode === "event") child.emit("error", new Error("IPC failed"));
else {
child.emit("message", { ok: false, message: "bad input", errorCode: "provider_input" });
child.emit("close", 0, null);
}
}
await result;
},
);
it("advances queued work only after the preceding process closes", async () => {
const first = childFixture();
const second = childFixture();
const spawn = vi.fn().mockReturnValueOnce(first).mockReturnValueOnce(second);
const parser = createNativeParserIsolation({ spawn, maxConcurrency: 1 }).wrap(
createNativeStructuredDataParser(),
{},
);
const failed = expect(parser.parse(input)).rejects.toMatchObject({
code: "provider_response_invalid",
});
const artifact = await createNativeStructuredDataParser().parse(input);
const next = parser.parse(input);
await vi.waitFor(() => expect(spawn).toHaveBeenCalledOnce());
first.emit("close", 1, null);
await failed;
await vi.waitFor(() => expect(spawn).toHaveBeenCalledTimes(2));
second.emit("message", { ok: true, artifact });
second.emit("close", 0, null);
expect(await next).toEqual(artifact);
});
it("handles cancellation during spawn and rejects accidental remote adapters", async () => {
expect(() =>
createNativeParserIsolation().wrap({ kind: "unstructured", parse: vi.fn() }, {}),
).toThrow("remote");
const controller = new AbortController();
const child = childFixture();
const parser = createNativeParserIsolation({
spawn: () => {
controller.abort(new Error("spawn cancelled"));
return child;
},
}).wrap(createNativeStructuredDataParser(), {});
await expect(parser.parse({ ...input, signal: controller.signal })).rejects.toThrow(
"spawn cancelled",
);
expect(child.send).not.toHaveBeenCalled();
});
});

73
knowledge-fs/apps/api/src/native-parser-isolation.ts Normal file
View File

@ -0,0 +1,73 @@
import { type ChildProcess, fork } from "node:child_process";
import {
type ParserAdapter,
ProviderInputError,
ProviderResponseError,
ProviderUnsupportedFileTypeError,
} from "@knowledge/parsers";
import {
type IsolatedProcessOptions,
createIsolatedProcessExecutor,
} from "./isolated-process-executor";
import {
type NativeParserRequest,
type SerializableNativeParserOptions,
decodeNativeParserResponse,
} from "./native-parser-protocol";
function spawnNativeParser(): ChildProcess {
const compiled = import.meta.url.endsWith(".mjs");
return fork(
new URL(compiled ? "./native-parser-worker.mjs" : "./native-parser-worker.ts", import.meta.url),
[],
{
execArgv: ["--max-old-space-size=256", ...(compiled ? [] : ["--import", "tsx"])],
// Native parsers need neither provider credentials nor arbitrary NODE_OPTIONS hooks.
env: { PATH: process.env.PATH, SYSTEMROOT: process.env.SYSTEMROOT },
serialization: "advanced",
stdio: ["ignore", "ignore", "ignore", "ipc"],
},
);
}
export function createNativeParserIsolation(
options: Omit<IsolatedProcessOptions, "spawn"> & { readonly spawn?: () => ChildProcess } = {},
) {
const executor = createIsolatedProcessExecutor<NativeParserRequest, unknown>({
maxRssBytes: 512 * 1024 * 1024,
...options,
spawn: options.spawn ?? spawnNativeParser,
});
return {
wrap(parser: ParserAdapter, options: SerializableNativeParserOptions): ParserAdapter {
if (parser.kind === "unstructured")
throw new Error("Native isolation cannot wrap a remote parser");
if (Object.values(options).some((value) => typeof value === "function"))
throw new Error(
"Native isolation options must be serializable (callbacks are not supported)",
);
const kind = parser.kind;
const frozenOptions = Object.freeze({ ...options });
return {
...parser,
async parse(input) {
const { signal, ...serializedInput } = input;
const payload = await executor.execute(
{ input: serializedInput, kind, options: frozenOptions },
input.body.byteLength,
signal,
);
const response = decodeNativeParserResponse(payload);
if (!response)
throw new ProviderResponseError("Native parser worker returned an invalid response");
if (response.ok) return response.artifact;
throw response.errorCode === "document_parser_unsupported_type"
? new ProviderUnsupportedFileTypeError(response.message)
: response.errorCode === "provider_input"
? new ProviderInputError(response.message)
: new ProviderResponseError(response.message);
},
};
},
};
}

22
knowledge-fs/apps/api/src/native-parser-protocol.test.ts Normal file
View File

@ -0,0 +1,22 @@
import { ProviderInputError, ProviderUnsupportedFileTypeError } from "@knowledge/parsers";
import { describe, expect, it } from "vitest";
import { decodeNativeParserResponse, encodeNativeParserFailure } from "./native-parser-protocol";
describe("native parser failure protocol", () => {
it.each([
[new ProviderInputError("invalid source"), "provider_input"],
[new ProviderUnsupportedFileTypeError("unsupported"), "document_parser_unsupported_type"],
[new Error("unexpected implementation failure"), "provider_response_invalid"],
[null, "provider_response_invalid"],
])("preserves only known failure classifications %#", (error, errorCode) => {
const result = encodeNativeParserFailure(error);
expect(result).toMatchObject({ ok: false, errorCode });
expect(decodeNativeParserResponse(result)).toEqual(result);
});
it("bounds error text and rejects unrecognized failure codes", () => {
expect(encodeNativeParserFailure(new Error("x".repeat(2000))).message).toHaveLength(1024);
expect(
decodeNativeParserResponse({ ok: false, message: "x", errorCode: "unknown" }),
).toBeUndefined();
});
});

68
knowledge-fs/apps/api/src/native-parser-protocol.ts Normal file
View File

@ -0,0 +1,68 @@
import { type ParseArtifact, ParseArtifactSchema } from "@knowledge/core";
import type {
ParseDocumentInput,
ParserKind,
StructuredDataParserOptions,
} from "@knowledge/parsers";
export interface NativeParserRequest {
readonly input: Omit<ParseDocumentInput, "signal">;
readonly kind: Exclude<ParserKind, "unstructured">;
readonly options: SerializableNativeParserOptions;
}
export type SerializableNativeParserOptions = Omit<
StructuredDataParserOptions,
"generateId" | "now"
>;
export type NativeParserResponse =
| { readonly ok: true; readonly artifact: ParseArtifact }
| {
readonly ok: false;
readonly message: string;
readonly errorCode:
| "provider_input"
| "document_parser_unsupported_type"
| "provider_response_invalid";
};
export function encodeNativeParserFailure(
error: unknown,
): Extract<NativeParserResponse, { ok: false }> {
const code =
typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
return {
ok: false,
message:
error instanceof Error ? error.message.slice(0, 1024) : "Native document parsing failed",
errorCode:
code === "provider_input" || code === "document_parser_unsupported_type"
? code
: "provider_response_invalid",
};
}
export function decodeNativeParserResponse(value: unknown): NativeParserResponse | undefined {
if (typeof value !== "object" || value === null || !("ok" in value)) return undefined;
if (value.ok === true && "artifact" in value) {
const parsed = ParseArtifactSchema.safeParse(value.artifact);
return parsed.success ? { ok: true, artifact: parsed.data } : undefined;
}
if (
value.ok === false &&
"message" in value &&
typeof value.message === "string" &&
value.message.length <= 1024 &&
"errorCode" in value &&
(value.errorCode === "provider_input" ||
value.errorCode === "document_parser_unsupported_type" ||
value.errorCode === "provider_response_invalid")
) {
return { ok: false, message: value.message, errorCode: value.errorCode };
}
return undefined;
}
// Serialized output is bounded independently from the child's V8 heap ceiling.
export const nativeParserOutputBytes = 32 * 1024 * 1024;

48
knowledge-fs/apps/api/src/native-parser-worker.ts Normal file
View File

@ -0,0 +1,48 @@
import {
createNativeHtmlParser,
createNativeMarkdownParser,
createNativeStructuredDataParser,
} from "@knowledge/parsers";
import {
type NativeParserRequest,
type NativeParserResponse,
encodeNativeParserFailure,
nativeParserOutputBytes,
} from "./native-parser-protocol";
// A single immutable request per process. No providers, credentials or document filesystem IO.
process.once("message", async (request: NativeParserRequest) => {
const started = performance.now();
let response: NativeParserResponse;
try {
const parser =
request.kind === "native-html"
? createNativeHtmlParser(request.options)
: request.kind === "native-markdown"
? createNativeMarkdownParser(request.options)
: createNativeStructuredDataParser(request.options);
const artifact = await parser.parse(request.input);
const outputBytes = Buffer.byteLength(JSON.stringify(artifact));
if (outputBytes > nativeParserOutputBytes)
throw new Error("Native parser output exceeds 32 MiB");
response = {
ok: true,
artifact: {
...artifact,
metadata: {
...artifact.metadata,
parserExecution: {
isolation: "child-process",
inputBytes: request.input.body.byteLength,
outputBytes,
wallMs: performance.now() - started,
peakRssKiB: process.resourceUsage().maxRSS,
},
},
},
};
} catch (error) {
response = encodeNativeParserFailure(error);
}
process.send?.(response, () => process.disconnect());
});

136
knowledge-fs/apps/api/src/parser-options.test.ts
View File

@ -1,4 +1,6 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import { ProviderInputError } from "@knowledge/parsers";
import type { Dispatcher } from "undici";
@ -7,40 +9,109 @@ import {
createApiUnstructuredConcurrencyOptions,
createNodeUnstructuredFetch,
} from "./parser-options";
import { createPdfParserPreflight } from "./pdf-parser-preflight";
// Parser routing/timeout tests use tiny transport fixtures. Real PDF inspection, child process
// limits and page geometry are exercised independently in pdf-parser-preflight.test.ts.
vi.mock("./pdf-parser-preflight", () => ({
createPdfParserPreflight: vi.fn(() => ({
check: vi.fn(async () => {}),
policyFingerprint: "pdf-preflight-test",
})),
}));
const encoder = new TextEncoder();
function ordinaryDocx(): Uint8Array {
const filename = encoder.encode("word/document.xml");
const localHeader = new Uint8Array(30);
const centralDirectory = new Uint8Array(46 + filename.byteLength);
const endOfCentralDirectory = new Uint8Array(22);
const localView = new DataView(localHeader.buffer);
const centralView = new DataView(centralDirectory.buffer);
const endView = new DataView(endOfCentralDirectory.buffer);
localView.setUint32(0, 0x04034b50, true);
centralView.setUint32(0, 0x02014b50, true);
centralView.setUint32(20, 1, true);
centralView.setUint32(24, 1, true);
centralView.setUint16(28, filename.byteLength, true);
centralDirectory.set(filename, 46);
endView.setUint32(0, 0x06054b50, true);
endView.setUint16(8, 1, true);
endView.setUint16(10, 1, true);
endView.setUint32(12, centralDirectory.byteLength, true);
endView.setUint32(16, localHeader.byteLength, true);
const body = new Uint8Array(
localHeader.byteLength + centralDirectory.byteLength + endOfCentralDirectory.byteLength,
// Real deflated OOXML ZIP, generated with fflate.zipSync (fixed mtime), containing
// [Content_Types].xml, _rels/.rels and a one-paragraph word/document.xml.
return new Uint8Array(
Buffer.from(
"UEsDBBQAAAAIAACYn090JJxTuwAAAD4BAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbJWQuQ7CMAyGX6XKiqgRAwNquwArMPACVuq2EbkUm+vtSTk6sDHa//FZrk6PSFzcnfVcq0EkrgFYD+SQyxDJZ6ULyaHkMfUQUZ+xJ1guFivQwQt5mcvYoZpqSx1erBS7e16zCb5WiSyrYvM2jqxaYYzWaJSsw9W3P5T5h1Dm5MvDg4k8ywYFTXW4UkqmpeKISfboch3cQmqhDfriMqIcjX/xQtcZTVN+bIspaGI2vne2nBSHxn/vgNfbmidQSwMEFAAAAAgAAJifT2F7L0OIAAAA8gAAAAsAAABfcmVscy8ucmVsc43POQ7CMBAF0KtEPkAmUFCg2BVNWsQFLHu8iHjReBBwe1xQEERBOYve15/PuGqOJbcQaxseac1NisBcjwDNBEy6jaVi7hdXKGnuI3mo2ly1R9hP0wHo0xBqYw6LlYIWuxPD5VnxH7s4Fw2eirklzPwj4uujy5o8shT3Qhbsez12VoCaYVNRvQBQSwMEFAAAAAgAAJifT/fkxAV3AAAAowAAABEAAAB3b3JkL2RvY3VtZW50LnhtbDWNXQ6DIAyAr2I8wGr2sAfiuMLOwIApiW1JYUFvb4nx5evf13ZuJrD/Y6Q67LhRMe09rrVmA1D8GtGVB+dIOvuxoKtaygKNJWRhH0tJtOAGz2l6AbpEo9WTXw5Hj7lDOqr9SEjk5BjufzP0dqcaSpWV16omt2ZPUEsBAhQAFAAAAAgAAJifT3QknFO7AAAAPgEAABMAAAAAAAAAAAAAAAAAAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECFAAUAAAACAAAmJ9PYXsvQ4gAAADyAAAACwAAAAAAAAAAAAAAAADsAAAAX3JlbHMvLnJlbHNQSwECFAAUAAAACAAAmJ9P9+TEBXcAAACjAAAAEQAAAAAAAAAAAAAAAACdAQAAd29yZC9kb2N1bWVudC54bWxQSwUGAAAAAAMAAwC5AAAAQwIAAAAA",
"base64",
),
);
body.set(localHeader);
body.set(centralDirectory, localHeader.byteLength);
body.set(endOfCentralDirectory, localHeader.byteLength + centralDirectory.byteLength);
return body;
}
describe("createApiDocumentParser", () => {
it("includes the configured provider semantic revision in checkpoint identity", () => {
const input = {
body: ordinaryDocx(),
documentAssetId: "00000000-0000-4000-8000-000000000001",
filename: "a.docx",
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
version: 1,
};
const make = (revision: string) =>
createApiDocumentParser({
env: {
UNSTRUCTURED_API_URL: "https://parser.example.test",
UNSTRUCTURED_BACKEND_REVISION: revision,
},
});
expect(make("parser-policy-v1").policyFingerprint?.(input)).not.toBe(
make("parser-policy-v2").policyFingerprint?.(input),
);
});
it("keeps a structured upload above 10 MiB on the native parser within the 15 MiB admission limit", async () => {
const fetch = vi.fn(async () => new Response("[]"));
const parser = createApiDocumentParser({
env: { UNSTRUCTURED_API_URL: "https://unstructured.example.test" },
fetch,
});
const artifact = await parser.parse({
body: encoder.encode(`${" ".repeat(10 * 1024 * 1024)}{"id":9007199254740993}`),
documentAssetId: "00000000-0000-4000-8000-000000000001",
filename: "large.json",
mimeType: "application/json",
version: 1,
});
expect(artifact.metadata.routedParser).toBe("native-structured");
expect(artifact.metadata.parserExecution).toMatchObject({ isolation: "child-process" });
expect(artifact.elements[0]?.text).toContain("9007199254740993");
expect(fetch).not.toHaveBeenCalled();
});
it("applies a configured input limit to native structured parsing even without a remote endpoint", async () => {
const parser = createApiDocumentParser({ env: { UNSTRUCTURED_MAX_INPUT_BYTES: "8" } });
await expect(
parser.parse({
body: encoder.encode('{"value":123}'),
documentAssetId: "00000000-0000-4000-8000-000000000001",
filename: "data.json",
mimeType: "application/json",
version: 1,
}),
).rejects.toThrow("maxInputBytes=8");
});
it("always runs PDF safety inspection before the configured remote transport", async () => {
const check = vi.fn(async () => {
throw new ProviderInputError("PDF page 1 exceeds the raster pixel budget");
});
vi.mocked(createPdfParserPreflight).mockReturnValueOnce({
check,
policyFingerprint: "pdf-preflight-test",
});
const fetch = vi.fn(async () => new Response("[]"));
const parser = createApiDocumentParser({
env: { UNSTRUCTURED_API_URL: "https://unstructured.example.test" },
fetch,
});
await expect(
parser.parse({
body: encoder.encode("%PDF-1.7"),
documentAssetId: "00000000-0000-4000-8000-000000000001",
filename: "80x180cm.pdf",
mimeType: "application/pdf",
version: 1,
}),
).rejects.toMatchObject({ code: "provider_input", retryable: false });
expect(check).toHaveBeenCalledOnce();
expect(fetch).not.toHaveBeenCalled();
});
it("resolves parser lane widths and preserves the legacy heavy alias", () => {
expect(createApiUnstructuredConcurrencyOptions({})).toEqual({
heavyMaxConcurrency: 2,
@ -136,7 +207,7 @@ describe("createApiDocumentParser", () => {
expect(fetchCalls).toBe(0);
});
it("routes an admitted 11 MiB structured upload to the remote parser", async () => {
it("preserves native CSV semantics for an admitted 11 MiB structured upload", async () => {
let fetchCalls = 0;
const parser = createApiDocumentParser({
env: { UNSTRUCTURED_API_URL: "https://unstructured.example.test" },
@ -147,18 +218,19 @@ describe("createApiDocumentParser", () => {
});
const artifact = await parser.parse({
body: new Uint8Array(11 * 1024 * 1024),
body: encoder.encode(`${"\n".repeat(11 * 1024 * 1024)}name\nAda`),
documentAssetId: "00000000-0000-4000-8000-000000000020",
filename: "large.csv",
mimeType: "text/csv",
version: 1,
});
expect(fetchCalls).toBe(1);
expect(fetchCalls).toBe(0);
expect(artifact).toMatchObject({
metadata: { routeReason: "native-size-limit", routedParser: "unstructured" },
parser: "unstructured",
metadata: { routeReason: "structured-file-type", routedParser: "native-structured" },
parser: "native-structured",
});
expect(artifact.elements[0]?.text).toBe("name: Ada");
});
it("routes complex documents to the configured Unstructured API", async () => {

36
knowledge-fs/apps/api/src/parser-options.ts
View File

@ -8,6 +8,9 @@ import {
} from "@knowledge/parsers";
import { Agent, type Dispatcher, fetch as undiciFetch } from "undici";
import { createNativeParserIsolation } from "./native-parser-isolation";
import { createPdfParserPreflight } from "./pdf-parser-preflight";
const defaultUnstructuredRequestTimeoutMs = 600_000;
const defaultUnstructuredMaxConcurrency = 2;
const defaultUnstructuredMaxInputBytes = 15 * 1024 * 1024;
@ -34,6 +37,7 @@ export interface ApiParserEnv {
readonly NODE_ENV?: string | undefined;
readonly UNSTRUCTURED_API_KEY?: string | undefined;
readonly UNSTRUCTURED_API_URL?: string | undefined;
readonly UNSTRUCTURED_BACKEND_REVISION?: string | undefined;
readonly UNSTRUCTURED_DEFAULT_LANGUAGE?: string | undefined;
readonly UNSTRUCTURED_HEAVY_MAX_CONCURRENCY?: string | undefined;
readonly UNSTRUCTURED_HEAVY_REQUEST_TIMEOUT_MS?: string | undefined;
@ -100,11 +104,14 @@ export function createApiDocumentParser({
env,
...(fetchImpl ? { fetch: fetchImpl } : {}),
});
const nativeOptions = Object.freeze({ maxInputBytes: resolveParserMaxInputBytes(env) });
const isolation = createNativeParserIsolation({ maxInputBytes: nativeOptions.maxInputBytes });
return createParserRouter({
html: createNativeHtmlParser(),
markdown: createNativeMarkdownParser(),
structured: createNativeStructuredDataParser(),
html: isolation.wrap(createNativeHtmlParser(nativeOptions), nativeOptions),
markdown: isolation.wrap(createNativeMarkdownParser(nativeOptions), nativeOptions),
structured: isolation.wrap(createNativeStructuredDataParser(nativeOptions), nativeOptions),
maxNativeInputBytes: nativeOptions.maxInputBytes,
unstructured,
});
}
@ -157,6 +164,10 @@ function createApiUnstructuredParser({
const concurrency = createApiUnstructuredConcurrencyOptions(env);
return createUnstructuredParserClient({
...(env.UNSTRUCTURED_BACKEND_REVISION?.trim()
? { backendRevision: env.UNSTRUCTURED_BACKEND_REVISION.trim() }
: {}),
requestPreflight: createPdfParserPreflight(),
...(env.UNSTRUCTURED_DEFAULT_LANGUAGE?.trim()
? { defaultLanguage: env.UNSTRUCTURED_DEFAULT_LANGUAGE.trim() }
: {}),
@ -179,14 +190,7 @@ function createApiUnstructuredParser({
),
}
: {}),
maxInputBytes:
env.UNSTRUCTURED_MAX_INPUT_BYTES === undefined
? defaultUnstructuredMaxInputBytes
: parseBoundedPositiveInteger(
env.UNSTRUCTURED_MAX_INPUT_BYTES,
"UNSTRUCTURED_MAX_INPUT_BYTES",
maxUnstructuredInputBytes,
),
maxInputBytes: resolveParserMaxInputBytes(env),
...(env.UNSTRUCTURED_MAX_RETRIES !== undefined
? {
maxRetries: parseNonNegativeInteger(
@ -207,6 +211,16 @@ function createApiUnstructuredParser({
});
}
function resolveParserMaxInputBytes(env: ApiParserEnv): number {
return env.UNSTRUCTURED_MAX_INPUT_BYTES === undefined
? defaultUnstructuredMaxInputBytes
: parseBoundedPositiveInteger(
env.UNSTRUCTURED_MAX_INPUT_BYTES,
"UNSTRUCTURED_MAX_INPUT_BYTES",
maxUnstructuredInputBytes,
);
}
export function createNodeUnstructuredFetch({
createDispatcher = (options) => new Agent(options),
fetch: fetchImpl = undiciFetch as unknown as typeof fetch,

344
knowledge-fs/apps/api/src/pdf-parser-preflight.test.ts Normal file
View File

@ -0,0 +1,344 @@
import { execFile } from "node:child_process";
import { mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { type ParseDocumentInput, ProviderInputError } from "@knowledge/parsers";
import { afterEach, describe, expect, it } from "vitest";
import { createPdfParserPreflight } from "./pdf-parser-preflight";
const directories: string[] = [];
const pdfInput = (overrides: Partial<ParseDocumentInput> = {}): ParseDocumentInput => ({
body: new TextEncoder().encode("%PDF-1.7\n"),
documentAssetId: "asset-pdf",
filename: "document.pdf",
mimeType: "application/pdf",
version: 1,
...overrides,
});
function pageInfo(width: number, height: number, page = 1): string {
return `Page ${page} size: ${width} x ${height} pts\nPage ${page} MediaBox: 0 0 ${width} ${height}\nPage ${page} CropBox: 0 0 ${width} ${height}\n`;
}
async function temporaryDirectory(): Promise<string> {
const directory = await mkdtemp(join(tmpdir(), "pdf-preflight-test-"));
directories.push(directory);
return directory;
}
async function executableFixture(
source: string,
): Promise<{ directory: string; executable: string }> {
const directory = await temporaryDirectory();
const executable = join(directory, "pdfinfo-fixture");
await writeFile(executable, `#!${process.execPath}\n${source}`, { mode: 0o700 });
return { directory, executable };
}
async function waitForFile(path: string): Promise<string> {
for (let attempt = 0; attempt < 200; attempt += 1) {
try {
return await readFile(path, "utf8");
} catch {
await new Promise((resolve) => setTimeout(resolve, 5));
}
}
throw new Error("Fixture child did not start");
}
function syntheticPdf({
pages,
inherited = false,
}: { pages: readonly [number, number][]; inherited?: boolean }): Uint8Array {
const firstPage = pages[0] ?? [595, 842];
const box = ([width, height]: readonly [number, number]) =>
`/MediaBox [0 0 ${width} ${height}] /CropBox [0 0 ${width} ${height}]`;
const objects = [
"<< /Type /Catalog /Pages 2 0 R >>",
`<< /Type /Pages /Kids [${pages.map((_, index) => `${index + 3} 0 R`).join(" ")}] /Count ${pages.length} ${inherited ? box(firstPage) : ""} >>`,
...pages.map((page) => `<< /Type /Page /Parent 2 0 R ${inherited ? "" : box(page)} >>`),
];
let body = "%PDF-1.7\n";
const offsets = [0];
for (const [index, object] of objects.entries()) {
offsets.push(Buffer.byteLength(body));
body += `${index + 1} 0 obj\n${object}\nendobj\n`;
}
const xref = Buffer.byteLength(body);
body += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
body += offsets
.slice(1)
.map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`)
.join("");
body += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
return new TextEncoder().encode(body);
}
afterEach(async () => {
await Promise.all(
directories.splice(0).map((directory) => rm(directory, { force: true, recursive: true })),
);
});
describe("PDF raster preflight", () => {
it.each([
[595.28, 841.89],
[841.89, 1190.55],
])("admits ordinary %s by %s point pages", async (width, height) => {
const preflight = createPdfParserPreflight({
executePdfinfo: async () => `Pages: 1\n${pageInfo(width, height)}`,
});
await expect(preflight.check(pdfInput())).resolves.toBeUndefined();
});
it("rejects an 80 by 180 cm banner despite its tiny file body", async () => {
const preflight = createPdfParserPreflight({
executePdfinfo: async () => `Pages: 1\n${pageInfo(2267.72, 5102.36)}`,
});
await expect(preflight.check(pdfInput())).rejects.toMatchObject({
code: "provider_input",
retryable: false,
message: expect.stringContaining("page 1"),
});
});
it("checks every page, including an oversized page after a normal first page", async () => {
const preflight = createPdfParserPreflight({
executePdfinfo: async () => `Pages: 2\n${pageInfo(595, 842)}${pageInfo(2267, 5102, 2)}`,
});
await expect(preflight.check(pdfInput())).rejects.toThrow("page 2");
});
it("uses the conservative MediaBox even when page size and CropBox are small", async () => {
const output = `Pages: 1\n${pageInfo(595, 842).replace("MediaBox: 0 0 595 842", "MediaBox: -1000 0 2267 5102")}`;
await expect(
createPdfParserPreflight({ executePdfinfo: async () => output }).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
});
it("also bounds single-edge length when the total pixel area is small", async () => {
await expect(
createPdfParserPreflight({
executePdfinfo: async () => `Pages: 1\n${pageInfo(3000, 1)}`,
}).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
});
it("bounds total pixel area even when both individual edges fit", async () => {
await expect(
createPdfParserPreflight({
executePdfinfo: async () => `Pages: 1\n${pageInfo(1190.55, 1683.78)}`,
}).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
});
it("checks the CropBox independently of the page size and MediaBox", async () => {
const output = `Pages: 1\n${pageInfo(595, 842).replace("CropBox: 0 0 595 842", "CropBox: 0 0 2267 5102")}`;
await expect(
createPdfParserPreflight({ executePdfinfo: async () => output }).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
});
it.each([
"Pages: 0\n",
"Pages: 10001\n",
`Pages: 1\nPages: 1\n${pageInfo(595, 842)}`,
`Pages: 2\n${pageInfo(595, 842)}`,
`Pages: 1\n${pageInfo(595, 842)}${pageInfo(595, 842)}`,
`Pages: 1\n${pageInfo(595, 842, 2)}`,
`Pages: 1\n${pageInfo(595, 842).replace(/Page 1 CropBox:.*\n/, "")}`,
`Pages: 1\n${pageInfo(Number.NaN, 842)}`,
`Pages: 1\n${pageInfo(Number.POSITIVE_INFINITY, 842)}`,
`Pages: 1\n${pageInfo(-595, 842)}`,
`Pages: 1\n${pageInfo(0, 842)}`,
`Pages: 1\n${pageInfo(595, 842).replace("size: 595 x 842 pts", "size: invalid")}`,
`Pages: 1\n${pageInfo(595, 842).replace("CropBox: 0 0 595 842", "CropBox: 0 0 595")}`,
`Pages: 1\n${pageInfo(595, 842).replace("MediaBox: 0 0 595 842", "MediaBox: 10 0 0 842")}`,
"Encrypted: yes\n",
])("fails closed for incomplete, duplicate or invalid geometry: %s", async (output) => {
await expect(
createPdfParserPreflight({ executePdfinfo: async () => output }).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
});
it.each([
{ filename: "document.bin", mimeType: "application/pdf", body: new Uint8Array() },
{ filename: "DOCUMENT.PDF", mimeType: "application/octet-stream", body: new Uint8Array() },
{
filename: "document.docx",
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
body: new TextEncoder().encode("garbage\n%PDF-1.7\n"),
},
])("applies the PDF guard if any identity signal identifies a PDF", async (identity) => {
await expect(
createPdfParserPreflight({
executePdfinfo: async () => `Pages: 1\n${pageInfo(2267, 5102)}`,
}).check(pdfInput(identity)),
).rejects.toBeInstanceOf(ProviderInputError);
});
it("performs no filesystem or subprocess I/O for non-PDF documents", async () => {
const preflight = createPdfParserPreflight({
temporaryDirectory: "/does-not-exist/pdf-preflight",
executePdfinfo: async () => {
throw new Error("must not execute");
},
});
await expect(
preflight.check(
pdfInput({
filename: "document.docx",
mimeType: "application/octet-stream",
body: new TextEncoder().encode("PK"),
}),
),
).resolves.toBeUndefined();
});
it.each([{ timeoutMs: 0 }, { timeoutMs: Number.NaN }, { maxOutputBytes: -1 }])(
"refuses invalid inspection limits: %s",
(options) => {
expect(() => createPdfParserPreflight(options)).toThrow("must be a positive safe integer");
},
);
it("writes the input privately and cleans it up after successful checking", async () => {
const directory = await temporaryDirectory();
const input = pdfInput();
const preflight = createPdfParserPreflight({
temporaryDirectory: directory,
executePdfinfo: async ({ path }) => {
expect((await stat(path)).mode & 0o777).toBe(0o600);
expect(await readFile(path)).toEqual(Buffer.from(input.body));
return `Pages: 1\n${pageInfo(595, 842)}`;
},
});
await preflight.check(input);
expect(await readdir(directory)).toEqual([]);
});
it("cleans up its private temporary input when geometry is rejected", async () => {
const directory = await temporaryDirectory();
await expect(
createPdfParserPreflight({
temporaryDirectory: directory,
executePdfinfo: async () => "invalid",
}).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
expect(await readdir(directory)).toEqual([]);
});
it("fails clearly when pdfinfo is missing, rather than silently bypassing the guard", async () => {
const directory = await temporaryDirectory();
await expect(
createPdfParserPreflight({
temporaryDirectory: directory,
pdfinfoExecutable: join(directory, "missing"),
}).check(pdfInput()),
).rejects.toThrow("requires the pdfinfo executable");
expect(await readdir(directory)).toEqual([]);
});
it("preserves an already-aborted parent reason without doing I/O", async () => {
const controller = new AbortController();
const reason = new Error("lease expired");
controller.abort(reason);
await expect(
createPdfParserPreflight({ temporaryDirectory: "/does-not-exist/pdf-preflight" }).check(
pdfInput({ signal: controller.signal }),
),
).rejects.toBe(reason);
});
it("kills the actual subprocess and cleans up when the parent is cancelled", async () => {
const { directory, executable } = await executableFixture(
`const fs = require('node:fs');\nfs.writeFileSync(__dirname + '/child.pid', String(process.pid));\nsetInterval(() => {}, 1000);`,
);
const controller = new AbortController();
const reason = new Error("lease lost");
const result = createPdfParserPreflight({
temporaryDirectory: directory,
pdfinfoExecutable: executable,
}).check(pdfInput({ signal: controller.signal }));
const rejected = expect(result).rejects.toBe(reason);
const pid = Number(await waitForFile(join(directory, "child.pid")));
controller.abort(reason);
await rejected;
expect(() => process.kill(pid, 0)).toThrow();
expect((await readdir(directory)).sort()).toEqual(["child.pid", "pdfinfo-fixture"]);
});
it("kills timed-out subprocesses even when they ignore SIGTERM, then cleans up", async () => {
const { directory, executable } = await executableFixture(
"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);",
);
await expect(
createPdfParserPreflight({
temporaryDirectory: directory,
pdfinfoExecutable: executable,
timeoutMs: 100,
}).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
expect(await readdir(directory)).toEqual(["pdfinfo-fixture"]);
});
it("bounds subprocess stdout and cleans up after output overflow", async () => {
const { directory, executable } = await executableFixture(
"process.stdout.write('a'.repeat(4096)); setInterval(() => {}, 1000);",
);
await expect(
createPdfParserPreflight({
temporaryDirectory: directory,
pdfinfoExecutable: executable,
maxOutputBytes: 64,
}).check(pdfInput()),
).rejects.toBeInstanceOf(ProviderInputError);
expect(await readdir(directory)).toEqual(["pdfinfo-fixture"]);
});
it("rejects a real subprocess parse failure without exposing its stderr", async () => {
const { directory, executable } = await executableFixture(
"process.stderr.write('private-document-text'); process.exit(1);",
);
const result = createPdfParserPreflight({
temporaryDirectory: directory,
pdfinfoExecutable: executable,
}).check(pdfInput());
await expect(result).rejects.toThrow("Unable to safely inspect PDF");
await expect(result).rejects.not.toThrow("private-document-text");
expect(await readdir(directory)).toEqual(["pdfinfo-fixture"]);
});
it("uses real Poppler to resolve inherited boxes and detect later oversized pages", async () => {
await promisify(execFile)("pdfinfo", ["-v"]);
const preflight = createPdfParserPreflight();
await expect(
preflight.check(
pdfInput({
body: syntheticPdf({
pages: [
[595, 842],
[595, 842],
],
inherited: true,
}),
}),
),
).resolves.toBeUndefined();
await expect(
preflight.check(
pdfInput({
body: syntheticPdf({
pages: [
[595, 842],
[2267, 5102],
],
}),
}),
),
).rejects.toThrow("page 2");
});
});

251
knowledge-fs/apps/api/src/pdf-parser-preflight.ts Normal file
View File

@ -0,0 +1,251 @@
import { execFile } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { type ParseDocumentInput, ProviderInputError } from "@knowledge/parsers";
const RASTER_DPI = 350;
const MAX_PAGE_PIXELS = 25_000_000;
const MAX_PAGE_EDGE_PIXELS = 10_000;
const MAX_PAGES = 10_000;
const DEFAULT_TIMEOUT_MS = 10_000;
const DEFAULT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
interface PdfinfoExecutionInput {
readonly path: string;
readonly signal?: AbortSignal;
}
interface PdfParserPreflightOptions {
/** Substitute only the bounded subprocess boundary; useful for deterministic geometry tests. */
readonly executePdfinfo?: (input: PdfinfoExecutionInput) => Promise<string>;
readonly maxOutputBytes?: number;
readonly pdfinfoExecutable?: string;
readonly temporaryDirectory?: string;
readonly timeoutMs?: number;
}
export interface PdfParserPreflight {
readonly policyFingerprint: string;
check(input: ParseDocumentInput): Promise<void>;
}
/**
* Metadata-only admission prevents huge pages from reaching rasterization. Poppler resolves
* inherited boxes without decoding page images. The provider's own pre-render pixel guard is
* still required: PDF engines can disagree on malformed objects and features such as UserUnit.
*/
export function createPdfParserPreflight(
options: PdfParserPreflightOptions = {},
): PdfParserPreflight {
const timeoutMs = positiveInteger(options.timeoutMs ?? DEFAULT_TIMEOUT_MS, "timeoutMs");
const maxOutputBytes = positiveInteger(
options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES,
"maxOutputBytes",
);
const executePdfinfo =
options.executePdfinfo ??
((input: PdfinfoExecutionInput) =>
runPdfinfo(input, {
executable: options.pdfinfoExecutable ?? "pdfinfo",
maxOutputBytes,
timeoutMs,
}));
return {
policyFingerprint: `pdf-raster-preflight-v1:dpi=${RASTER_DPI}:pixels=${MAX_PAGE_PIXELS}:edge=${MAX_PAGE_EDGE_PIXELS}:pages=${MAX_PAGES}`,
async check(input) {
if (!isPdf(input)) {
return;
}
input.signal?.throwIfAborted();
const directory = await mkdtemp(
join(options.temporaryDirectory ?? tmpdir(), "knowledge-fs-pdf-"),
);
try {
input.signal?.throwIfAborted();
const path = join(directory, "input.pdf");
await writeFile(path, input.body, {
flag: "wx",
mode: 0o600,
...(input.signal ? { signal: input.signal } : {}),
});
input.signal?.throwIfAborted();
const output = await executePdfinfo({
path,
...(input.signal ? { signal: input.signal } : {}),
});
input.signal?.throwIfAborted();
assertSafeGeometry(output);
} catch (error) {
if (input.signal?.aborted) {
throw input.signal.reason;
}
throw error;
} finally {
await rm(directory, { force: true, recursive: true });
}
},
};
}
function isPdf(input: ParseDocumentInput): boolean {
return (
input.mimeType.split(";", 1)[0]?.trim().toLowerCase() === "application/pdf" ||
input.filename.toLowerCase().endsWith(".pdf") ||
Buffer.from(
input.body.buffer,
input.body.byteOffset,
Math.min(input.body.byteLength, 1024),
).indexOf("%PDF-") !== -1
);
}
function runPdfinfo(
input: PdfinfoExecutionInput,
options: {
readonly executable: string;
readonly maxOutputBytes: number;
readonly timeoutMs: number;
},
): Promise<string> {
input.signal?.throwIfAborted();
return new Promise((resolve, reject) => {
const child = execFile(
options.executable,
["-f", "1", "-l", String(MAX_PAGES + 1), "-box", input.path],
{
encoding: "utf8",
env: { ...process.env, LANG: "C", LC_ALL: "C" },
killSignal: "SIGKILL",
maxBuffer: options.maxOutputBytes,
timeout: options.timeoutMs,
},
(error, stdout) => {
input.signal?.removeEventListener("abort", cancel);
if (input.signal?.aborted) {
reject(input.signal.reason);
} else if (error?.code === "ENOENT") {
reject(
new Error("PDF safety inspection requires the pdfinfo executable (poppler-utils)."),
);
} else if (error) {
// stderr can contain document content; do not include it in errors or their causes.
reject(
new ProviderInputError(
"Unable to safely inspect PDF page geometry within the inspection limits.",
),
);
} else {
resolve(stdout);
}
},
);
// Wait for the execFile callback after termination so temporary input is not removed while
// a child still uses it, and callers never continue while abandoned work is still running.
const cancel = () => child.kill("SIGKILL");
input.signal?.addEventListener("abort", cancel, { once: true });
if (input.signal?.aborted) {
cancel();
}
});
}
type PageGeometry = Map<"size" | "MediaBox" | "CropBox", readonly [number, number]>;
function assertSafeGeometry(output: string): void {
const pageCounts = [...output.matchAll(/^Pages:\s*(.*?)\s*$/gm)];
const rawPageCount = pageCounts[0]?.[1] ?? "";
if (pageCounts.length !== 1 || !/^\d+$/.test(rawPageCount)) {
throw invalidGeometry();
}
const pageCount = Number(rawPageCount);
if (!Number.isSafeInteger(pageCount) || pageCount < 1 || pageCount > MAX_PAGES) {
throw new ProviderInputError(
`PDF safety inspection supports between 1 and ${MAX_PAGES} pages.`,
);
}
const pages = new Map<number, PageGeometry>();
for (const match of output.matchAll(/^Page\s+(\d+)\s+(size|MediaBox|CropBox):\s*(.*?)\s*$/gm)) {
const pageNumber = Number(match[1]);
const kind = match[2] as "size" | "MediaBox" | "CropBox";
const value = match[3] ?? "";
if (!Number.isSafeInteger(pageNumber) || pageNumber < 1 || pageNumber > pageCount) {
throw invalidGeometry();
}
const page = pages.get(pageNumber) ?? new Map();
if (page.has(kind)) {
throw invalidGeometry();
}
const dimensions = kind === "size" ? parsePageSize(value) : parseBox(value);
page.set(kind, dimensions);
pages.set(pageNumber, page);
}
if (pages.size !== pageCount) {
throw invalidGeometry();
}
for (const [pageNumber, geometry] of pages) {
if (geometry.size !== 3) {
throw invalidGeometry();
}
for (const [width, height] of geometry.values()) {
const rasterWidth = Math.ceil((width / 72) * RASTER_DPI);
const rasterHeight = Math.ceil((height / 72) * RASTER_DPI);
if (
rasterWidth > MAX_PAGE_EDGE_PIXELS ||
rasterHeight > MAX_PAGE_EDGE_PIXELS ||
rasterWidth * rasterHeight > MAX_PAGE_PIXELS
) {
throw new ProviderInputError(
`PDF page ${pageNumber} exceeds the safe rasterization budget (${MAX_PAGE_PIXELS} pixels per page, ${MAX_PAGE_EDGE_PIXELS} pixels per edge at ${RASTER_DPI} DPI). Resize the page before importing.`,
);
}
}
}
}
function parsePageSize(value: string): readonly [number, number] {
const match = /^(\S+)\s+x\s+(\S+)\s+pts(?:\s+\([^\r\n]*\))?$/.exec(value);
if (!match) {
throw invalidGeometry();
}
return positiveDimensions(finiteNumber(match[1] ?? ""), finiteNumber(match[2] ?? ""));
}
function parseBox(value: string): readonly [number, number] {
const coordinates = value.trim().split(/\s+/);
if (coordinates.length !== 4) {
throw invalidGeometry();
}
const [left = 0, bottom = 0, right = 0, top = 0] = coordinates.map(finiteNumber);
return positiveDimensions(right - left, top - bottom);
}
function finiteNumber(value: string): number {
const number = Number(value);
if (!/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/.test(value) || !Number.isFinite(number)) {
throw invalidGeometry();
}
return number;
}
function positiveDimensions(width: number, height: number): readonly [number, number] {
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
throw invalidGeometry();
}
return [width, height];
}
function invalidGeometry(): ProviderInputError {
return new ProviderInputError(
"Unable to safely inspect PDF: page geometry is incomplete or invalid.",
);
}
function positiveInteger(value: number, name: string): number {
if (!Number.isSafeInteger(value) || value < 1) {
throw new Error(`PDF preflight ${name} must be a positive safe integer.`);
}
return value;
}

2
knowledge-fs/apps/api/src/profile-visual-embedding-options.ts
View File

@ -85,7 +85,7 @@ export function createApiProfileVisualEmbeddingOptions({
"KNOWLEDGE_VISUAL_EMBEDDING_MAX_BATCH_BYTES",
),
objectStorage,
preferredVariant: env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT?.trim() || "thumbnail",
preferredVariant: env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT?.trim() || "analysis",
provider: imageEmbeddingProviderFactory(profile),
});
return lifecycleGate.run(() => objectProvider.embedAssets(input), { signal: input.signal });

4
knowledge-fs/apps/api/src/visual-embedding-options.test.ts
View File

@ -46,7 +46,7 @@ describe("createApiVisualEmbeddingOptions", () => {
await adapter.objectStorage.putObject({
body: new Uint8Array([1, 2, 3]),
contentType: "image/png",
key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png",
key: "tenant/spaces/space/documents/doc/assets/chart.png",
});
globalThis.fetch = (async (input, init) => {
const request = new Request(input, init);
@ -123,7 +123,7 @@ describe("createApiVisualEmbeddingOptions", () => {
{
content: "AQID",
content_type: "image",
file_id: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png",
file_id: "tenant/spaces/space/documents/doc/assets/chart.png",
},
],
input_type: "document",

2
knowledge-fs/apps/api/src/visual-embedding-options.ts
View File

@ -130,7 +130,7 @@ export function createApiVisualEmbeddingOptions({
objectStorage,
...(trimmed(env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT)
? { preferredVariant: trimmed(env.KNOWLEDGE_VISUAL_EMBEDDING_PREFERRED_VARIANT) }
: { preferredVariant: "thumbnail" }),
: { preferredVariant: "analysis" }),
provider: imageBytesProvider,
});

50
knowledge-fs/docs/production-deployment.md
View File

@ -92,6 +92,7 @@ the service:
| `KNOWLEDGE_BUFFERED_DOCUMENT_UPLOAD_IDLE_TIMEOUT_MS` | Maximum continuous idle interval while reading a direct multipart or small-file fallback body. Defaults to `30000`; expiry cancels the reader, returns 408, and releases admission. |
| `KNOWLEDGE_BUFFERED_DOCUMENT_UPLOAD_TOTAL_TIMEOUT_MS` | Generous total body-read deadline for a direct multipart or small-file fallback request. Defaults to `600000`, must be at least the idle timeout, and returns 408 on expiry. |
| `UNSTRUCTURED_API_URL` | Parser endpoint for complex formats. |
| `UNSTRUCTURED_BACKEND_REVISION` | Non-secret identity of the actual parser image, models and output policy. Bump it on semantic backend changes to invalidate raw parse checkpoint identity. An unset revision is explicitly `external-unversioned`, not proof of reproducibility. Admission-only limits need no revision bump. |
| `UNSTRUCTURED_API_KEY` | Optional parser authentication. |
| `UNSTRUCTURED_MAX_CONCURRENCY` | Process-wide limit shared by every remote parser request; defaults to `2`. |
| `UNSTRUCTURED_HEAVY_MAX_CONCURRENCY` | Nested limit for every PDF and structurally/byte-heavy Office, email, EPUB, ODT, or RTF request. The bundled parser profile uses `1`; it must not exceed `UNSTRUCTURED_MAX_CONCURRENCY`. The materialization pre-admission lane follows this value but is capped at `KNOWLEDGE_DOCUMENT_MATERIALIZATION_MAX_CONCURRENCY - 1` (minimum `1`) to preserve ordinary-document progress. `UNSTRUCTURED_PDF_MAX_CONCURRENCY` remains a lower-precedence compatibility alias. |
@ -118,7 +119,8 @@ including response consumption. It defaults to `60000`; transport failures and
The optional `knowledge-fs-unstructured` Compose profile starts an isolated parser service with six
pages per child request, three child workers, and zero child retries. It does not modify Dify's
existing `unstructured` service or legacy ETL traffic. The tracked
`knowledge-fs-unstructured-service.defaults` file contains only service-side page-parallel values;
`knowledge-fs-unstructured-service.defaults` file contains service-side page-parallel and PDF
raster-safety values;
operator-owned `knowledge-fs-unstructured.env` is loaded afterwards and can override it. The
required file deliberately uses a non-`.env` suffix so it remains tracked in clean checkouts.
The `.env.example` files remain copy-only templates and are never loaded as runtime configuration.
@ -127,6 +129,30 @@ resource benchmarks; review the same contracts before intentionally changing tha
requests are at or below the split size and therefore partition locally instead of recursively
spawning more requests.
PDF import also checks page geometry before sending the file to Unstructured. Compressed file
size and page count alone cannot bound raster memory: an 80 × 180 cm single-page PDF needs about
273 million pixels at the pinned parser's 350 DPI even if the file is only 248 KB. The client
checks every page with bounded Poppler metadata inspection, estimates its raster size at 350 DPI,
and rejects pages above 25 million pixels or 10,000 pixels on either side. It also rejects files
whose geometry cannot be safely inspected. Oversized documents must be exported at smaller page
dimensions or tiled into smaller pages before import; splitting only between existing giant pages
does not reduce the per-page risk. The original file, parsing strategy, text, and image extraction
remain unchanged for admitted documents. The client does not silently switch to text-only parsing.
The isolated service explicitly keeps `PDF_RENDER_DPI=350` (the pinned image's existing default)
and sets `PDF_RENDER_MAX_PIXELS_PER_PAGE=25000000`. The latter is a second guard in the pinned
`unstructured-inference` renderer: it rejects an oversized page before allocating its bitmap.
That renderer is shared by `hi_res` layout inference, PDF OCR (including `auto` fallback), and
image-block extraction. Its PDF rendering uses PDFium; Poppler in this upstream path is used for
metadata inspection. These process settings are not HTTP request parameters: the pinned API does
not expose `pdf_image_dpi`. Do not raise the service DPI above 350, increase the pixel ceiling, or
disable the guard (`0`) without changing and validating the client policy. Operator env overrides
still take precedence, so existing deployments must remove conflicting overrides and recreate
the parser service as well as deploy the new KnowledgeFS image. The Kubernetes baseline does not
deploy Unstructured; configure the same settings on its external parser service. Geometry bounds
address oversized page rasters, not every possible embedded-image decompression or PDF complexity
attack; the service memory limit and workload admission remain necessary.
The KnowledgeFS client keeps a process-wide limit of `2` and adds a heavy-workload nested limit of
`1`. Every PDF remains heavy because compressed PDF object streams make a bounded page-count scan
unreliable. ZIP-backed remote formats become heavy when their admitted body exceeds 8 MiB or a
@ -146,6 +172,28 @@ own child thread pool. The Kubernetes baseline does not own an Unstructured depl
generic client limits. Operators with a different resource envelope must benchmark representative
narrative, table, and scanned pages before changing either concurrency limit.
### Document structure admission
Before sending ZIP-backed Office, ODT or EPUB documents to the service, KnowledgeFS also streams
their actual expanded entries within the same admission slot. This checks XML structure and
spreadsheet cell spans, not just compressed upload size. Default limits include 512 MiB total
expansion, 64 MiB XML (16 MiB per part), XML depth 128, 256 worksheets, and 250,000 dense cells per
worksheet / 500,000 per workbook. Sparse distant cells and repeated worksheet references count
toward the dense-cell cost; harmless whole-column formatting does not. Unsafe paths, damaged
archives/XML, or missing/external worksheet relationships fail before a provider request. Standard
EPUB 2 XHTML declarations and optional image-relationship fallbacks remain supported.
Native structured parsing shares the admitted upload-byte limit (15 MiB by default), so an upload
above 10 MiB no longer silently switches JSON/CSV to a different parser. Decoded structure,
document-wide table expansion, and projected output have independent finite budgets. Exceeding a
budget is an explicit non-retryable error, not truncated content. Split unusually complex inputs
before retrying; these are resource limits, not the file formats' theoretical maximum sizes.
These checks do not replace process isolation or memory limits. Legacy DOC/PPT/XLS conversion,
nested mail attachments, native image decoding, and synchronous in-process parsing still require
further isolation work. Existing indexes are unchanged until re-indexing; new parses use the
updated parser policy identity. No new database migration or model setting is required.
## PDF image rasterization
The production API image installs Poppler and verifies `pdftoppm` during the image build. Its image

2
knowledge-fs/infra/local/.env.example
View File

@ -14,6 +14,8 @@ UNSTRUCTURED_PORT=8000
# Host-run API processes use the published localhost port. Compose must use the separate internal
# service URL below because 127.0.0.1 inside the API container refers to that container itself.
UNSTRUCTURED_API_URL=http://127.0.0.1:8000
# Non-secret identity for the actual parser image/model/output policy; bump on semantic upgrades.
UNSTRUCTURED_BACKEND_REVISION=
UNSTRUCTURED_CONTAINER_API_URL=http://unstructured:8000
UNSTRUCTURED_API_KEY=
UNSTRUCTURED_PARALLEL_MODE_ENABLED=true

21
knowledge-fs/infra/local/compose.unstructured-sandbox.yaml Normal file
View File

@ -0,0 +1,21 @@
# Explicit local opt-in; retains the same unstructured hostname and API URL.
services:
unstructured:
build:
context: ../../services/unstructured-sandbox
dockerfile: Dockerfile
image: knowledge-fs-unstructured-sandbox:local
init: true
read_only: true
pids_limit: 192
cap_drop: [ALL]
security_opt:
- no-new-privileges:true
tmpfs:
- /tmp:rw,nosuid,nodev,size=1073741824,mode=1777
environment:
UNSTRUCTURED_PARALLEL_MODE_ENABLED: "false"
deploy:
resources:
limits:
pids: 192

4
knowledge-fs/infra/local/compose.yaml
View File

@ -21,6 +21,9 @@ services:
unstructured:
image: downloads.unstructured.io/unstructured-io/unstructured-api@sha256:0df934a22e4e893cf15e7aeaf35c463ecc75937758a83099aefdc13041619a1d
environment:
# Keep geometry admission aligned with the KnowledgeFS client and reject before allocation.
PDF_RENDER_DPI: "350"
PDF_RENDER_MAX_PIXELS_PER_PAGE: "25000000"
UNSTRUCTURED_PARALLEL_MODE_ENABLED: ${UNSTRUCTURED_PARALLEL_MODE_ENABLED:-true}
UNSTRUCTURED_PARALLEL_MODE_URL: ${UNSTRUCTURED_PARALLEL_MODE_URL:-http://127.0.0.1:8000/general/v0/general}
UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE: ${UNSTRUCTURED_PARALLEL_MODE_SPLIT_SIZE:-6}
@ -81,6 +84,7 @@ services:
KNOWLEDGE_DEV_SUBJECT_ID: ${KNOWLEDGE_DEV_SUBJECT_ID:-dev-user}
KNOWLEDGE_DEV_TENANT_ID: ${KNOWLEDGE_DEV_TENANT_ID:-tenant-dev}
UNSTRUCTURED_API_URL: ${UNSTRUCTURED_CONTAINER_API_URL:-http://unstructured:8000}
UNSTRUCTURED_BACKEND_REVISION: ${UNSTRUCTURED_BACKEND_REVISION:-}
UNSTRUCTURED_MAX_CONCURRENCY: ${UNSTRUCTURED_MAX_CONCURRENCY:-2}
UNSTRUCTURED_HEAVY_MAX_CONCURRENCY: ${UNSTRUCTURED_HEAVY_MAX_CONCURRENCY:-1}
UNSTRUCTURED_MAX_INPUT_BYTES: ${UNSTRUCTURED_MAX_INPUT_BYTES:-15728640}

7
knowledge-fs/package.json
View File

@ -13,7 +13,7 @@
"benchmark:ingestion": "node --import tsx scripts/benchmark-ingestion-model-optimizations.mjs",
"build": "turbo run build",
"capability:export": "node --import tsx scripts/export-capability-v2-operations.mjs",
"check": "pnpm typecheck && pnpm test && pnpm openapi:export:test && pnpm test:coverage:ci && pnpm eval:regression && pnpm eval:phase4 && pnpm swagger:test && pnpm db:migrations:check && pnpm p9:bundle:test && pnpm ci:workflow:test && pnpm compose:middleware:config && pnpm compose:config && pnpm dify:compose:config && pnpm compose:middleware:test && pnpm compose:apps:test && pnpm docker:context:test && pnpm local:happy-path:test && pnpm semantic:rollout:test && pnpm docker:api:bundle-smoke:test && pnpm docker:admin:http-smoke:test && pnpm docker:apps:smoke:test",
"check": "pnpm typecheck && pnpm test && pnpm openapi:export:test && pnpm test:coverage:ci && pnpm eval:regression && pnpm eval:phase4 && pnpm swagger:test && pnpm db:migrations:check && pnpm p9:bundle:test && pnpm ci:workflow:test && pnpm compose:middleware:config && pnpm compose:config && pnpm dify:compose:config && pnpm compose:middleware:test && pnpm compose:apps:test && pnpm docker:context:test && pnpm local:happy-path:test && pnpm semantic:rollout:test && pnpm docker:api:bundle-smoke:test && pnpm docker:admin:http-smoke:test && pnpm docker:apps:smoke:test && pnpm parser:regression:test",
"ci:workflow:test": "node --test scripts/github-actions-workflow.test.mjs scripts/secret-scan.test.mjs",
"compose:config": "docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml config",
"compose:apps:test": "node --test scripts/compose-apps.test.mjs",
@ -61,7 +61,10 @@
"test": "turbo run test",
"test:coverage": "turbo run test:coverage",
"test:coverage:ci": "turbo run test:coverage --filter=!@knowledge/api",
"typecheck": "turbo run typecheck"
"typecheck": "turbo run typecheck",
"parser:regression:test": "node --test scripts/parser-benchmark.test.mjs scripts/pdf-thumbnail-benchmark.test.mjs",
"benchmark:parsers": "node --import tsx scripts/parser-benchmark.mjs",
"benchmark:pdf-thumbnail": "node --import tsx scripts/pdf-thumbnail-benchmark.mjs"
},
"devDependencies": {
"@biomejs/biome": "^1.9.4",

163
knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts
View File

@ -26,6 +26,134 @@ const firstArtifactId = "30000000-0000-4000-8000-000000000001";
const retryArtifactId = "30000000-0000-4000-8000-000000000002";
describe("compileDocumentArtifact canonical artifact", () => {
it("uses provider fallback when synchronous external PDF images cannot be resolved", async () => {
const hints: unknown[] = [];
const deps = pipelineDeps();
const artifact = await compileDocumentArtifact(
{
asset: {
...documentAsset(),
mimeType: "application/pdf",
filename: "image.pdf",
metadata: { language: "zh", requiresTables: true },
},
body: new TextEncoder().encode("%PDF"),
knowledgeSpaceId,
permissionScope: [],
tenantId: "tenant-1",
traceId: randomUUID(),
},
{
...deps,
profileImageExtractionEnabled: true,
documentPdfRasterizer: {
render: async () => {
throw new Error("Missing page cannot render");
},
},
documentParser: {
kind: "unstructured",
parse: async (input) => {
hints.push(input.parserHints);
return ParseArtifactSchema.parse({
artifactHash: "a".repeat(64),
contentType: "mixed",
createdAt: "2026-07-13T00:00:00.000Z",
documentAssetId,
elements: [
{
id: "figure",
type: "image",
sectionPath: [],
metadata: input.parserHints?.imagesHandledExternally
? {}
: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } },
},
],
id: firstArtifactId,
metadata: {},
parser: "unstructured",
version: 1,
});
},
},
},
);
expect(hints).toEqual([
expect.objectContaining({
imagesHandledExternally: true,
requiresImages: true,
language: "zh",
requiresTables: true,
}),
expect.objectContaining({
imagesHandledExternally: false,
requiresImages: true,
language: "zh",
requiresTables: true,
}),
]);
expect(artifact.elements[0]?.metadata.assetRef).toMatchObject({
objectKey: expect.any(String),
});
});
it("uses resolved text-only capabilities for synchronous parsing without losing source references", async () => {
const deps = pipelineDeps();
const artifact = await compileDocumentArtifact(
{
asset: documentAsset(),
body: new Uint8Array([1]),
knowledgeSpaceId,
permissionScope: [],
tenantId: "tenant-1",
traceId: randomUUID(),
},
{
...deps,
resolveDocumentMediaCapabilities: async () => ({
imageExtractionEnabled: false,
visualEmbeddingEnabled: false,
}),
documentMultimodalRemoteAssetFetcher: {
fetch: async () => {
throw new Error("Text-only profile must not download");
},
},
documentParser: {
kind: "native-markdown",
parse: async (input) => {
expect(input.parserHints).toMatchObject({ requiresImages: false });
return ParseArtifactSchema.parse({
artifactHash: "a".repeat(64),
contentType: "mixed",
createdAt: "2026-07-13T00:00:00.000Z",
documentAssetId,
elements: [
{
id: "figure",
type: "image",
sectionPath: [],
metadata: { assetRef: { uri: "https://example.test/image.png" } },
},
],
id: firstArtifactId,
metadata: {},
parser: "native-markdown",
version: 1,
});
},
},
},
);
expect(artifact.elements[0]?.metadata.assetRef).toEqual({
uri: "https://example.test/image.png",
});
expect(artifact.metadata).toMatchObject({
mediaExecutionPlan: { materializeImages: false },
parseCoverage: { media: { status: "not-requested" } },
});
});
it("materializes parser-provided remote image refs before persisting the manifest", async () => {
const adapter = createNodePlatformAdapter({ env: {} });
const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 });
@ -107,7 +235,11 @@ describe("compileDocumentArtifact canonical artifact", () => {
);
expect(remoteFetches).toEqual([
{ maxBytes: 10 * 1024 * 1024, url: "https://cdn.example.test/office-linked.png" },
{
maxBytes: 10 * 1024 * 1024,
signal: expect.any(AbortSignal),
url: "https://cdn.example.test/office-linked.png",
},
]);
expect(artifact.elements[0]?.metadata).toMatchObject({
assetRef: {
@ -342,3 +474,32 @@ function documentAsset(): DocumentAsset {
version: 1,
};
}
function pipelineDeps(): Omit<CompileDocumentArtifactDeps, "documentParser"> {
return {
artifacts: createInMemoryParseArtifactRepository({ maxArtifacts: 4 }),
artifactSegments: createInMemoryArtifactSegmentRepository({
maxBatchSize: 10,
maxListLimit: 10,
maxSegments: 10,
}),
documentMultimodalManifests: createInMemoryDocumentMultimodalManifestRepository({
maxManifests: 4,
}),
generateArtifactSegmentId: randomUUID,
generateKnowledgePathId: randomUUID,
knowledgePaths: createInMemoryKnowledgePathRepository({ maxListLimit: 20, maxPaths: 20 }),
now: () => "2026-07-13T00:00:00.000Z",
objectStorage: createNodePlatformAdapter({ env: {} }).objectStorage,
outlineBuilder: createDocumentOutlineBuilder({
generateId: randomUUID,
maxElements: 10,
maxNodes: 10,
maxSummaryChars: 1000,
now: () => "2026-07-13T00:00:00.000Z",
}),
outlines: createInMemoryDocumentOutlineRepository({ maxOutlines: 4 }),
synchronousUploadReindexer: null,
traces: createNoopTraceRecorder(),
};
}

164
knowledge-fs/packages/api/src/document-compilation-pipeline.ts
View File

@ -18,6 +18,12 @@ import {
buildDocumentOutlineKnowledgePath,
buildDocumentSectionKnowledgePaths,
} from "./document-knowledge-paths";
import {
type DocumentMediaCapabilityResolver,
createDocumentMediaExecutionPlan,
documentParserHints,
withDocumentMediaExecutionPlan,
} from "./document-media-execution-plan";
import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact";
import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor";
import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor";
@ -34,6 +40,7 @@ import type { DocumentOutlineRepository } from "./document-outline-repository";
import type { DocumentOutlineSummaryEnhancer } from "./document-outline-summary-enhancer";
import {
type DocumentPdfRasterizer,
DocumentPdfRenderError,
rasterizeDocumentPdfMultimodalAssets,
} from "./document-pdf-rasterizer";
import { sha256Hex } from "./document-upload-utils";
@ -60,6 +67,7 @@ export interface CompileDocumentArtifactInput {
readonly publicationGenerationId?: string | undefined;
readonly tenantId: string;
readonly traceId: string;
readonly signal?: AbortSignal | undefined;
}
export interface CompileDocumentArtifactDeps {
@ -89,6 +97,8 @@ export interface CompileDocumentArtifactDeps {
readonly synchronousUploadReindexer: IncrementalReindexer | null;
readonly traces: TraceRecorder;
readonly visualEmbeddingModel?: string | undefined;
readonly profileImageExtractionEnabled?: boolean | undefined;
readonly resolveDocumentMediaCapabilities?: DocumentMediaCapabilityResolver | undefined;
}
export async function compileDocumentArtifact(
@ -103,6 +113,7 @@ export async function compileDocumentArtifact(
publicationGenerationId: requestedPublicationGenerationId,
tenantId,
traceId,
signal,
} = input;
const {
artifacts,
@ -131,6 +142,8 @@ export async function compileDocumentArtifact(
synchronousUploadReindexer,
traces,
visualEmbeddingModel,
profileImageExtractionEnabled,
resolveDocumentMediaCapabilities,
} = deps;
const publicationGenerationId =
requestedPublicationGenerationId === undefined
@ -149,58 +162,115 @@ export async function compileDocumentArtifact(
}
: null;
let stagedProjectionIds: readonly string[] = [];
signal?.throwIfAborted();
const mediaCapabilities = await resolveDocumentMediaCapabilities?.({
knowledgeSpaceId,
tenantId,
signal,
});
const mediaPlan = createDocumentMediaExecutionPlan({
profileImageExtractionEnabled:
mediaCapabilities?.imageExtractionEnabled ?? profileImageExtractionEnabled,
hasPdfRasterizer: Boolean(documentPdfRasterizer),
hasImageVariantGenerator: Boolean(documentMultimodalImageVariantGenerator),
visualEmbeddingEnabled:
mediaCapabilities?.visualEmbeddingEnabled ?? Boolean(visualEmbeddingModel),
});
const artifact = await traceAsync(traces, traceId, "ingestion.parser_parse", () =>
documentParser.parse({
body,
documentAssetId: asset.id,
filename: asset.filename,
mimeType: asset.mimeType,
version: asset.version,
}),
);
const rasterizedArtifact = await traceAsync(traces, traceId, "ingestion.pdf_rasterize", () =>
rasterizeDocumentPdfMultimodalAssets({
artifact,
documentBody: body,
documentMimeType: asset.mimeType,
knowledgeSpaceId,
...(documentMultimodalMaxPdfRasterizedAssets
? { maxRasterizedAssets: documentMultimodalMaxPdfRasterizedAssets }
: {}),
objectStorage,
...(documentPdfRasterizer ? { rasterizer: documentPdfRasterizer } : {}),
tenantId,
}),
const externalPdfImages =
Boolean(documentPdfRasterizer) &&
asset.mimeType.split(";", 1)[0]?.trim().toLowerCase() === "application/pdf";
const parse = (imagesHandledExternally: boolean) =>
traceAsync(traces, traceId, "ingestion.parser_parse", () =>
documentParser.parse({
body,
documentAssetId: asset.id,
filename: asset.filename,
mimeType: asset.mimeType,
parserHints: documentParserHints({
assetMetadata: asset.metadata,
imagesHandledExternally,
requiresImages: mediaPlan.requestParserImages,
}),
...(signal ? { signal } : {}),
version: asset.version,
}),
);
const artifact = await parse(externalPdfImages);
const rasterizedArtifact = await traceAsync(
traces,
traceId,
"ingestion.pdf_rasterize",
async () => {
if (!mediaPlan.materializeImages) return { artifact };
try {
const result = await rasterizeDocumentPdfMultimodalAssets({
artifact,
documentBody: body,
documentMimeType: asset.mimeType,
knowledgeSpaceId,
...(documentMultimodalMaxPdfRasterizedAssets
? { maxRasterizedAssets: documentMultimodalMaxPdfRasterizedAssets }
: {}),
objectStorage,
...(documentPdfRasterizer ? { rasterizer: documentPdfRasterizer } : {}),
tenantId,
...(signal ? { signal } : {}),
});
if (
externalPdfImages &&
mediaPlan.requestParserImages &&
result.rasterizedCount === 0 &&
result.unresolvedCount > 0
)
return { artifact: await parse(false) };
return result;
} catch (error) {
signal?.throwIfAborted();
if (
!(error instanceof DocumentPdfRenderError) ||
!externalPdfImages ||
!mediaPlan.requestParserImages
)
throw error;
return { artifact: await parse(false) };
}
},
);
const assetExtractionResult = await traceAsync(
traces,
traceId,
"ingestion.multimodal_assets_extract",
() =>
extractDocumentMultimodalAssets({
...(documentMultimodalLocalAssetAllowlist
? { allowLocalAssetPaths: documentMultimodalLocalAssetAllowlist }
: {}),
artifact: rasterizedArtifact.artifact,
knowledgeSpaceId,
...(documentMultimodalMaxExtractedAssets
? { maxExtractedAssets: documentMultimodalMaxExtractedAssets }
: {}),
...(documentMultimodalMaxLocalAssetBytes
? { maxLocalAssetBytes: documentMultimodalMaxLocalAssetBytes }
: {}),
...(documentMultimodalImageVariantGenerator
? { imageVariantGenerator: documentMultimodalImageVariantGenerator }
: {}),
objectStorage,
...(documentMultimodalRemoteAssetFetcher
? { remoteAssetFetcher: documentMultimodalRemoteAssetFetcher }
: {}),
tenantId,
}),
mediaPlan.materializeImages
? extractDocumentMultimodalAssets({
...(documentMultimodalLocalAssetAllowlist
? { allowLocalAssetPaths: documentMultimodalLocalAssetAllowlist }
: {}),
artifact: rasterizedArtifact.artifact,
knowledgeSpaceId,
...(documentMultimodalMaxExtractedAssets
? { maxExtractedAssets: documentMultimodalMaxExtractedAssets }
: {}),
...(documentMultimodalMaxLocalAssetBytes
? { maxLocalAssetBytes: documentMultimodalMaxLocalAssetBytes }
: {}),
...(documentMultimodalImageVariantGenerator
? { imageVariantGenerator: documentMultimodalImageVariantGenerator }
: {}),
objectStorage,
...(documentMultimodalRemoteAssetFetcher
? { remoteAssetFetcher: documentMultimodalRemoteAssetFetcher }
: {}),
tenantId,
...(signal ? { signal } : {}),
})
: Promise.resolve({ artifact: rasterizedArtifact.artifact, extractedCount: 0 }),
);
signal?.throwIfAborted();
const materializedArtifact = finalizeDocumentMultimodalArtifact(
withDocumentMediaExecutionPlan(assetExtractionResult.artifact, mediaPlan),
);
const materializedArtifact = finalizeDocumentMultimodalArtifact(assetExtractionResult.artifact);
const artifactToPersist = ParseArtifactSchema.parse({
...materializedArtifact,
metadata: {
@ -300,7 +370,11 @@ export async function compileDocumentArtifact(
projectionVersion: asset.version,
...(publicationGenerationId !== undefined ? { publicationGenerationId } : {}),
tenantId,
...(visualEmbeddingModel ? { visualModel: visualEmbeddingModel } : {}),
...(visualEmbeddingModel && mediaPlan.visualEmbedding
? { visualModel: visualEmbeddingModel }
: mediaCapabilities || profileImageExtractionEnabled === false
? { skipVisual: true as const }
: {}),
}),
);
if (stagedProjectionPublication && reindexResult.status === "rebuilt") {

32
knowledge-fs/packages/api/src/document-compilation-worker.test.ts
View File

@ -223,7 +223,14 @@ describe("createDocumentCompilationWorker lease integration", () => {
contentType: "text",
createdAt: "2026-09-01T12:00:00.000Z",
documentAssetId: input.documentAssetId,
elements: [],
elements: [
{
id: "inline-profile-image",
type: "image",
sectionPath: [],
metadata: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } },
},
],
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f6a16",
metadata: {},
parser: "unstructured",
@ -238,13 +245,22 @@ describe("createDocumentCompilationWorker lease integration", () => {
},
profileImageExtractionEnabled,
reindexer: {
reindex: async (input) => ({
artifact: input.parseArtifact,
nodesCreated: 0,
projectionIds: [],
projectionsCreated: 0,
status: "rebuilt",
}),
reindex: async (input) => {
const assetRef = input.parseArtifact.elements[0]?.metadata.assetRef;
if (!profileImageExtractionEnabled) expect(input.skipVisual).toBe(true);
expect(assetRef).toMatchObject(
profileImageExtractionEnabled
? { objectKey: expect.any(String) }
: { uri: "data:image/png;base64,AQIDBA==" },
);
return {
artifact: input.parseArtifact,
nodesCreated: 0,
projectionIds: [],
projectionsCreated: 0,
status: "rebuilt",
};
},
},
});

122
knowledge-fs/packages/api/src/document-compilation-worker.ts
View File

@ -53,6 +53,11 @@ import {
buildDocumentOutlineKnowledgePath,
buildDocumentSectionKnowledgePaths,
} from "./document-knowledge-paths";
import {
createDocumentMediaExecutionPlan,
documentParserHints,
withDocumentMediaExecutionPlan,
} from "./document-media-execution-plan";
import type { DocumentModelBudget } from "./document-model-budget";
import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact";
import {
@ -465,14 +470,14 @@ export function createDocumentCompilationWorker({
signal ? { signal } : undefined,
);
} else {
const requiresImages =
profileImageExtractionEnabled ??
Boolean(
visualEmbeddingMode === "profile" ||
visualEmbeddingModel ||
multimodalImageVariantGenerator ||
pdfRasterizer,
);
const mediaPlan = createDocumentMediaExecutionPlan({
profileImageExtractionEnabled,
hasPdfRasterizer: Boolean(pdfRasterizer),
hasImageVariantGenerator: Boolean(multimodalImageVariantGenerator),
visualEmbeddingEnabled:
visualEmbeddingMode === "profile" || Boolean(visualEmbeddingModel),
});
const requiresImages = mediaPlan.requestParserImages;
const externalPdfImages = Boolean(pdfRasterizer) && isPdfDocument(activeAsset.mimeType);
const primaryParserHints = documentParserHints({
assetMetadata: activeAsset.metadata,
@ -629,7 +634,10 @@ export function createDocumentCompilationWorker({
let multimodalArtifact: ParseArtifact;
try {
if (activeParserOutput.route === "provider-fallback") {
if (
activeParserOutput.route === "provider-fallback" ||
!mediaPlan.materializeImages
) {
multimodalArtifact = activeParserOutput.artifact;
} else {
const rasterized = await rasterizeDocumentPdfMultimodalAssets({
@ -672,31 +680,35 @@ export function createDocumentCompilationWorker({
multimodalArtifact = activeParserOutput.artifact;
}
await assertWritable();
const { artifact } = await extractDocumentMultimodalAssets({
...(multimodalLocalAssetAllowlist
? { allowLocalAssetPaths: multimodalLocalAssetAllowlist }
: {}),
artifact: multimodalArtifact,
knowledgeSpaceId: input.knowledgeSpaceId,
...(multimodalMaxExtractedAssets
? { maxExtractedAssets: multimodalMaxExtractedAssets }
: {}),
...(multimodalMaxLocalAssetBytes
? { maxLocalAssetBytes: multimodalMaxLocalAssetBytes }
: {}),
...(multimodalImageVariantGenerator
? { imageVariantGenerator: multimodalImageVariantGenerator }
: {}),
objectStorage: multimodalObjectStorage,
...(multimodalRemoteAssetFetcher
? { remoteAssetFetcher: multimodalRemoteAssetFetcher }
: {}),
...(signal ? { signal } : {}),
tenantId: input.tenantId,
writeOwnerId: multimodalWriteOwnerId,
});
const { artifact } = mediaPlan.materializeImages
? await extractDocumentMultimodalAssets({
...(multimodalLocalAssetAllowlist
? { allowLocalAssetPaths: multimodalLocalAssetAllowlist }
: {}),
artifact: multimodalArtifact,
knowledgeSpaceId: input.knowledgeSpaceId,
...(multimodalMaxExtractedAssets
? { maxExtractedAssets: multimodalMaxExtractedAssets }
: {}),
...(multimodalMaxLocalAssetBytes
? { maxLocalAssetBytes: multimodalMaxLocalAssetBytes }
: {}),
...(multimodalImageVariantGenerator
? { imageVariantGenerator: multimodalImageVariantGenerator }
: {}),
objectStorage: multimodalObjectStorage,
...(multimodalRemoteAssetFetcher
? { remoteAssetFetcher: multimodalRemoteAssetFetcher }
: {}),
...(signal ? { signal } : {}),
tenantId: input.tenantId,
writeOwnerId: multimodalWriteOwnerId,
})
: { artifact: multimodalArtifact };
await assertWritable();
const finalizedArtifact = finalizeDocumentMultimodalArtifact(artifact);
const finalizedArtifact = finalizeDocumentMultimodalArtifact(
withDocumentMediaExecutionPlan(artifact, mediaPlan),
);
const checkpointPolicyFingerprint =
activeParserOutput.rawCheckpointPolicyFingerprint ??
persistedCheckpoint?.policyFingerprint;
@ -948,12 +960,18 @@ export function createDocumentCompilationWorker({
: (resolvedEmbedding?.vectorSpaceId ?? denseEmbeddingModel);
await assertWritable();
const resolvedVisualEmbeddingModel =
visualEmbeddingMode === "profile"
? frozenEmbeddingProfile?.model
: visualEmbeddingMode === "disabled"
? undefined
: visualEmbeddingModel;
if (visualEmbeddingMode === "profile" && !resolvedVisualEmbeddingModel) {
profileImageExtractionEnabled === false
? undefined
: visualEmbeddingMode === "profile"
? frozenEmbeddingProfile?.model
: visualEmbeddingMode === "disabled"
? undefined
: visualEmbeddingModel;
if (
profileImageExtractionEnabled !== false &&
visualEmbeddingMode === "profile" &&
!resolvedVisualEmbeddingModel
) {
throw new Error("Profile-driven visual embedding requires a frozen embedding profile");
}
const reindexResult = await reindexer.reindex({
@ -985,7 +1003,7 @@ export function createDocumentCompilationWorker({
tenantId: input.tenantId,
...(resolvedVisualEmbeddingModel
? { visualModel: resolvedVisualEmbeddingModel }
: visualEmbeddingMode === "disabled"
: visualEmbeddingMode === "disabled" || profileImageExtractionEnabled === false
? { skipVisual: true as const }
: {}),
});
@ -1307,30 +1325,6 @@ function isPdfDocument(mimeType: string): boolean {
return mimeType.split(";", 1)[0]?.trim().toLowerCase() === "application/pdf";
}
function documentParserHints(input: {
readonly assetMetadata: Readonly<Record<string, unknown>>;
readonly imagesHandledExternally: boolean;
readonly requiresImages: boolean;
}): ParserRouteHints {
const language =
typeof input.assetMetadata.language === "string" && input.assetMetadata.language.trim()
? input.assetMetadata.language.trim()
: undefined;
const layoutComplexity =
input.assetMetadata.layoutComplexity === "complex" ||
input.assetMetadata.layoutComplexity === "simple"
? input.assetMetadata.layoutComplexity
: undefined;
return {
imagesHandledExternally: input.imagesHandledExternally,
...(language ? { language } : {}),
...(layoutComplexity ? { layoutComplexity } : {}),
requiresImages: input.requiresImages,
...(input.assetMetadata.requiresOcr === true ? { requiresOcr: true } : {}),
...(input.assetMetadata.requiresTables === true ? { requiresTables: true } : {}),
};
}
type ParseCheckpointRoute = "primary" | "provider-fallback";
interface ParserCheckpointPolicy {

42
knowledge-fs/packages/api/src/document-image-variant-generator.test.ts
View File

@ -3,6 +3,48 @@ import { describe, expect, it } from "vitest";
import { createSharpImageThumbnailVariantGenerator } from "./document-image-variant-generator";
describe("createSharpImageThumbnailVariantGenerator", () => {
it.each([0, 319, 4097, Number.NaN])(
"rejects unsafe analysis dimension %s",
(analysisMaxDimension) => {
expect(() => createSharpImageThumbnailVariantGenerator({ analysisMaxDimension })).toThrow(
"Sharp image analysis dimension",
);
},
);
it("requires distinct variant names and skips non-image/empty input", async () => {
expect(() =>
createSharpImageThumbnailVariantGenerator({
analysisMaxDimension: 640,
variantName: "analysis",
}),
).toThrow("distinct variant names");
const generator = createSharpImageThumbnailVariantGenerator({ analysisMaxDimension: 640 });
await expect(
generator.generate({
body: new Uint8Array([1]),
contentType: "text/plain",
elementId: "text",
}),
).resolves.toEqual([]);
await expect(
generator.generate({ body: new Uint8Array(), contentType: "image/png", elementId: "empty" }),
).resolves.toEqual([]);
});
it("keeps a separate analysis image with enough detail instead of feeding the preview thumbnail to vision", async () => {
const sharp = (await import("sharp")).default;
const body = await sharp({
create: { background: "white", channels: 3, width: 800, height: 400 },
})
.png()
.toBuffer();
const variants = await createSharpImageThumbnailVariantGenerator({
analysisMaxDimension: 640,
}).generate({ body, contentType: "image/png", elementId: "figure" });
expect(variants.map(({ name, width, height }) => ({ name, width, height }))).toEqual([
{ name: "thumbnail", width: 320, height: 160 },
{ name: "analysis", width: 640, height: 320 },
]);
});
it("generates bounded PNG thumbnail variants from image bytes", async () => {
const sharp = (await import("sharp")).default;
const generator = createSharpImageThumbnailVariantGenerator({

58
knowledge-fs/packages/api/src/document-image-variant-generator.ts
View File

@ -2,9 +2,19 @@ export interface GenerateDocumentImageVariantsInput {
readonly body: Uint8Array;
readonly contentType: string;
readonly elementId: string;
readonly signal?: AbortSignal | undefined;
}
export interface GeneratedDocumentImageVariant {
readonly execution?:
| {
readonly isolation: "child-process";
readonly inputBytes: number;
readonly outputBytes: number;
readonly wallMs: number;
readonly peakRssKiB: number;
}
| undefined;
readonly body: Uint8Array;
readonly contentType: string;
readonly height?: number | undefined;
@ -19,6 +29,8 @@ export interface DocumentImageVariantGenerator {
}
export interface SharpImageThumbnailVariantGeneratorOptions {
/** Separate vision input; preview dimensions must never become model input resolution. */
readonly analysisMaxDimension?: number | undefined;
readonly maxDimension?: number | undefined;
readonly maxInputPixels?: number | undefined;
readonly maxOutputBytes?: number | undefined;
@ -31,6 +43,7 @@ const defaultThumbnailMaxOutputBytes = 8 * 1024 * 1024;
const defaultThumbnailVariantName = "thumbnail";
export function createSharpImageThumbnailVariantGenerator({
analysisMaxDimension,
maxDimension = defaultThumbnailMaxDimension,
maxInputPixels = defaultThumbnailMaxInputPixels,
maxOutputBytes = defaultThumbnailMaxOutputBytes,
@ -52,8 +65,50 @@ export function createSharpImageThumbnailVariantGenerator({
throw new Error("Sharp image thumbnail variantName must be non-empty");
}
if (
analysisMaxDimension !== undefined &&
(!Number.isSafeInteger(analysisMaxDimension) ||
analysisMaxDimension < maxDimension ||
analysisMaxDimension > 4096 ||
variantName === "analysis")
) {
throw new Error(
"Sharp image analysis dimension must be between thumbnail dimension and 4096, with distinct variant names",
);
}
if (analysisMaxDimension !== undefined) {
const analysisGenerator = createSharpImageThumbnailVariantGenerator({
maxDimension: analysisMaxDimension,
maxInputPixels,
maxOutputBytes,
variantName: "analysis",
});
const previewGenerator = createSharpImageThumbnailVariantGenerator({
maxDimension,
maxInputPixels,
maxOutputBytes,
variantName,
});
return {
generate: async (input) => {
input.signal?.throwIfAborted();
const analysis = await analysisGenerator.generate(input);
const main = analysis[0];
if (!main) return [];
const preview = await previewGenerator.generate({
...input,
body: main.body,
contentType: main.contentType,
});
return [...preview, ...analysis];
},
};
}
return {
generate: async ({ body, contentType }) => {
generate: async ({ body, contentType, signal }) => {
signal?.throwIfAborted();
if (!contentType.toLowerCase().startsWith("image/") || body.byteLength === 0) {
return [];
}
@ -69,6 +124,7 @@ export function createSharpImageThumbnailVariantGenerator({
})
.png()
.toBuffer({ resolveWithObject: true });
signal?.throwIfAborted();
if (data.byteLength > maxOutputBytes) {
throw new Error(

2
knowledge-fs/packages/api/src/document-knowledge-paths.test.ts
View File

@ -107,6 +107,7 @@ describe("document KnowledgeFS paths", () => {
).toEqual([
expect.objectContaining({
metadata: expect.objectContaining({
analysisUnavailable: { reason: "variant-pixel-budget" },
assetContentType: "image/png",
contentKind: "document-multimodal-asset",
filename: "image-架构图--018f0d60.json",
@ -400,6 +401,7 @@ function documentMultimodalManifest(
items: [
{
assetRef: {
analysisUnavailable: { reason: "variant-pixel-budget" },
contentType: "image/png",
objectKey:
"tenant-dev/spaces/ab96d0c6-5853-4979-ba1a-98128e54fa7d/documents/6c07b8ca-dd64-4ccd-a6e9-91f816795412/assets/figure-1.png",

6
knowledge-fs/packages/api/src/document-knowledge-paths.ts
View File

@ -144,6 +144,9 @@ export function buildDocumentMultimodalAssetKnowledgePaths({
}),
knowledgeSpaceId: asset.knowledgeSpaceId,
metadata: {
...(item.assetRef?.analysisUnavailable
? { analysisUnavailable: { ...item.assetRef.analysisUnavailable } }
: {}),
...(item.assetRef?.contentType ? { assetContentType: item.assetRef.contentType } : {}),
...(item.assetRef?.objectKey ? { objectKey: item.assetRef.objectKey } : {}),
...(item.assetRef?.sha256 ? { sha256: item.assetRef.sha256 } : {}),
@ -398,6 +401,9 @@ function buildDocumentMultimodalItemResourceKnowledgePath({
}),
knowledgeSpaceId: asset.knowledgeSpaceId,
metadata: {
...(item.assetRef?.analysisUnavailable
? { analysisUnavailable: { ...item.assetRef.analysisUnavailable } }
: {}),
...(item.assetRef?.contentType ? { assetContentType: item.assetRef.contentType } : {}),
...(item.assetRef?.objectKey ? { objectKey: item.assetRef.objectKey } : {}),
...(item.assetRef?.sha256 ? { sha256: item.assetRef.sha256 } : {}),

259
knowledge-fs/packages/api/src/document-media-analysis-safety.test.ts Normal file
View File

@ -0,0 +1,259 @@
import { createNodePlatformAdapter } from "@knowledge/adapters/node";
import { createTypeScriptComputeRuntime } from "@knowledge/compute";
import { DocumentMultimodalAssetRefSchema, type ParseArtifact } from "@knowledge/core";
import { describe, expect, it, vi } from "vitest";
import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor";
import { createUnderstandingDocumentMultimodalEnrichmentProvider } from "./document-multimodal-enrichment-providers";
import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder";
import { createDocumentMultimodalManifestEnhancer } from "./document-multimodal-manifest-enhancer";
import { createObjectStorageVisualEmbeddingProvider } from "./index-projection-builders";
import {
createContentBlockMultimodalAnswerProvider,
createObjectStorageContentBlockMultimodalAnswerProvider,
} from "./llm-multimodal-answer-provider";
const knowledgeSpaceId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42";
const rejected = { reason: "variant-pixel-budget" as const };
const assetRef = {
analysisUnavailable: rejected,
contentType: "image/png",
objectKey: "tenant/assets/image.png",
};
describe("rejected document image analysis", () => {
it.each([
["count", ""],
["bytes", ""],
["count", " \n\t"],
["bytes", " \n\t"],
])(
"keeps overlong inline sources recoverable but bounded in manifests after %s limits with prefix %j",
async (limit, prefix) => {
const uri = `${prefix}data:image/png;base64,${Buffer.alloc(4096).toString("base64")}`;
const artifact: ParseArtifact = {
artifactHash: "a".repeat(64),
contentType: "mixed",
createdAt: "2026-06-23T00:00:00.000Z",
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43",
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
elements: ["data:image/png;base64,AQID", uri].map((source, index) => ({
id: `image-${index}`,
type: "image",
sectionPath: [],
metadata: { assetRef: { uri: source } },
})),
metadata: {},
parser: "native-markdown",
version: 1,
};
const extracted = await extractDocumentMultimodalAssets({
artifact,
knowledgeSpaceId,
tenantId: "tenant",
objectStorage: createNodePlatformAdapter({ env: {} }).objectStorage,
...(limit === "count" ? { maxExtractedAssets: 1 } : { maxTotalAssetBytes: 3 }),
});
expect(extracted.artifact.elements[1]?.metadata.assetRef).toMatchObject({ uri });
const manifest = createDocumentMultimodalManifestBuilder().build({
artifact: extracted.artifact,
knowledgeSpaceId,
});
const item = manifest.items[1];
expect(item?.assetRef).toEqual({
analysisUnavailable: {
reason: limit === "count" ? "asset-count-budget" : "materialized-byte-budget",
},
});
expect(item?.enrichment.asset).toBe("missing");
expect(item?.sourceMetadata.assetRef).toMatchObject({
sourceUriSha256: expect.stringMatching(/^[a-f0-9]{64}$/u),
});
expect(JSON.stringify(manifest)).not.toContain(uri);
const enriched = await createDocumentMultimodalManifestEnhancer({
maxItems: 2,
maxSourceTextChars: 100,
model: "vision",
promptVersion: "v1",
provider: {
enrich: async () => ({
caption: "Text-only metadata",
visualEmbeddingStatus: "provided",
}),
},
}).enhance({ manifest, parseArtifact: extracted.artifact });
expect(enriched.items[1]?.enrichment).toMatchObject({
asset: "missing",
visualEmbedding: "unsupported",
});
},
);
it("bounds the reason contract and preserves the marker through extraction, manifest, and nodes", async () => {
expect(DocumentMultimodalAssetRefSchema.parse(assetRef)).toEqual(assetRef);
expect(
DocumentMultimodalAssetRefSchema.safeParse({
...assetRef,
analysisUnavailable: { reason: "x".repeat(5000) },
}).success,
).toBe(false);
const objectStorage = createNodePlatformAdapter({ env: {} }).objectStorage;
const parseArtifact: ParseArtifact = {
artifactHash: "a".repeat(64),
contentType: "mixed",
createdAt: "2026-06-23T00:00:00.000Z",
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43",
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
elements: [
{
id: "image-1",
type: "image",
sectionPath: [],
text: "Readable caption",
metadata: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } },
},
],
metadata: {},
parser: "native-markdown",
version: 1,
};
const { artifact } = await extractDocumentMultimodalAssets({
artifact: parseArtifact,
objectStorage,
knowledgeSpaceId,
tenantId: "tenant",
maxTotalVariantPixels: 1,
imageVariantGenerator: { generate: vi.fn() },
});
const manifest = createDocumentMultimodalManifestBuilder().build({
artifact,
knowledgeSpaceId,
});
const item = manifest.items[0];
expect(item?.assetRef).toMatchObject({
analysisUnavailable: rejected,
objectKey: expect.any(String),
});
expect(item?.enrichment.visualEmbedding).toBe("unsupported");
const nodes = createTypeScriptComputeRuntime().chunkParseArtifact({
parseArtifact: artifact,
knowledgeSpaceId,
});
expect(nodes[0]?.metadata.assetRef).toMatchObject({ analysisUnavailable: rejected });
expect(nodes[0]?.text).toBe("Readable caption");
const originalKey = item?.assetRef?.objectKey;
const node = nodes[0];
if (!originalKey || !node) throw new Error("Expected retained original and text node");
expect(await objectStorage.getObject(originalKey)).not.toBeNull();
const embedImages = vi.fn();
const embedding = createObjectStorageVisualEmbeddingProvider({
objectStorage,
provider: { embedImages },
});
const result = await embedding.embedAssets({
model: "vision",
assets: [
{
assetRef: node.metadata.assetRef as Record<string, unknown>,
documentAssetId: artifact.documentAssetId,
metadata: {},
modality: "image",
nodeId: node.id,
sourceText: "Readable caption",
},
],
});
expect(result.dense).toEqual([]);
expect(embedImages).not.toHaveBeenCalled();
});
it.each(["object", "url"])(
"does not load rejected originals in %s answer consumers",
async (kind) => {
const objectStorage = createNodePlatformAdapter({ env: {} }).objectStorage;
await objectStorage.putObject({
key: assetRef.objectKey,
body: new Uint8Array([1, 2, 3]),
contentType: "image/png",
});
const get = vi.spyOn(objectStorage, "getObjectStream");
const generate = vi.fn(async () => ({ text: "Answer from caption" }));
const resolve = vi.fn(() => "https://example.com/original.png");
const provider =
kind === "object"
? createObjectStorageContentBlockMultimodalAnswerProvider({
model: "vision",
objectStorage,
provider: { generate },
})
: createContentBlockMultimodalAnswerProvider({
model: "vision",
assetUrlResolver: resolve,
provider: { generate },
});
await expect(
provider.generate({
query: "What does it show?",
evidence: [],
multimodalEvidence: [
{
assetRef,
documentAssetId: "doc-1",
modality: "image",
parseElementId: "image-1",
sectionPath: [],
caption: "Readable caption",
},
],
}),
).resolves.toMatchObject({ metadata: { imageBlockCount: 0 } });
expect(get).not.toHaveBeenCalled();
expect(resolve).not.toHaveBeenCalled();
expect(JSON.stringify(generate.mock.calls)).toContain("parseElement=image-1");
},
);
it("does not delegate flagged images to understanding providers or let enrichment clear the marker", async () => {
const artifact: ParseArtifact = {
artifactHash: "a".repeat(64),
contentType: "mixed",
createdAt: "2026-06-23T00:00:00.000Z",
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43",
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
elements: [{ id: "image-1", type: "image", sectionPath: [], metadata: { assetRef } }],
metadata: {},
parser: "native-markdown",
version: 1,
};
const manifest = createDocumentMultimodalManifestBuilder().build({
artifact,
knowledgeSpaceId,
});
const understand = vi.fn(async () => ({ caption: "Should not execute" }));
const enhancer = createDocumentMultimodalManifestEnhancer({
maxItems: 1,
maxSourceTextChars: 100,
model: "vision",
promptVersion: "v1",
provider: createUnderstandingDocumentMultimodalEnrichmentProvider({
provider: { understand },
}),
});
await enhancer.enhance({ manifest, parseArtifact: artifact });
expect(understand).not.toHaveBeenCalled();
const rewriting = createDocumentMultimodalManifestEnhancer({
maxItems: 1,
maxSourceTextChars: 100,
model: "vision",
promptVersion: "v1",
provider: {
enrich: async () => ({
assetRef: { objectKey: assetRef.objectKey, contentType: "image/png" },
caption: "Text-only caption",
}),
},
});
expect(
(await rewriting.enhance({ manifest, parseArtifact: artifact })).items[0]?.assetRef,
).toMatchObject({ analysisUnavailable: rejected });
});
});

136
knowledge-fs/packages/api/src/document-media-execution-plan.test.ts Normal file
View File

@ -0,0 +1,136 @@
import { describe, expect, it } from "vitest";
import {
createDocumentMediaExecutionPlan,
createProfileDocumentMediaCapabilityResolver,
documentParserHints,
withDocumentMediaExecutionPlan,
} from "./document-media-execution-plan";
import type { KnowledgeSpaceProfileHead } from "./knowledge-space-profile-repository";
describe("immutable document media execution plan", () => {
it("persists the immutable plan without replacing other completeness diagnostics", () => {
const original = {
metadata: { parseCoverage: { text: { status: "complete" }, media: { status: "unknown" } } },
} as unknown as import("@knowledge/core").ParseArtifact;
const plan = createDocumentMediaExecutionPlan({
hasPdfRasterizer: false,
hasImageVariantGenerator: false,
visualEmbeddingEnabled: true,
});
expect(withDocumentMediaExecutionPlan(original, plan).metadata).toMatchObject({
mediaExecutionPlan: plan,
parseCoverage: original.metadata.parseCoverage,
});
expect(
withDocumentMediaExecutionPlan(original, { ...plan, materializeImages: false }).metadata,
).toMatchObject({
parseCoverage: { text: { status: "complete" }, media: { status: "not-requested" } },
});
expect(withDocumentMediaExecutionPlan({ ...original, metadata: {} }, plan).metadata).toEqual({
mediaExecutionPlan: plan,
});
});
it("keeps source references but disables rasterization and decoding for a text-only profile", () => {
const plan = createDocumentMediaExecutionPlan({
profileImageExtractionEnabled: false,
hasPdfRasterizer: true,
hasImageVariantGenerator: true,
visualEmbeddingEnabled: false,
});
expect(plan).toMatchObject({
preserveSourceReferences: true,
materializeImages: false,
rasterizePdf: false,
generateVariants: false,
visualEmbedding: false,
});
expect(Object.isFrozen(plan)).toBe(true);
});
it("enables vision materialization independently of available embedding support", () => {
expect(
createDocumentMediaExecutionPlan({
profileImageExtractionEnabled: true,
hasPdfRasterizer: true,
hasImageVariantGenerator: true,
visualEmbeddingEnabled: false,
}),
).toMatchObject({
materializeImages: true,
rasterizePdf: true,
generateVariants: true,
visualEmbedding: false,
});
});
it("preserves legacy inline-image materialization without explicitly configured profiles", () => {
expect(
createDocumentMediaExecutionPlan({
hasPdfRasterizer: false,
hasImageVariantGenerator: false,
visualEmbeddingEnabled: false,
}),
).toMatchObject({ materializeImages: true, rasterizePdf: false, generateVariants: false });
});
it("captures identical bounded parser hints for synchronous and durable compilation", () => {
expect(
documentParserHints({
assetMetadata: {
language: " zh ",
layoutComplexity: "complex",
requiresOcr: true,
requiresTables: true,
},
imagesHandledExternally: false,
requiresImages: true,
}),
).toEqual({
language: "zh",
layoutComplexity: "complex",
requiresOcr: true,
requiresTables: true,
imagesHandledExternally: false,
requiresImages: true,
});
expect(
documentParserHints({
assetMetadata: { language: 1, layoutComplexity: "invalid" },
imagesHandledExternally: false,
requiresImages: false,
}),
).toEqual({ imagesHandledExternally: false, requiresImages: false });
});
it("resolves synchronous capabilities once from the space's immutable profile snapshots", async () => {
const resolver = createProfileDocumentMediaCapabilityResolver({
profiles: {
getHead: async ({ kind }) =>
({
profile: {
capabilitySnapshot:
kind === "embedding" ? { image: false } : { reasoning: { image: true } },
},
}) as unknown as KnowledgeSpaceProfileHead,
},
modalities: {
resolve: async ({ snapshot }) =>
(snapshot as { image: boolean }).image ? ["text", "image"] : ["text"],
},
});
await expect(resolver({ knowledgeSpaceId: "space", tenantId: "tenant" })).resolves.toEqual({
imageExtractionEnabled: true,
visualEmbeddingEnabled: false,
});
});
it("fails closed when neither profile contains a vision capability", async () => {
const resolver = createProfileDocumentMediaCapabilityResolver({
profiles: { getHead: async () => null },
modalities: {
resolve: async () => {
throw new Error("No snapshot should resolve");
},
},
});
await expect(resolver({ knowledgeSpaceId: "space", tenantId: "tenant" })).resolves.toEqual({
imageExtractionEnabled: false,
visualEmbeddingEnabled: false,
});
});
});

130
knowledge-fs/packages/api/src/document-media-execution-plan.ts Normal file
View File

@ -0,0 +1,130 @@
import type { ParseArtifact } from "@knowledge/core";
import type { ParserRouteHints } from "@knowledge/parsers";
import { isPlainObject } from "./json-utils";
import type { KnowledgeSpaceProfileRepository } from "./knowledge-space-profile-repository";
import type { ModelInputModality } from "./model-capability-preflight";
import type { ModelInputModalityResolver } from "./model-input-modality-resolver";
export type DocumentMediaCapabilityResolver = (input: {
readonly knowledgeSpaceId: string;
readonly tenantId: string;
readonly signal?: AbortSignal | undefined;
}) => Promise<{
readonly imageExtractionEnabled: boolean;
readonly visualEmbeddingEnabled: boolean;
}>;
/** Legacy synchronous ingestion has no publication attempt: freeze its two heads before parsing. */
export function createProfileDocumentMediaCapabilityResolver(input: {
readonly profiles: Pick<KnowledgeSpaceProfileRepository, "getHead">;
readonly modalities: ModelInputModalityResolver;
}): DocumentMediaCapabilityResolver {
return async ({ knowledgeSpaceId, tenantId, signal }) => {
signal?.throwIfAborted();
const [embedding, retrieval] = await Promise.all([
input.profiles.getHead({ knowledgeSpaceId, tenantId, kind: "embedding" }),
input.profiles.getHead({ knowledgeSpaceId, tenantId, kind: "retrieval" }),
]);
const [embeddingModalities, reasoningModalities] = await Promise.all([
embedding
? input.modalities.resolve({
snapshot: embedding.profile.capabilitySnapshot,
tenantId,
signal,
})
: ([] as readonly ModelInputModality[]),
retrieval?.profile.capabilitySnapshot.reasoning
? input.modalities.resolve({
snapshot: retrieval.profile.capabilitySnapshot.reasoning,
tenantId,
signal,
})
: ([] as readonly ModelInputModality[]),
]);
signal?.throwIfAborted();
const visualEmbeddingEnabled = embeddingModalities.includes("image");
return Object.freeze({
imageExtractionEnabled: visualEmbeddingEnabled || reasoningModalities.includes("image"),
visualEmbeddingEnabled,
});
};
}
export interface DocumentMediaExecutionPlan {
readonly version: "document-media-v1";
readonly preserveSourceReferences: true;
readonly materializeImages: boolean;
readonly requestParserImages: boolean;
readonly rasterizePdf: boolean;
readonly generateVariants: boolean;
readonly visualEmbedding: boolean;
}
/** Capabilities are captured once; installed tools must not override an explicit text-only profile. */
export function createDocumentMediaExecutionPlan(input: {
readonly profileImageExtractionEnabled?: boolean | undefined;
readonly hasPdfRasterizer: boolean;
readonly hasImageVariantGenerator: boolean;
readonly visualEmbeddingEnabled: boolean;
}): DocumentMediaExecutionPlan {
const materializeImages = input.profileImageExtractionEnabled !== false;
return Object.freeze({
version: "document-media-v1",
preserveSourceReferences: true,
materializeImages,
requestParserImages:
input.profileImageExtractionEnabled ??
Boolean(
input.hasPdfRasterizer || input.hasImageVariantGenerator || input.visualEmbeddingEnabled,
),
rasterizePdf: materializeImages && input.hasPdfRasterizer,
generateVariants: materializeImages && input.hasImageVariantGenerator,
visualEmbedding: materializeImages && input.visualEmbeddingEnabled,
});
}
export function documentParserHints(input: {
readonly assetMetadata: Readonly<Record<string, unknown>>;
readonly imagesHandledExternally: boolean;
readonly requiresImages: boolean;
}): ParserRouteHints {
const language =
typeof input.assetMetadata.language === "string" ? input.assetMetadata.language.trim() : "";
const layoutComplexity =
input.assetMetadata.layoutComplexity === "complex" ||
input.assetMetadata.layoutComplexity === "simple"
? input.assetMetadata.layoutComplexity
: undefined;
return Object.freeze({
imagesHandledExternally: input.imagesHandledExternally,
...(language ? { language } : {}),
...(layoutComplexity ? { layoutComplexity } : {}),
requiresImages: input.requiresImages,
...(input.assetMetadata.requiresOcr === true ? { requiresOcr: true } : {}),
...(input.assetMetadata.requiresTables === true ? { requiresTables: true } : {}),
});
}
export function withDocumentMediaExecutionPlan(
artifact: ParseArtifact,
plan: DocumentMediaExecutionPlan,
): ParseArtifact {
const coverage = isPlainObject(artifact.metadata.parseCoverage)
? artifact.metadata.parseCoverage
: {};
return {
...artifact,
metadata: {
...artifact.metadata,
mediaExecutionPlan: { ...plan },
...(!plan.materializeImages
? {
parseCoverage: {
...coverage,
media: { status: "not-requested", reasons: ["profile-text-only"] },
},
}
: {}),
},
};
}

397
knowledge-fs/packages/api/src/document-multimodal-asset-budget.test.ts Normal file
View File

@ -0,0 +1,397 @@
import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { createNodePlatformAdapter } from "@knowledge/adapters/node";
import type { ParseArtifact } from "@knowledge/core";
import { describe, expect, it, vi } from "vitest";
import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor";
import { createDocumentRemoteMediaBudget } from "./document-remote-media-budget";
const options = () => ({
knowledgeSpaceId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42",
objectStorage: createNodePlatformAdapter({ env: {} }).objectStorage,
tenantId: "tenant-1",
});
function artifact(uris: string[]): ParseArtifact {
return {
artifactHash: "a".repeat(64),
contentType: "mixed",
createdAt: "2026-06-23T00:00:00.000Z",
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43",
id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
elements: uris.map((uri, index) => ({
id: `image-${index}`,
type: "image",
sectionPath: [],
metadata: { assetRef: { uri }, caption: `Source ${index}` },
})),
metadata: {},
parser: "native-markdown",
version: 1,
};
}
describe("document-wide media budgets", () => {
it("validates the remote budget's per-image cap independently of its caller", () => {
expect(() =>
createDocumentRemoteMediaBudget({
maxAttempts: 1,
maxBytes: 0,
maxTotalBytes: 1,
timeoutMs: 1,
}),
).toThrow("maxRemoteAssetBytes");
});
it("charges derived image bytes to the same document materialization budget", async () => {
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA=="]),
maxTotalAssetBytes: 8,
imageVariantGenerator: {
generate: async () => [
{ name: "thumbnail", contentType: "image/png", body: new Uint8Array([1, 2, 3]) },
{ name: "analysis", contentType: "image/png", body: new Uint8Array([1, 2, 3]) },
],
},
});
expect(result.artifact.elements[0]?.metadata.assetRef).not.toHaveProperty("variants");
expect(result.artifact.elements[0]?.metadata.assetRef).toHaveProperty("analysisUnavailable", {
reason: "materialized-byte-budget",
});
expect(result.artifact.metadata).toMatchObject({
multimodalAssets: { materializedBytes: 4 },
parseCoverage: { media: { reasons: ["materialized-byte-budget"] } },
});
});
it.each([
new Error("decoder transport unavailable"),
Object.assign(new Error("retry decoder"), { code: "provider_input", retryable: true }),
Object.assign(new Error("service unavailable"), {
code: "provider_request_failed",
retryable: false,
}),
])("propagates transient or unrelated variant failures", async (error) => {
await expect(
extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA=="]),
imageVariantGenerator: {
generate: async () => {
throw error;
},
},
}),
).rejects.toBe(error);
});
it("does not accept a variant returned after its deadline", async () => {
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA=="]),
maxVariantDurationMs: 5,
imageVariantGenerator: {
generate: async () => {
await new Promise((resolve) => setTimeout(resolve, 10));
return [];
},
},
});
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { reasons: ["variant-deadline"] } },
});
});
it("propagates lease loss during variant generation", async () => {
const controller = new AbortController();
await expect(
extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA=="]),
signal: controller.signal,
imageVariantGenerator: {
generate: async () => {
controller.abort(new Error("lease lost during decode"));
throw controller.signal.reason;
},
},
}),
).rejects.toThrow("lease lost during decode");
});
it("does not materialize a single image larger than the remaining byte budget", async () => {
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA=="]),
maxTotalAssetBytes: 3,
});
expect(result.extractedCount).toBe(0);
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { reasons: ["materialized-byte-budget"] } },
});
});
it("ignores directory-shaped image paths and unavailable allowlisted roots", async () => {
const root = await mkdtemp(join(tmpdir(), "kfs-media-directory-"));
try {
const directory = join(root, "folder.png");
await mkdir(directory);
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact([directory]),
allowLocalAssetPaths: [root, join(root, "missing")],
});
expect(result.extractedCount).toBe(0);
} finally {
await rm(root, { recursive: true, force: true });
}
});
it("cancels variant work on its document deadline and does not start more decoders", async () => {
const generate = vi.fn(
async ({ signal }: { signal?: AbortSignal | undefined }) =>
new Promise<never>((_resolve, reject) => {
if (signal?.aborted) reject(signal.reason);
signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
}),
);
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA==", "data:image/png;base64,AQIDBA=="]),
maxVariantDurationMs: 5,
imageVariantGenerator: { generate },
});
expect(generate).toHaveBeenCalledTimes(1);
expect(result.extractedCount).toBe(2);
expect(result.artifact.elements[0]?.metadata.assetRef).toHaveProperty("analysisUnavailable", {
reason: "variant-deadline",
});
expect(result.artifact.elements[1]?.metadata.assetRef).toHaveProperty("analysisUnavailable", {
reason: "variant-deadline",
});
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { status: "partial", reasons: ["variant-deadline"] } },
});
});
it("propagates cancellation of in-flight downloads rather than treating it as missing media", async () => {
const controller = new AbortController();
const fetch = vi.fn(async () => {
controller.abort(new Error("lease lost during download"));
return null;
});
await expect(
extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["https://a.test/1.png"]),
signal: controller.signal,
remoteAssetFetcher: { fetch },
}),
).rejects.toThrow("lease lost during download");
});
it.each([
"maxRemoteAssetAttempts",
"maxTotalRemoteAssetBytes",
"remoteAssetTimeoutMs",
"maxTotalAssetBytes",
"maxVariantPixels",
"maxTotalVariantPixels",
"maxVariantDurationMs",
])("rejects invalid %s", async (field) => {
await expect(
extractDocumentMultimodalAssets({ ...options(), artifact: artifact([]), [field]: 0 }),
).rejects.toThrow("must be at least 1");
});
it("records unavailable remote capability without losing references or earlier coverage", async () => {
const source = artifact(["https://a.test/image.png"]);
source.metadata = {
parseCoverage: {
text: { status: "complete", reasons: [] },
media: { status: "partial", reasons: ["unsupported-media"] },
},
};
const result = await extractDocumentMultimodalAssets({ ...options(), artifact: source });
expect(result.artifact.elements).toEqual(source.elements);
expect(result.artifact.metadata).toMatchObject({
parseCoverage: {
text: { status: "complete" },
media: { status: "partial", reasons: ["unsupported-media", "remote-fetcher-unavailable"] },
},
});
});
it("preserves originals without decoding variants beyond the aggregate pixel budget", async () => {
const png = new Uint8Array([
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 2, 0, 0, 0, 3,
]);
const generate = vi.fn(async () => [
{ body: new Uint8Array([1]), contentType: "image/png", name: "thumbnail" },
]);
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact([
`data:image/png;base64,${Buffer.from(png).toString("base64")}`,
`data:image/png;base64,${Buffer.from(png).toString("base64")}`,
]),
maxTotalVariantPixels: 6,
imageVariantGenerator: { generate },
});
expect(generate).toHaveBeenCalledTimes(1);
expect(result.extractedCount).toBe(2);
expect(result.artifact.elements[0]?.metadata.assetRef).not.toHaveProperty(
"analysisUnavailable",
);
expect(result.artifact.elements[1]?.metadata.assetRef).toHaveProperty("analysisUnavailable", {
reason: "variant-pixel-budget",
});
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { status: "partial", reasons: ["variant-pixel-budget"] } },
});
});
it("leaves images inline before exceeding the document's materialized byte cap", async () => {
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA==", "data:image/png;base64,AQIDBA=="]),
maxTotalAssetBytes: 4,
});
expect(result.extractedCount).toBe(1);
expect(result.artifact.elements[1]?.metadata.assetRef).toEqual({
analysisUnavailable: { reason: "materialized-byte-budget" },
uri: "data:image/png;base64,AQIDBA==",
});
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { status: "partial", reasons: ["materialized-byte-budget"] } },
});
});
it.each(["empty", "invalid"])("preserves the source when variants are %s", async (failure) => {
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["data:image/png;base64,AQIDBA=="]),
imageVariantGenerator: {
generate: async () => {
if (failure === "invalid")
throw Object.assign(new Error("Image decoder rejected input"), {
code: "provider_input",
retryable: false,
});
return [];
},
},
});
expect(result.extractedCount).toBe(1);
expect(result.artifact.elements[0]?.metadata.assetRef).toMatchObject({
analysisUnavailable: {
reason: failure === "empty" ? "variant-unavailable" : "variant-input-rejected",
},
objectKey: expect.any(String),
});
expect(result.artifact.metadata).toMatchObject({
parseCoverage: {
media: {
status: "partial",
reasons: [failure === "empty" ? "variant-unavailable" : "variant-input-rejected"],
},
},
});
});
it("counts failed unique URL attempts and preserves unresolved references", async () => {
const fetch = vi.fn(async () => null);
const source = artifact([
"https://a.test/1.png",
"https://a.test/2.png",
"https://a.test/3.png",
]);
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: source,
maxRemoteAssetAttempts: 2,
remoteAssetFetcher: { fetch },
});
expect(fetch).toHaveBeenCalledTimes(2);
expect(result.artifact.elements).toEqual(source.elements);
expect(result.artifact.metadata).toMatchObject({
parseCoverage: {
media: {
status: "partial",
reasons: expect.arrayContaining(["remote-attempt-budget", "remote-unavailable"]),
},
},
multimodalAssets: { remoteAttempts: 2, remoteDownloadedBytes: 0 },
});
});
it("fetches repeated URLs once while preserving every element and caption", async () => {
const fetch = vi.fn(async () => ({ body: new Uint8Array([1, 2]), contentType: "image/png" }));
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["https://a.test/1.png", "https://a.test/1.png"]),
remoteAssetFetcher: { fetch },
});
expect(fetch).toHaveBeenCalledTimes(1);
expect(result.extractedCount).toBe(2);
expect(result.artifact.elements.map((element) => element.metadata.caption)).toEqual([
"Source 0",
"Source 1",
]);
expect(result.artifact.metadata).toMatchObject({
multimodalAssets: { remoteAttempts: 1, remoteDownloadedBytes: 2 },
});
});
it("bounds aggregate remote bytes before requesting the next image", async () => {
const fetch = vi.fn(async () => ({ body: new Uint8Array([1, 2]), contentType: "image/png" }));
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["https://a.test/1.png", "https://a.test/2.png"]),
maxTotalRemoteAssetBytes: 2,
remoteAssetFetcher: { fetch },
});
expect(fetch).toHaveBeenCalledTimes(1);
expect(result.extractedCount).toBe(1);
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { status: "partial", reasons: ["remote-byte-budget"] } },
});
});
it("aborts a stalled fetch on the document-wide deadline and reports partial media", async () => {
const fetch = vi.fn(async () => new Promise<null>(() => {}));
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["https://a.test/1.png", "https://a.test/2.png"]),
remoteAssetTimeoutMs: 10,
remoteAssetFetcher: { fetch },
});
expect(fetch).toHaveBeenCalledTimes(1);
expect(result.artifact.metadata).toMatchObject({
parseCoverage: { media: { status: "partial", reasons: ["remote-deadline"] } },
});
});
it("does not start fetching after cancellation", async () => {
const fetch = vi.fn(async () => null);
const controller = new AbortController();
controller.abort(new Error("lease lost"));
await expect(
extractDocumentMultimodalAssets({
...options(),
artifact: artifact(["https://a.test/1.png"]),
signal: controller.signal,
remoteAssetFetcher: { fetch },
}),
).rejects.toThrow("lease lost");
expect(fetch).not.toHaveBeenCalled();
});
it("denies symlinks escaping an allowed local root", async () => {
const allowed = await mkdtemp(join(tmpdir(), "kfs-media-allowed-"));
const outside = await mkdtemp(join(tmpdir(), "kfs-media-outside-"));
try {
const privatePath = join(outside, "secret.png");
await writeFile(privatePath, new Uint8Array([1, 2]));
const link = join(allowed, "image.png");
await symlink(privatePath, link);
const result = await extractDocumentMultimodalAssets({
...options(),
artifact: artifact([link]),
allowLocalAssetPaths: [allowed],
});
expect(result.extractedCount).toBe(0);
expect(result.artifact.elements[0]?.metadata.assetRef).toEqual({ uri: link });
} finally {
await rm(allowed, { recursive: true, force: true });
await rm(outside, { recursive: true, force: true });
}
});
});

24
knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts
View File

@ -58,8 +58,16 @@ describe("extractDocumentMultimodalAssets", () => {
expect(result.extractedCount).toBe(2);
expect(fetchCalls).toEqual([
{ maxBytes: 1024, url: "https://cdn.example.test/markdown.png" },
{ maxBytes: 1024, url: "https://cdn.example.test/office.png" },
{
maxBytes: 1024,
signal: expect.any(AbortSignal),
url: "https://cdn.example.test/markdown.png",
},
{
maxBytes: 1024,
signal: expect.any(AbortSignal),
url: "https://cdn.example.test/office.png",
},
]);
for (const element of result.artifact.elements) {
expect(element.metadata).toMatchObject({
@ -127,7 +135,7 @@ describe("extractDocumentMultimodalAssets", () => {
expect(remoteFetches).toEqual([
{
maxBytes: 10 * 1024 * 1024,
signal: controller.signal,
signal: expect.any(AbortSignal),
url: "https://cdn.example.test/missing.png",
},
]);
@ -227,7 +235,10 @@ describe("extractDocumentMultimodalAssets", () => {
expect(result).toMatchObject({ extractedCount: 1, skippedForCapCount: 1 });
expect(fetchCalls).toEqual([]);
expect(result.artifact.elements[1]?.metadata).toEqual({
assetRef: { uri: "https://cdn.example.test/second.png" },
assetRef: {
uri: "https://cdn.example.test/second.png",
analysisUnavailable: { reason: "asset-count-budget" },
},
});
expect(result.artifact.elements[2]?.metadata).toEqual({
assetRef: { uri: "https://user:secret@cdn.example.test/private.png" },
@ -546,7 +557,10 @@ describe("extractDocumentMultimodalAssets", () => {
});
// The over-cap image is left inline (its data URI preserved), not extracted.
const figure2 = result.artifact.elements.find((element) => element.id === "figure-2");
expect(figure2?.metadata.assetRef).toEqual({ uri: "data:image/png;base64,BQYHCA==" });
expect(figure2?.metadata.assetRef).toEqual({
uri: "data:image/png;base64,BQYHCA==",
analysisUnavailable: { reason: "asset-count-budget" },
});
// Exactly one object was written (the extracted figure-1), so no orphan from figure-2.
const figure1 = result.artifact.elements.find((element) => element.id === "figure-1");
const objectKey = (figure1?.metadata.assetRef as { objectKey?: string } | undefined)?.objectKey;

330
knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts
View File

@ -1,14 +1,22 @@
import { createHash } from "node:crypto";
import { readFile, stat } from "node:fs/promises";
import { constants } from "node:fs";
import { open, realpath } from "node:fs/promises";
import { extname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { type ParseArtifact, ParseArtifactSchema, type PlatformAdapter } from "@knowledge/core";
import {
type DocumentMultimodalAssetRef,
type ParseArtifact,
ParseArtifactSchema,
type ParseElement,
type PlatformAdapter,
} from "@knowledge/core";
import type {
DocumentImageVariantGenerator,
GeneratedDocumentImageVariant,
} from "./document-image-variant-generator";
import { createDocumentRemoteMediaBudget } from "./document-remote-media-budget";
import { cloneJsonObject, isPlainObject } from "./json-utils";
import {
createDocumentMultimodalAssetObjectKey,
@ -23,6 +31,13 @@ export interface ExtractDocumentMultimodalAssetsInput {
readonly maxExtractedAssets?: number | undefined;
readonly maxLocalAssetBytes?: number | undefined;
readonly maxRemoteAssetBytes?: number | undefined;
readonly maxRemoteAssetAttempts?: number | undefined;
readonly maxTotalRemoteAssetBytes?: number | undefined;
readonly remoteAssetTimeoutMs?: number | undefined;
readonly maxTotalAssetBytes?: number | undefined;
readonly maxVariantPixels?: number | undefined;
readonly maxTotalVariantPixels?: number | undefined;
readonly maxVariantDurationMs?: number | undefined;
readonly imageVariantGenerator?: DocumentImageVariantGenerator | undefined;
readonly objectStorage: PlatformAdapter["objectStorage"];
readonly remoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined;
@ -58,6 +73,10 @@ interface ImageDimensions {
readonly width: number;
}
type AnalysisUnavailableReason = NonNullable<
DocumentMultimodalAssetRef["analysisUnavailable"]
>["reason"];
const dataUriPattern = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)$/iu;
const defaultMaxEmbeddedAssetBytes = 10 * 1024 * 1024;
const defaultMaxExtractedAssets = 1_000;
@ -78,6 +97,13 @@ export async function extractDocumentMultimodalAssets({
maxExtractedAssets = defaultMaxExtractedAssets,
maxLocalAssetBytes = defaultMaxLocalAssetBytes,
maxRemoteAssetBytes = defaultMaxRemoteAssetBytes,
maxRemoteAssetAttempts = 100,
maxTotalRemoteAssetBytes = 32 * 1024 * 1024,
remoteAssetTimeoutMs = 60_000,
maxTotalAssetBytes = 64 * 1024 * 1024,
maxVariantPixels = 20_000_000,
maxTotalVariantPixels = 100_000_000,
maxVariantDurationMs = 60_000,
imageVariantGenerator,
objectStorage,
remoteAssetFetcher,
@ -100,14 +126,38 @@ export async function extractDocumentMultimodalAssets({
if (!Number.isSafeInteger(maxRemoteAssetBytes) || maxRemoteAssetBytes < 1) {
throw new Error("Document multimodal remote asset max bytes must be at least 1");
}
for (const [name, value] of Object.entries({
maxTotalAssetBytes,
maxVariantPixels,
maxTotalVariantPixels,
maxVariantDurationMs,
})) {
if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be at least 1`);
}
let extractedCount = 0;
let skippedForCapCount = 0;
let materializedBytes = 0;
let variantPixels = 0;
const variantDeadline = performance.now() + maxVariantDurationMs;
const mediaReasons = new Set<string>();
const remoteBudget = createDocumentRemoteMediaBudget({
fetcher: remoteAssetFetcher,
maxAttempts: maxRemoteAssetAttempts,
maxBytes: maxRemoteAssetBytes,
maxTotalBytes: maxTotalRemoteAssetBytes,
signal,
timeoutMs: remoteAssetTimeoutMs,
});
const extractionSources = new Set<string>();
const elements = [];
const allowedLocalRoots = normalizeAllowedLocalAssetPaths(allowLocalAssetPaths);
const canonicalAllowedLocalRoots = (
await Promise.all(allowedLocalRoots.map((root) => realpath(root).catch(() => null)))
).filter((root): root is string => root !== null);
for (const element of artifact.elements) {
signal?.throwIfAborted();
if (element.type !== "image" && element.type !== "table") {
elements.push(element);
continue;
@ -115,32 +165,34 @@ export async function extractDocumentMultimodalAssets({
const assetRef = isPlainObject(element.metadata.assetRef) ? element.metadata.assetRef : null;
const uri = typeof assetRef?.uri === "string" ? assetRef.uri.trim() : "";
if (
extractedCount >= maxExtractedAssets &&
(dataUriPattern.test(uri) ||
isRemoteHttpUri(uri) ||
(allowedLocalRoots.length > 0 && localPathFromUri(uri)))
) {
skippedForCapCount += 1;
elements.push(withAnalysisUnavailable(element, "asset-count-budget"));
continue;
}
if (materializedBytes >= maxTotalAssetBytes && uri) {
mediaReasons.add("materialized-byte-budget");
elements.push(withAnalysisUnavailable(element, "materialized-byte-budget"));
continue;
}
let image =
parseDataUriImage(uri, maxEmbeddedAssetBytes) ??
(await readLocalImageAsset({
allowedRoots: allowedLocalRoots,
canonicalAllowedRoots: canonicalAllowedLocalRoots,
assetRef,
maxLocalAssetBytes,
uri,
}));
if (!image && remoteAssetFetcher && isRemoteHttpUri(uri)) {
if (extractedCount >= maxExtractedAssets) {
skippedForCapCount += 1;
elements.push(element);
continue;
}
const fetched = await remoteAssetFetcher.fetch({
maxBytes: maxRemoteAssetBytes,
...(signal ? { signal } : {}),
url: uri,
});
if (!image && isRemoteHttpUri(uri)) {
const fetched = await remoteBudget.fetch(uri);
if (fetched) {
if (fetched.body.byteLength > maxRemoteAssetBytes) {
throw new Error(
`Document multimodal remote asset exceeds maxRemoteAssetBytes=${maxRemoteAssetBytes}`,
);
}
const contentType = normalizeRemoteImageContentType(fetched.contentType);
if (!contentType) {
throw new Error("Document multimodal remote asset content type is unsupported");
@ -163,13 +215,12 @@ export async function extractDocumentMultimodalAssets({
continue;
}
if (extractedCount >= maxExtractedAssets) {
// Soft cap: leave the remaining extractable images inline instead of throwing (which would
// abort ingestion and orphan the assets already written to object storage this run).
skippedForCapCount += 1;
elements.push(element);
if (image.body.byteLength > maxTotalAssetBytes - materializedBytes) {
mediaReasons.add("materialized-byte-budget");
elements.push(withAnalysisUnavailable(element, "materialized-byte-budget"));
continue;
}
materializedBytes += image.body.byteLength;
const sha256 = sha256Hex(image.body);
const objectKey = createDocumentMultimodalAssetObjectKey({
@ -195,18 +246,47 @@ export async function extractDocumentMultimodalAssets({
...(writeOwnerId ? { writeOwnerId } : {}),
},
});
const variants = imageVariantGenerator
? await storeGeneratedImageVariants({
assetId: artifact.documentAssetId,
elementId: element.id,
generator: imageVariantGenerator,
image,
knowledgeSpaceId,
objectStorage,
tenantId,
...(writeOwnerId ? { writeOwnerId } : {}),
})
: {};
signal?.throwIfAborted();
let analysisUnavailableReason: AnalysisUnavailableReason | undefined;
const markAnalysisUnavailable = (reason: AnalysisUnavailableReason) => {
analysisUnavailableReason = reason;
mediaReasons.add(reason);
};
const imagePixels = image.dimensions
? image.dimensions.width * image.dimensions.height
: maxVariantPixels;
const canGenerateVariants =
imageVariantGenerator &&
imagePixels > 0 &&
Number.isSafeInteger(imagePixels) &&
imagePixels <= maxVariantPixels &&
imagePixels <= maxTotalVariantPixels - variantPixels;
if (imageVariantGenerator && !canGenerateVariants)
markAnalysisUnavailable("variant-pixel-budget");
if (canGenerateVariants) variantPixels += imagePixels;
const variantTimeRemaining = Math.ceil(variantDeadline - performance.now());
if (canGenerateVariants && variantTimeRemaining <= 0)
markAnalysisUnavailable("variant-deadline");
const variants =
canGenerateVariants && variantTimeRemaining > 0
? await storeGeneratedImageVariants({
assetId: artifact.documentAssetId,
elementId: element.id,
generator: imageVariantGenerator,
image,
knowledgeSpaceId,
objectStorage,
onUnavailable: markAnalysisUnavailable,
onMaterializedBytes: (bytes) => {
materializedBytes += bytes;
},
remainingBytes: maxTotalAssetBytes - materializedBytes,
timeoutMs: variantTimeRemaining,
tenantId,
...(signal ? { signal } : {}),
...(writeOwnerId ? { writeOwnerId } : {}),
})
: {};
extractedCount += 1;
extractionSources.add(image.source);
@ -220,6 +300,9 @@ export async function extractDocumentMultimodalAssets({
...cloneJsonObject(element.metadata),
assetRef: {
...remainingAssetRef,
...(analysisUnavailableReason
? { analysisUnavailable: { reason: analysisUnavailableReason } }
: {}),
contentType: image.contentType,
...(image.dimensions ? image.dimensions : {}),
objectKey,
@ -241,7 +324,12 @@ export async function extractDocumentMultimodalAssets({
});
}
if (extractedCount === 0) {
if (
extractedCount === 0 &&
remoteBudget.reasons.size === 0 &&
mediaReasons.size === 0 &&
skippedForCapCount === 0
) {
return { artifact, extractedCount, skippedForCapCount };
}
@ -251,8 +339,17 @@ export async function extractDocumentMultimodalAssets({
elements,
metadata: {
...artifact.metadata,
...mediaCoverageMetadata(artifact, [
...remoteBudget.reasons,
...mediaReasons,
...(skippedForCapCount > 0 ? ["asset-count-budget"] : []),
]),
multimodalAssets: {
extractedCount,
remoteAttempts: remoteBudget.attempts,
remoteDownloadedBytes: remoteBudget.downloadedBytes,
materializedBytes,
variantPixels,
...(skippedForCapCount > 0 ? { skippedForCapCount } : {}),
sources: [...extractionSources].sort(),
},
@ -263,6 +360,43 @@ export async function extractDocumentMultimodalAssets({
};
}
function withAnalysisUnavailable(
element: ParseElement,
reason: AnalysisUnavailableReason,
): ParseElement {
return {
...element,
metadata: {
...element.metadata,
assetRef: {
...(isPlainObject(element.metadata.assetRef) ? element.metadata.assetRef : {}),
analysisUnavailable: { reason },
},
},
};
}
function mediaCoverageMetadata(artifact: ParseArtifact, reasons: readonly string[]) {
if (reasons.length === 0) return {};
const previous = isPlainObject(artifact.metadata.parseCoverage)
? artifact.metadata.parseCoverage
: {};
const media = isPlainObject(previous.media) ? previous.media : {};
const existingReasons = Array.isArray(media.reasons)
? media.reasons.filter((item): item is string => typeof item === "string").slice(0, 32)
: [];
return {
parseCoverage: {
...previous,
media: {
...media,
status: "partial",
reasons: [...new Set([...existingReasons, ...reasons])],
},
},
};
}
function isRemoteHttpUri(uri: string): boolean {
try {
const parsed = new URL(uri);
@ -288,7 +422,12 @@ async function storeGeneratedImageVariants({
image,
knowledgeSpaceId,
objectStorage,
onUnavailable,
onMaterializedBytes,
remainingBytes,
timeoutMs,
tenantId,
signal,
writeOwnerId,
}: {
readonly assetId: string;
@ -297,15 +436,61 @@ async function storeGeneratedImageVariants({
readonly image: DataUriImage;
readonly knowledgeSpaceId: string;
readonly objectStorage: PlatformAdapter["objectStorage"];
readonly onUnavailable: (reason: AnalysisUnavailableReason) => void;
readonly onMaterializedBytes: (bytes: number) => void;
readonly remainingBytes: number;
readonly timeoutMs: number;
readonly tenantId: string;
readonly signal?: AbortSignal | undefined;
readonly writeOwnerId?: string | undefined;
}): Promise<Record<string, Record<string, unknown>>> {
const variants: Record<string, Record<string, unknown>> = {};
const generated = await generator.generate({
body: image.body,
contentType: image.contentType,
elementId,
});
const controller = new AbortController();
const cancel = () => controller.abort(signal?.reason);
signal?.addEventListener("abort", cancel, { once: true });
const timer = setTimeout(() => controller.abort(new Error("variant-deadline")), timeoutMs);
let generated: readonly GeneratedDocumentImageVariant[];
try {
signal?.throwIfAborted();
generated = await generator.generate({
body: image.body,
contentType: image.contentType,
elementId,
signal: controller.signal,
});
signal?.throwIfAborted();
if (controller.signal.aborted) {
onUnavailable("variant-deadline");
return {};
}
} catch (error) {
signal?.throwIfAborted();
if (controller.signal.aborted) {
onUnavailable("variant-deadline");
return {};
}
if (
error instanceof Error &&
"code" in error &&
(error.code === "provider_input" || error.code === "provider_timeout") &&
"retryable" in error &&
error.retryable === false
) {
onUnavailable("variant-input-rejected");
return {};
}
throw error;
} finally {
clearTimeout(timer);
signal?.removeEventListener("abort", cancel);
}
if (generated.length === 0) onUnavailable("variant-unavailable");
const generatedBytes = generated.reduce((sum, variant) => sum + variant.body.byteLength, 0);
if (!Number.isSafeInteger(generatedBytes) || generatedBytes > remainingBytes) {
onUnavailable("materialized-byte-budget");
return {};
}
onMaterializedBytes(generatedBytes);
for (const variant of generated) {
const stored = await storeGeneratedImageVariant({
@ -369,6 +554,7 @@ async function storeGeneratedImageVariant({
return {
contentType: variant.contentType,
...(variant.execution ? { execution: { ...variant.execution } } : {}),
...(variant.height !== undefined ? { height: variant.height } : {}),
objectKey,
sha256,
@ -407,11 +593,13 @@ function parseDataUriImage(uri: string, maxEmbeddedAssetBytes: number): DataUriI
async function readLocalImageAsset({
allowedRoots,
canonicalAllowedRoots,
assetRef,
maxLocalAssetBytes,
uri,
}: {
readonly allowedRoots: readonly string[];
readonly canonicalAllowedRoots: readonly string[];
readonly assetRef: Readonly<Record<string, unknown>> | null;
readonly maxLocalAssetBytes: number;
readonly uri: string;
@ -432,27 +620,49 @@ async function readLocalImageAsset({
return null;
}
const metadata = await stat(localPath);
// Resolve both roots and the target before opening; O_NOFOLLOW also rejects replacement of
// the final component with a symlink between realpath and open.
const canonicalPath = await realpath(localPath);
if (!pathIsWithinAllowedRoots(canonicalPath, canonicalAllowedRoots)) return null;
const handle = await open(canonicalPath, constants.O_RDONLY | constants.O_NOFOLLOW);
try {
const metadata = await handle.stat();
if (!metadata.isFile()) {
return null;
if (!metadata.isFile()) {
return null;
}
if (metadata.size > maxLocalAssetBytes) {
throw new Error(
`Document multimodal local asset exceeds maxLocalAssetBytes=${maxLocalAssetBytes}`,
);
}
const buffer = Buffer.alloc(Math.min(metadata.size, maxLocalAssetBytes) + 1);
let bytesRead = 0;
while (bytesRead < buffer.length) {
const read = await handle.read(buffer, bytesRead, buffer.length - bytesRead, null);
if (read.bytesRead === 0) break;
bytesRead += read.bytesRead;
}
if (bytesRead > maxLocalAssetBytes)
throw new Error(
`Document multimodal local asset exceeds maxLocalAssetBytes=${maxLocalAssetBytes}`,
);
if (bytesRead !== metadata.size)
throw new Error("Document multimodal local asset changed during read");
const body = new Uint8Array(buffer.subarray(0, bytesRead));
const dimensions = readImageDimensions(body, contentType);
return {
body,
contentType,
...(dimensions ? { dimensions } : {}),
source: "local-file",
};
} finally {
await handle.close();
}
if (metadata.size > maxLocalAssetBytes) {
throw new Error(
`Document multimodal local asset exceeds maxLocalAssetBytes=${maxLocalAssetBytes}`,
);
}
const body = new Uint8Array(await readFile(localPath));
const dimensions = readImageDimensions(body, contentType);
return {
body,
contentType,
...(dimensions ? { dimensions } : {}),
source: "local-file",
};
}
function readImageDimensions(body: Uint8Array, contentType: string): ImageDimensions | undefined {

11
knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.test.ts
View File

@ -38,11 +38,18 @@ describe("document multimodal candidate resolver", () => {
id: "figure-1",
metadata: {
assetRef: {
analysisUnavailable: { reason: "variant-pixel-budget" },
contentType: "image/png",
objectKey:
"tenant-dev/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/018f0d60-7a49-7cc2-9c1b-5b36f18f2c43/assets/figure-1.png",
sha256: "c".repeat(64),
uri: "data:image/png;base64,AAAA",
variants: {
analysis: {
contentType: "image/png",
objectKey: "tenant-dev/assets/analysis.png",
},
},
},
boundingBox: { height: 120, width: 240, x: 10, y: 20 },
caption: "Revenue bridge",
@ -95,10 +102,14 @@ describe("document multimodal candidate resolver", () => {
assetDescriptorPath:
"/knowledge/docs/Quarterly-Report.pdf--018f0d60/assets/image-Revenue-bridge--018f0d60.json",
assetRef: {
analysisUnavailable: { reason: "variant-pixel-budget" },
contentType: "image/png",
objectKey:
"tenant-dev/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/018f0d60-7a49-7cc2-9c1b-5b36f18f2c43/assets/figure-1.png",
sha256: "c".repeat(64),
variants: {
analysis: { contentType: "image/png", objectKey: "tenant-dev/assets/analysis.png" },
},
},
assetRoute:
"/knowledge-spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/018f0d60-7a49-7cc2-9c1b-5b36f18f2c43/multimodal/018f0d60-7a49-7cc2-9c1b-5b36f18f2c44%3A0%3Afigure-1/asset",

4
knowledge-fs/packages/api/src/document-multimodal-candidate-resolver.ts
View File

@ -134,6 +134,10 @@ function metadataInteger(
function cloneAssetRef(assetRef: DocumentMultimodalAssetRef): DocumentMultimodalAssetRef {
return {
...(assetRef.analysisUnavailable
? { analysisUnavailable: { ...assetRef.analysisUnavailable } }
: {}),
...(assetRef.variants ? { variants: structuredClone(assetRef.variants) } : {}),
...(assetRef.contentType ? { contentType: assetRef.contentType } : {}),
...(assetRef.objectKey ? { objectKey: assetRef.objectKey } : {}),
...(assetRef.sha256 ? { sha256: assetRef.sha256 } : {}),

2
knowledge-fs/packages/api/src/document-multimodal-enrichment-providers.ts
View File

@ -76,6 +76,8 @@ export function createUnderstandingDocumentMultimodalEnrichmentProvider({
return {
enrich: async (input) => {
// Metadata/OCR already present remains usable, but never forward a rejected original to a VLM.
if (input.item.assetRef?.analysisUnavailable !== undefined) return {};
const task = understandingTaskForItem(input.item);
if (!task) {

51
knowledge-fs/packages/api/src/document-multimodal-manifest-builder.ts
View File

@ -127,7 +127,8 @@ function documentMultimodalItemFromElement({
const caption = metadataString(element.metadata, "caption");
const ocrText = metadataString(element.metadata, "ocrText");
const title = metadataString(element.metadata, "title") ?? caption;
const assetRef = parseAssetRef(element.metadata);
const sourceMetadata = boundedMultimodalSourceMetadata(element.metadata);
const assetRef = parseAssetRef(sourceMetadata);
const boundingBox = parseBoundingBox(element.metadata.boundingBox);
const textPreview = textPreviewForElement(element, ocrText, maxTextPreviewChars);
const positionUnknown = element.metadata.positionUnknown === true;
@ -148,13 +149,35 @@ function documentMultimodalItemFromElement({
...(element.pageNumber ? { pageNumber: element.pageNumber } : {}),
parseElementId: element.id,
sectionPath: [...element.sectionPath],
sourceMetadata: cloneJsonObject(element.metadata),
sourceMetadata,
...(startOffset !== undefined ? { startOffset } : {}),
...(textPreview ? { textPreview } : {}),
...(title ? { title } : {}),
};
}
function boundedMultimodalSourceMetadata(
metadata: Readonly<Record<string, unknown>>,
): Record<string, unknown> {
const sourceMetadata = cloneJsonObject(metadata);
const ref = isPlainObject(sourceMetadata.assetRef) ? sourceMetadata.assetRef : undefined;
if (
ref?.analysisUnavailable !== undefined &&
typeof ref.uri === "string" &&
ref.uri.length > 2048 &&
ref.uri.trimStart().slice(0, 5).toLowerCase() === "data:"
) {
// The original parse element remains the recovery source. Manifests are descriptors, not a
// second copy of rejected image bytes; parseArtifactId + parseElementId preserve provenance.
const { uri, ...boundedRef } = ref;
sourceMetadata.assetRef = {
...boundedRef,
sourceUriSha256: createHash("sha256").update(uri).digest("hex"),
};
}
return sourceMetadata;
}
function multimodalModality(element: ParseElement): DocumentMultimodalItem["modality"] | null {
switch (element.type) {
case "code":
@ -182,17 +205,20 @@ function enrichmentForElement({
readonly ocrText: string | undefined;
}): DocumentMultimodalItem["enrichment"] {
return {
asset: assetRef
? "provided"
: element.type === "image" || element.type === "table"
? "missing"
: "unsupported",
asset:
assetRef && (assetRef.objectKey || assetRef.uri || assetRef.sha256)
? "provided"
: element.type === "image" || element.type === "table"
? "missing"
: "unsupported",
caption: caption ? "provided" : element.type === "image" ? "missing" : "unsupported",
ocr:
ocrText || element.text ? "provided" : element.type === "image" ? "missing" : "unsupported",
tableStructure: element.type === "table" ? tableStructureStatus(element) : "unsupported",
visualEmbedding:
assetRef && isVisualEmbeddingEligibleElement(element) ? "missing" : "unsupported",
assetRef && !assetRef.analysisUnavailable && isVisualEmbeddingEligibleElement(element)
? "missing"
: "unsupported",
};
}
@ -246,11 +272,18 @@ function parseAssetRef(
const contentType =
metadataString(candidate, "contentType") ?? metadataString(candidate, "mimeType");
if (!objectKey && !uri && !sha256) {
if (!objectKey && !uri && !sha256 && candidate.analysisUnavailable === undefined) {
return undefined;
}
return {
...(candidate.analysisUnavailable !== undefined
? {
analysisUnavailable: DocumentMultimodalAssetRefSchema.shape.analysisUnavailable.parse(
candidate.analysisUnavailable,
),
}
: {}),
...(contentType ? { contentType } : {}),
...(objectKey ? { objectKey } : {}),
...(sha256 ? { sha256 } : {}),

14
knowledge-fs/packages/api/src/document-multimodal-manifest-enhancer.ts
View File

@ -357,7 +357,10 @@ function mergeMultimodalItemEnrichment(
...(caption ? { caption } : {}),
enrichment: {
...item.enrichment,
asset: assetRef ? "provided" : item.enrichment.asset,
asset:
assetRef && (assetRef.objectKey || assetRef.uri || assetRef.sha256)
? "provided"
: item.enrichment.asset,
caption: caption ? "provided" : item.enrichment.caption,
ocr: ocrText ? "provided" : item.enrichment.ocr,
// Never downgrade a "provided" status to a weaker provider result.
@ -365,10 +368,9 @@ function mergeMultimodalItemEnrichment(
item.enrichment.tableStructure,
result.tableStructureStatus,
),
visualEmbedding: preferProvidedStatus(
item.enrichment.visualEmbedding,
result.visualEmbeddingStatus,
),
visualEmbedding: assetRef?.analysisUnavailable
? "unsupported"
: preferProvidedStatus(item.enrichment.visualEmbedding, result.visualEmbeddingStatus),
},
...(ocrText ? { ocrText } : {}),
sourceMetadata: {
@ -390,6 +392,8 @@ function pickBetterAssetRef(
current: DocumentMultimodalAssetRef | undefined,
incoming: DocumentMultimodalAssetRef | undefined,
): DocumentMultimodalAssetRef | undefined {
// Enrichment does not validate image resource bounds and cannot clear a decoder rejection.
if (current?.analysisUnavailable) return current;
if (!incoming) {
return current;
}

184
knowledge-fs/packages/api/src/document-pdf-rasterizer-geometry.test.ts Normal file
View File

@ -0,0 +1,184 @@
import sharp from "sharp";
import { describe, expect, it } from "vitest";
import { createPopplerPdfRasterizer } from "./document-pdf-rasterizer";
function geometryPdf({
cropBox,
mediaBox,
rotation = 0,
}: {
readonly cropBox: readonly [number, number, number, number];
readonly mediaBox: readonly [number, number, number, number];
readonly rotation?: number;
}): Uint8Array {
const [x, y, right, top] = mediaBox;
const halfWidth = (right - x) / 2;
const halfHeight = (top - y) / 2;
const content = [
`1 0 0 rg ${x} ${y + halfHeight} ${halfWidth} ${halfHeight} re f`,
`0 1 0 rg ${x + halfWidth} ${y + halfHeight} ${halfWidth} ${halfHeight} re f`,
`0 0 1 rg ${x} ${y} ${halfWidth} ${halfHeight} re f`,
`1 1 0 rg ${x + halfWidth} ${y} ${halfWidth} ${halfHeight} re f`,
].join("\n");
const objects = [
"<< /Type /Catalog /Pages 2 0 R >>",
"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
`<< /Type /Page /Parent 2 0 R /MediaBox [${mediaBox.join(" ")}] /CropBox [${cropBox.join(" ")}] /Rotate ${rotation} /Resources << >> /Contents 4 0 R >>`,
`<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`,
];
let pdf = "%PDF-1.7\n";
const offsets: number[] = [];
for (const [index, object] of objects.entries()) {
offsets.push(Buffer.byteLength(pdf));
pdf += `${index + 1} 0 obj\n${object}\nendobj\n`;
}
const xref = Buffer.byteLength(pdf);
pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
pdf += offsets.map((offset) => `${String(offset).padStart(10, "0")} 00000 n \n`).join("");
pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
return new TextEncoder().encode(pdf);
}
describe("real Poppler MediaBox geometry admission", () => {
it("caps the rendered MediaBox before allocation even when the CropBox is smaller", async () => {
const rasterizer = createPopplerPdfRasterizer({
dpi: 72,
maxPageDimension: 1_000,
maxPagePixels: 10_000,
thumbnailDpi: 72,
});
const image = await rasterizer.render({
documentBody: geometryPdf({ cropBox: [36, 36, 108, 108], mediaBox: [0, 0, 144, 144] }),
elementId: "small-crop-box",
pageNumber: 1,
});
expect(image).not.toBeNull();
await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 100, width: 100 });
});
it("accounts for rounded pdfinfo box coordinates before a boundary-sized allocation", async () => {
const rasterizer = createPopplerPdfRasterizer({
dpi: 72,
maxPageDimension: 1_000,
maxPagePixels: 10_000,
thumbnailDpi: 72,
});
const image = await rasterizer.render({
documentBody: geometryPdf({
cropBox: [0, 0, 100.004, 100],
mediaBox: [0, 0, 100.004, 100],
}),
elementId: "rounded-media-box",
pageNumber: 1,
});
await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 100, width: 100 });
});
it("bounds the shorter scaled edge when rounding conceals a larger aspect ratio", async () => {
const rasterizer = createPopplerPdfRasterizer({
dpi: 72,
maxPageDimension: 1_000,
maxPagePixels: 20_000,
thumbnailDpi: 72,
});
const image = await rasterizer.render({
documentBody: geometryPdf({
cropBox: [0, 0, 100.004, 199.996],
mediaBox: [0, 0, 100.004, 199.996],
}),
elementId: "rounded-aspect-ratio",
pageNumber: 1,
});
await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 199, width: 100 });
});
it("keeps the original rendering of an ordinary uncapped page", async () => {
const image = await createPopplerPdfRasterizer({ dpi: 72, thumbnailDpi: 72 }).render({
documentBody: geometryPdf({ cropBox: [0, 0, 144, 72], mediaBox: [0, 0, 144, 72] }),
elementId: "ordinary-page",
pageNumber: 1,
});
await expect(sharp(image?.body).metadata()).resolves.toMatchObject({ height: 72, width: 144 });
});
it.each([
{ color: [255, 0, 0], rotation: 0 },
{ color: [0, 0, 255], rotation: 90 },
{ color: [255, 255, 0], rotation: 180 },
{ color: [0, 255, 0], rotation: 270 },
])(
"preserves displayed crop coordinates with nonzero origin and rotation=$rotation",
async ({ color, rotation }) => {
const rasterizer = createPopplerPdfRasterizer({
dpi: 72,
maxPageDimension: 100,
maxPagePixels: 20_000,
thumbnailDpi: 72,
});
const rotated = rotation === 90 || rotation === 270;
const pageWidth = rotated ? 72 : 144;
const pageHeight = rotated ? 144 : 72;
const images = await rasterizer.renderBatch?.({
documentBody: geometryPdf({
cropBox: [72, 90, 144, 126],
mediaBox: [36, 72, 180, 144],
rotation,
}),
requests: [
{ elementId: "page", pageNumber: 1 },
{
boundingBox: { height: 0.2, width: 0.2, x: 0.1, y: 0.1 },
boundingBoxGeometry: { coordinateSystem: "relative" },
elementId: "relative-crop",
pageNumber: 1,
},
{
boundingBox: {
height: pageHeight * 0.2,
width: pageWidth * 0.2,
x: pageWidth * 0.1,
y: pageHeight * 0.1,
},
boundingBoxGeometry: { coordinateSystem: "pixel", pageHeight, pageWidth },
elementId: "pixel-crop",
pageNumber: 1,
},
{
boundingBox: {
height: pageHeight * 0.2,
width: pageWidth * 0.2,
x: pageWidth * 0.1,
y: pageHeight * 0.1,
},
boundingBoxGeometry: { coordinateSystem: "pdf-point", pageHeight, pageWidth },
elementId: "displayed-point-crop",
pageNumber: 1,
},
],
});
await expect(sharp(images?.[0]?.body).metadata()).resolves.toMatchObject({
height: rotated ? 100 : 50,
width: rotated ? 50 : 100,
});
for (const image of images?.slice(1) ?? []) {
const { data, info } = await sharp(image?.body)
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
expect(info).toMatchObject({ height: rotated ? 20 : 10, width: rotated ? 10 : 20 });
expect([...data.subarray(0, 3)]).toEqual(color);
expect([...data.subarray(data.length - 3)]).toEqual(color);
}
},
);
});

146
knowledge-fs/packages/api/src/document-pdf-rasterizer.test.ts
View File

@ -32,6 +32,11 @@ interface FakePopplerCommand {
async function createFakePopplerCommand(
mode: "failure" | "success" | "timeout" = "success",
options: {
readonly pdfInfoOutput?: string;
readonly pageSizePoints?: { readonly height: number; readonly width: number };
readonly renderedSize?: { readonly height: number; readonly width: number };
} = {},
): Promise<FakePopplerCommand> {
const root = await mkdtemp(join(tmpdir(), "knowledge-fs-fake-poppler-"));
const command = join(root, "pdftoppm.cjs");
@ -43,8 +48,8 @@ async function createFakePopplerCommand(
create: {
background: { alpha: 1, b: 255, g: 127, r: 63 },
channels: 4,
height: 80,
width: 100,
height: options.renderedSize?.height ?? 80,
width: options.renderedSize?.width ?? 100,
},
})
.png()
@ -57,7 +62,12 @@ if (args.includes("-box")) {
fs.appendFileSync(${JSON.stringify(pdfInfoInvocationLog)}, JSON.stringify(args) + "\\n");
fs.appendFileSync(${JSON.stringify(workDirLog)}, path.dirname(args.at(-1)) + "\\n");
const pageNumber = args[args.indexOf("-f") + 1];
process.stdout.write("Page " + pageNumber + " size: 612 x 792 pts\\n");
if (${options.pdfInfoOutput !== undefined}) {
process.stdout.write(${JSON.stringify(options.pdfInfoOutput ?? "")});
process.exit(0);
}
process.stdout.write("Page " + pageNumber + " size: ${options.pageSizePoints?.width ?? 612} x ${options.pageSizePoints?.height ?? 792} pts\\n");
process.stdout.write("Page " + pageNumber + " MediaBox: 0 0 ${options.pageSizePoints?.width ?? 612} ${options.pageSizePoints?.height ?? 792}\\n");
process.exit(0);
}
const inputPath = args.at(-2);
@ -1198,6 +1208,38 @@ describe("createPopplerPdfRasterizer batch rendering", () => {
}
});
it.each([
"Page 1 size: 72 x 72 pts\n",
"Page 2 MediaBox: 0 0 72 72\n",
"Page 1 MediaBox: 0 0 72\n",
"Page 1 MediaBox: 0 0 Infinity 72\n",
"Page 1 MediaBox: 0 0 NaN 72\n",
"Page 1 MediaBox: 72 0 0 72\n",
"Page 1 MediaBox: 0 72 72 0\n",
"Page 1 MediaBox: 0 0 1e200 1e200\n",
"Page 1 MediaBox: 0 0 72 72\nPage 1 MediaBox: 0 0 144 144\n",
])(
"rejects unsafe or ambiguous MediaBox metadata before rendering (%j)",
async (pdfInfoOutput) => {
const fakePoppler = await createFakePopplerCommand("success", { pdfInfoOutput });
try {
const rasterizer = createPopplerPdfRasterizer({
command: fakePoppler.command,
pdfInfoCommand: fakePoppler.pdfInfoCommand,
});
await expect(
rasterizer.render({ documentBody, elementId: "unsafe-page", pageNumber: 1 }),
).rejects.toThrow("could not determine MediaBox size");
expect(await fakePoppler.readInvocations()).toHaveLength(0);
await expectTemporaryDirectoriesRemoved(await fakePoppler.readWorkDirs());
} finally {
await fakePoppler.cleanup();
}
},
);
it("caches page sizes across batches in one document session and removes its work directory", async () => {
const fakePoppler = await createFakePopplerCommand();
@ -1570,6 +1612,104 @@ describe("createPopplerPdfRasterizer batch rendering", () => {
}
});
it.each([
{
height: 5_102.36,
maxPageDimension: 4_096,
maxPagePixels: 1_000_000,
scaleTo: 1_499,
width: 2_267.72,
},
{ height: 792, maxPageDimension: 4_096, maxPagePixels: 100_000, scaleTo: 359, width: 612 },
{ height: 100, maxPageDimension: 4_096, maxPagePixels: 10_000, scaleTo: 100, width: 100.1 },
{ height: 100_000, maxPageDimension: 20_000, maxPagePixels: 10_000, scaleTo: 10_000, width: 1 },
])(
"caps Poppler allocation before rendering a $width x $height point page to $maxPagePixels pixels",
async ({ height, maxPageDimension, maxPagePixels, scaleTo, width }) => {
const fakePoppler = await createFakePopplerCommand("success", {
pageSizePoints: { height, width },
});
try {
const rasterizer = createPopplerPdfRasterizer({
command: fakePoppler.command,
maxPageDimension,
maxPagePixels,
pdfInfoCommand: fakePoppler.pdfInfoCommand,
thumbnailDpi: 144,
});
await rasterizer.render({ documentBody, elementId: "page-1", pageNumber: 1 });
const [args] = await fakePoppler.readInvocations();
expect(args).toContain("-scale-to");
expect(Number(args?.[args.indexOf("-scale-to") + 1])).toBe(scaleTo);
const shortEdge = Math.max(
1,
Math.ceil((scaleTo * Math.min(width, height)) / Math.max(width, height)),
);
expect(scaleTo * shortEdge).toBeLessThanOrEqual(maxPagePixels);
expect(scaleTo).toBeLessThanOrEqual(maxPageDimension);
} finally {
await fakePoppler.cleanup();
}
},
);
it("preserves PDF, pixel, and relative crop positions after pixel-budget scaling", async () => {
const fakePoppler = await createFakePopplerCommand("success", {
pageSizePoints: { height: 800, width: 600 },
renderedSize: { height: 240, width: 180 },
});
try {
const rasterizer = createPopplerPdfRasterizer({
command: fakePoppler.command,
maxPagePixels: 43_200,
pdfInfoCommand: fakePoppler.pdfInfoCommand,
thumbnailDpi: 144,
});
const images = await rasterizer.renderBatch?.({
documentBody,
requests: [
{
boundingBox: { height: 400, width: 300, x: 60, y: 160 },
boundingBoxGeometry: { coordinateSystem: "pdf-point", pageHeight: 800, pageWidth: 600 },
elementId: "pdf-points",
pageNumber: 1,
},
{
boundingBox: { height: 800, width: 600, x: 120, y: 320 },
boundingBoxGeometry: { coordinateSystem: "pixel", pageHeight: 1_600, pageWidth: 1_200 },
elementId: "pixels",
pageNumber: 1,
},
{
boundingBox: { height: 0.5, width: 0.5, x: 0.1, y: 0.2 },
boundingBoxGeometry: { coordinateSystem: "relative" },
elementId: "relative",
pageNumber: 1,
},
],
});
const [args] = await fakePoppler.readInvocations();
expect(args?.[args.indexOf("-scale-to") + 1]).toBe("239");
expect(images).toHaveLength(3);
const sharp = (await import("sharp")).default;
for (const image of images ?? []) {
expect(image?.metadata).toMatchObject({
crop: { normalizedBoundingBox: { height: 120, width: 90, x: 18, y: 48 } },
});
await expect(sharp(image?.body).metadata()).resolves.toMatchObject({
height: 120,
width: 90,
});
}
} finally {
await fakePoppler.cleanup();
}
});
it("rejects encoded Poppler pages before reading oversized output files", async () => {
const fakePoppler = await createFakePopplerCommand();

77
knowledge-fs/packages/api/src/document-pdf-rasterizer.ts
View File

@ -1346,7 +1346,7 @@ async function renderPopplerPage({
}): Promise<PopplerRenderedPage | null> {
const outputName = `page-${pageNumber}-dpi-${dpi}`;
const outputPrefix = join(workDir, outputName);
const scaleTo = popplerScaleToForPage({ dpi, maxPageDimension, pageSize });
const scaleTo = popplerScaleToForPage({ dpi, maxPageDimension, maxPagePixels, pageSize });
await renderPopplerPng({
command,
dpi,
@ -1456,15 +1456,39 @@ async function readPopplerPdfPageSize({
const { stdout } = await execFileAsync(
command,
["-f", String(pageNumber), "-l", String(pageNumber), "-box", inputPath],
{ signal, timeout: timeoutMs, windowsHide: true },
{
env: { ...process.env, LANG: "C", LC_ALL: "C" },
signal,
timeout: timeoutMs,
windowsHide: true,
},
);
const match = /Page(?:\s+\d+)?\s+size:\s*([\d.]+)\s+x\s+([\d.]+)\s+pts/iu.exec(String(stdout));
const widthPoints = Number(match?.[1]);
const heightPoints = Number(match?.[2]);
// pdftoppm deliberately retains its default MediaBox rendering so provider coordinates keep
// their established displayed-page frame. pdfinfo's "Page size" reports the CropBox instead:
// budgeting from it can admit a small visible crop while allocating a much larger MediaBox.
// Subtract endpoints rather than assuming a zero origin. Rotation only swaps the axes, which
// cannot change the area, longest edge or aspect ratio used by the pre-render pixel budget.
const boxes = [
...String(stdout).matchAll(
new RegExp(`^Page(?:\\s+${pageNumber})?\\s+MediaBox:\\s*(.*?)\\s*$`, "gmu"),
),
];
const coordinates = boxes[0]?.[1]?.trim().split(/\s+/u).map(Number) ?? [];
const [left = Number.NaN, bottom = Number.NaN, right = Number.NaN, top = Number.NaN] =
coordinates;
const widthPoints = right - left;
const heightPoints = top - bottom;
if (!(widthPoints > 0) || !(heightPoints > 0)) {
if (
boxes.length !== 1 ||
coordinates.length !== 4 ||
!coordinates.every(Number.isFinite) ||
!(widthPoints > 0) ||
!(heightPoints > 0) ||
!Number.isFinite(widthPoints * heightPoints)
) {
throw new DocumentPdfRenderError(
`Poppler PDF rasterizer could not determine page size for pageNumber=${pageNumber}`,
`Poppler PDF rasterizer could not determine MediaBox size for pageNumber=${pageNumber}`,
);
}
@ -1474,15 +1498,50 @@ async function readPopplerPdfPageSize({
function popplerScaleToForPage({
dpi,
maxPageDimension,
maxPagePixels,
pageSize,
}: {
readonly dpi: number;
readonly maxPageDimension: number;
readonly maxPagePixels: number;
readonly pageSize: PopplerPdfPageSize;
}): number | undefined {
const naturalMaxDimension = (Math.max(pageSize.widthPoints, pageSize.heightPoints) * dpi) / 72;
// pdfinfo prints box endpoints to two decimal places. Each difference can hide up to 0.01 pt.
// Use upper dimensions for allocation and the worst-case aspect ratio for scale-to; otherwise
// a rounded 100.00 x 200.00 box can render a 101 x 200 bitmap just past the pixel ceiling.
const roundingAllowance = 0.01;
const naturalWidth = ((pageSize.widthPoints + roundingAllowance) * dpi) / 72;
const naturalHeight = ((pageSize.heightPoints + roundingAllowance) * dpi) / 72;
return naturalMaxDimension > maxPageDimension ? maxPageDimension : undefined;
if (
Math.max(naturalWidth, naturalHeight) <= maxPageDimension &&
Math.ceil(naturalWidth) * Math.ceil(naturalHeight) <= maxPagePixels
) {
return undefined;
}
const shortEdgeUpper = Math.min(pageSize.widthPoints, pageSize.heightPoints) + roundingAllowance;
const longEdgeLower = Math.max(pageSize.widthPoints, pageSize.heightPoints) - roundingAllowance;
const aspectRatio = longEdgeLower > 0 ? Math.min(1, shortEdgeUpper / longEdgeLower) : 1;
let lower = 1;
let upper = Math.floor(
Math.min(maxPageDimension, maxPagePixels, Math.sqrt(maxPagePixels / aspectRatio)),
);
// Poppler rounds the shorter edge up. A square-root area cap alone can therefore exceed
// the pixel budget; find the largest integer edge whose rounded bitmap is still bounded.
while (lower < upper) {
const candidate = lower + Math.ceil((upper - lower) / 2);
const shortEdge = Math.max(1, Math.ceil(candidate * aspectRatio));
if (candidate * shortEdge <= maxPagePixels) {
lower = candidate;
} else {
upper = candidate - 1;
}
}
return lower;
}
async function renderPopplerPng({

108
knowledge-fs/packages/api/src/document-remote-media-budget.ts Normal file
View File

@ -0,0 +1,108 @@
import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor";
type RemoteImage = Awaited<ReturnType<DocumentRemoteAssetFetcher["fetch"]>>;
/** One document, one bounded cache and one absolute deadline, including unsuccessful URLs. */
export function createDocumentRemoteMediaBudget(input: {
readonly fetcher?: DocumentRemoteAssetFetcher | undefined;
readonly maxAttempts: number;
readonly maxBytes: number;
readonly maxTotalBytes: number;
readonly signal?: AbortSignal | undefined;
readonly timeoutMs: number;
}) {
for (const [name, value] of Object.entries({
maxRemoteAssetAttempts: input.maxAttempts,
maxRemoteAssetBytes: input.maxBytes,
maxTotalRemoteAssetBytes: input.maxTotalBytes,
remoteAssetTimeoutMs: input.timeoutMs,
})) {
if (!Number.isSafeInteger(value) || value < 1) throw new Error(`${name} must be at least 1`);
}
const deadline = performance.now() + input.timeoutMs;
const cache = new Map<string, RemoteImage>();
const reasons = new Set<string>();
let attempts = 0;
let downloadedBytes = 0;
let reservedBytes = 0;
let deadlineReached = false;
return {
reasons,
get attempts() {
return attempts;
},
get downloadedBytes() {
return downloadedBytes;
},
async fetch(url: string): Promise<RemoteImage> {
input.signal?.throwIfAborted();
if (cache.has(url)) return cache.get(url) ?? null;
if (!input.fetcher) {
reasons.add("remote-fetcher-unavailable");
return null;
}
if (deadlineReached || performance.now() >= deadline) {
reasons.add("remote-deadline");
return null;
}
if (attempts >= input.maxAttempts) {
reasons.add("remote-attempt-budget");
return null;
}
if (reservedBytes >= input.maxTotalBytes) {
reasons.add("remote-byte-budget");
return null;
}
attempts += 1;
const maxBytes = Math.min(input.maxBytes, input.maxTotalBytes - reservedBytes);
// A failed fetch can have consumed its entire body limit without returning a body. Keep
// that reservation charged so failures cannot bypass the aggregate network budget.
reservedBytes += maxBytes;
const controller = new AbortController();
const abortFromParent = () => controller.abort(input.signal?.reason);
input.signal?.addEventListener("abort", abortFromParent, { once: true });
const timer = setTimeout(
() => controller.abort(new Error("remote-deadline")),
Math.max(1, deadline - performance.now()),
);
let rejectAbort: (() => void) | undefined;
try {
const aborted = new Promise<never>((_resolve, reject) => {
rejectAbort = () => reject(controller.signal.reason);
controller.signal.addEventListener("abort", rejectAbort, { once: true });
if (controller.signal.aborted) rejectAbort();
});
const image = await Promise.race([
input.fetcher.fetch({ maxBytes, signal: controller.signal, url }),
aborted,
]);
input.signal?.throwIfAborted();
if (image) {
if (image.body.byteLength > maxBytes)
throw new RemoteMediaContractError(
`Document multimodal remote asset exceeds maxRemoteAssetBytes=${maxBytes}`,
);
downloadedBytes += image.body.byteLength;
reservedBytes -= maxBytes - image.body.byteLength;
} else {
reasons.add("remote-unavailable");
}
cache.set(url, image);
return image;
} catch (error) {
input.signal?.throwIfAborted();
if (error instanceof RemoteMediaContractError || !controller.signal.aborted) throw error;
deadlineReached = true;
reasons.add("remote-deadline");
cache.set(url, null);
return null;
} finally {
clearTimeout(timer);
input.signal?.removeEventListener("abort", abortFromParent);
if (rejectAbort) controller.signal.removeEventListener("abort", rejectAbort);
}
},
};
}
class RemoteMediaContractError extends Error {}

15
knowledge-fs/packages/api/src/document-upload-utils.test.ts
View File

@ -20,6 +20,21 @@ import {
} from "./document-upload-utils";
describe("document upload utilities", () => {
it.each(["constructor", "__proto__"])(
"rejects inherited-key extension %s as unsupported instead of crashing",
async (extension) => {
await expect(
readDocumentUpload(
{
parseBody: async () => ({
file: new File(["x"], `x.${extension}`, { type: "text/plain" }),
}),
},
10,
),
).rejects.toBeInstanceOf(DocumentUploadValidationError);
},
);
it("reads a single multipart file with bounded bytes and optional source id", async () => {
const file = new File(["hello"], "note.md", { type: "text/markdown" });

57
knowledge-fs/packages/api/src/document-upload-utils.ts
View File

@ -1,3 +1,8 @@
import {
DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION,
documentMimeTypesForFilename,
} from "@knowledge/parsers";
export interface ParsedDocumentUpload {
readonly body: Uint8Array;
readonly documentId?: string;
@ -20,39 +25,6 @@ export interface BulkDocumentRevisionTarget {
readonly expectedDocumentRowVersion: number;
}
const DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION = {
csv: ["text/csv", "application/vnd.ms-excel"],
doc: ["application/msword"],
docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
eml: ["message/rfc822"],
epub: ["application/epub+zip"],
htm: ["text/html"],
html: ["text/html"],
json: ["application/json"],
jsonl: ["application/x-ndjson", "application/jsonl", "application/ndjson", "application/json"],
markdown: ["text/markdown", "text/x-markdown", "text/plain"],
md: ["text/markdown", "text/x-markdown", "text/plain"],
mdx: ["text/mdx", "text/markdown", "text/plain"],
msg: ["application/vnd.ms-outlook", "application/x-msg"],
odt: ["application/vnd.oasis.opendocument.text"],
pdf: ["application/pdf"],
ppt: ["application/vnd.ms-powerpoint", "application/mspowerpoint", "application/x-mspowerpoint"],
pptx: ["application/vnd.openxmlformats-officedocument.presentationml.presentation"],
properties: ["text/x-java-properties", "text/plain"],
rtf: ["application/rtf", "text/rtf", "application/x-rtf"],
text: ["text/plain"],
txt: ["text/plain"],
vtt: ["text/vtt", "text/plain"],
xls: [
"application/vnd.ms-excel",
"application/excel",
"application/x-excel",
"application/x-msexcel",
],
xlsx: ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
xml: ["application/xml", "text/xml"],
} as const satisfies Readonly<Record<string, readonly string[]>>;
export const SUPPORTED_DOCUMENT_UPLOAD_MIME_TYPES = new Set(
Object.values(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION).flat(),
);
@ -539,32 +511,17 @@ function isJsonObject(value: unknown): value is Record<string, unknown> {
export function normalizeDocumentMimeType(file: File): string {
const declared = file.type.trim().toLocaleLowerCase();
const extension = documentExtension(file.name);
const inferred = extension
? (DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION as Readonly<Record<string, readonly string[]>>)[
extension
]?.[0]
: undefined;
const inferred = documentMimeTypesForFilename(file.name)?.[0];
return !declared || declared === "application/octet-stream"
? (inferred ?? "application/octet-stream")
: declared;
}
function isSupportedDocumentUpload(file: File, mimeType: string): boolean {
const extension = documentExtension(file.name);
if (extension === undefined) return false;
const allowedMimeTypes = (
DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION as Readonly<Record<string, readonly string[]>>
)[extension];
const allowedMimeTypes = documentMimeTypesForFilename(file.name);
return allowedMimeTypes?.includes(mimeType) === true;
}
function documentExtension(filename: string): string | undefined {
const normalized = filename.trim().toLocaleLowerCase();
const dot = normalized.lastIndexOf(".");
return dot >= 0 && dot < normalized.length - 1 ? normalized.slice(dot + 1) : undefined;
}
export async function sha256Hex(bytes: Uint8Array): Promise<string> {
const buffer = bytes.buffer.slice(
bytes.byteOffset,

5
knowledge-fs/packages/api/src/document-write-handlers.ts
View File

@ -39,6 +39,7 @@ import type { DocumentCompilationJobStateMachine } from "./document-compilation-
import { compileDocumentArtifact } from "./document-compilation-pipeline";
import type { DocumentImageVariantGenerator } from "./document-image-variant-generator";
import { buildDocumentKnowledgePath } from "./document-knowledge-paths";
import type { DocumentMediaCapabilityResolver } from "./document-media-execution-plan";
import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor";
import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository";
import type { DocumentOutlineBuilder } from "./document-outline-builder";
@ -194,6 +195,7 @@ export interface RegisterDocumentWriteHandlersOptions {
readonly synchronousUploadDenseModel?: string | undefined;
readonly traces: TraceRecorder;
readonly visualEmbeddingModel?: string | undefined;
readonly resolveDocumentMediaCapabilities?: DocumentMediaCapabilityResolver | undefined;
}
interface BulkUploadAcceptedItem {
@ -269,6 +271,7 @@ export function registerDocumentWriteHandlers({
synchronousUploadDenseModel,
traces,
visualEmbeddingModel,
resolveDocumentMediaCapabilities,
}: RegisterDocumentWriteHandlersOptions): void {
const effectiveBufferedDocumentUploadAdmission =
bufferedDocumentUploadAdmission ??
@ -1480,6 +1483,7 @@ export function registerDocumentWriteHandlers({
body: upload.body,
knowledgeSpaceId,
permissionScope: [],
signal: context.req.raw.signal,
tenantId: subject.tenantId,
traceId,
},
@ -1510,6 +1514,7 @@ export function registerDocumentWriteHandlers({
synchronousUploadReindexer,
traces,
visualEmbeddingModel,
resolveDocumentMediaCapabilities,
},
);
await assertWritable();

39
knowledge-fs/packages/api/src/index-projection-builders.test.ts
View File

@ -74,6 +74,45 @@ function createRecordingProjectionRepository() {
}
describe("index projection builders", () => {
it("excludes rejected image nodes before invoking any visual provider", async () => {
const { repository } = createRecordingProjectionRepository();
const calls: EmbedVisualAssetsInput[] = [];
const builder = createVisualEmbeddingProjectionBuilder({
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f9006",
maxBatchSize: 2,
projections: repository,
provider: {
embedAssets: async (input) => {
calls.push(input);
return {
dense: [[0.2, 0.8]],
metadata: { provider: "test", model: "vision" },
model: "vision",
};
},
},
});
await expect(
builder.build({
model: "vision",
projectionVersion: 1,
nodes: [
knowledgeNode({
kind: "image",
metadata: {
assetRef: {
contentType: "image/png",
objectKey: "tenant/assets/image.png",
analysisUnavailable: { reason: "variant-pixel-budget" },
},
elementTypes: ["image"],
},
}),
],
}),
).resolves.toEqual([]);
expect(calls).toEqual([]);
});
it("propagates cancellation to embedding calls and never persists an aborted batch", async () => {
const controller = new AbortController();
const { created, repository } = createRecordingProjectionRepository();

3
knowledge-fs/packages/api/src/index-projection-builders.ts
View File

@ -1047,6 +1047,7 @@ export function createObjectStorageVisualEmbeddingProvider({
for (const asset of assets) {
signal?.throwIfAborted();
if (asset.assetRef.analysisUnavailable !== undefined) continue;
// We cannot know the next object's exact byte length without an extra HEAD request. Flush
// whenever the remaining raw-byte budget cannot admit the configured per-asset maximum;
// the following bounded GET therefore cannot create a currentBatchBytes + maxAssetBytes
@ -1313,7 +1314,7 @@ function visualEmbeddingAssetCandidateFromNode(
const modality =
metadataString(multimodal ?? {}, "modality") ?? multimodalProjectionModality(node);
if (!assetRef || !modality) {
if (!assetRef || assetRef.analysisUnavailable !== undefined || !modality) {
return null;
}

10
knowledge-fs/packages/api/src/index.ts
View File

@ -456,6 +456,7 @@ import {
createInMemoryDocumentAssetRepository,
} from "./document-asset-repository";
import { registerDocumentCompilationHandlers } from "./document-compilation-handlers";
import { createProfileDocumentMediaCapabilityResolver } from "./document-media-execution-plan";
import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder";
import { createCachedDocumentMultimodalManifestEnhancer } from "./document-multimodal-manifest-enhancer";
import { createInMemoryDocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository";
@ -2036,6 +2037,13 @@ export function createKnowledgeGateway({
})
: undefined;
const resolveDocumentMediaCapabilities =
knowledgeSpaceProfiles && modelInputModalityResolver
? createProfileDocumentMediaCapabilityResolver({
profiles: knowledgeSpaceProfiles,
modalities: modelInputModalityResolver,
})
: undefined;
registerDocumentWriteHandlers({
access: accessService,
adapter,
@ -2069,6 +2077,7 @@ export function createKnowledgeGateway({
: {}),
documentMultimodalManifests: multimodalManifestRepository,
documentParser,
resolveDocumentMediaCapabilities,
...(documentPdfRasterizer ? { documentPdfRasterizer } : {}),
effectiveMaxBulkUploadBytes,
generateArtifactSegmentId,
@ -2109,6 +2118,7 @@ export function createKnowledgeGateway({
});
const sourceDocumentMaterializer = createSourceDocumentMaterializer({
resolveDocumentMediaCapabilities,
artifacts,
artifactSegments: segments,
assets,

5
knowledge-fs/packages/api/src/llm-multimodal-answer-provider.test.ts
View File

@ -272,6 +272,11 @@ describe("createObjectStorageContentBlockMultimodalAnswerProvider", () => {
await adapter.objectStorage.putObject({
body: new Uint8Array([1, 2, 3]),
contentType: "image/png",
key: "tenant/spaces/space/documents/doc/assets/chart.png",
});
await adapter.objectStorage.putObject({
body: new Uint8Array([9]),
contentType: "image/png",
key: "tenant/spaces/space/documents/doc/assets/chart-thumbnail.png",
});
const calls: GenerateMultimodalAnswerContentInput[] = [];

12
knowledge-fs/packages/api/src/llm-multimodal-answer-provider.ts
View File

@ -155,7 +155,7 @@ export function createObjectStorageContentBlockMultimodalAnswerProvider({
maxImageBytes = 10 * 1024 * 1024,
maxTotalImageBytes = 32 * 1024 * 1024,
objectStorage,
preferredVariant = "thumbnail",
preferredVariant = "analysis",
...options
}: ObjectStorageContentBlockMultimodalAnswerProviderOptions): MultimodalAnswerProvider {
if (!Number.isSafeInteger(maxImageBytes) || maxImageBytes < 1) {
@ -385,7 +385,10 @@ function objectBackedImageAssetRef({
readonly attachment: MultimodalAnswerProviderInput["multimodalEvidence"][number];
readonly preferredVariant: string;
}): { readonly contentType: string; readonly objectKey: string } | undefined {
if (!isPlainObject(attachment.assetRef)) {
if (
!isPlainObject(attachment.assetRef) ||
attachment.assetRef.analysisUnavailable !== undefined
) {
return undefined;
}
@ -472,7 +475,10 @@ function multimodalContentBlockMessages({
for (const [index, attachment] of input.multimodalEvidence.entries()) {
const label = `M${index + 1}`;
const url = isVisualAttachment(attachment) ? assetUrlResolver(attachment) : undefined;
const url =
isVisualAttachment(attachment) && attachment.assetRef?.analysisUnavailable === undefined
? assetUrlResolver(attachment)
: undefined;
attachmentTextBlocks.push({
text: multimodalAttachmentLine(label, attachment, url),

4
knowledge-fs/packages/api/src/source-handlers-coverage.test.ts
View File

@ -1141,7 +1141,7 @@ describe("online document import edge branches", () => {
});
expect(response.status).toBe(200);
const body = await response.json();
expect(body.documents).toHaveLength(1);
expect(body.documents, JSON.stringify(body)).toHaveLength(1);
expect(body.failed).toEqual([
{
code: SOURCE_OPERATION_FAILURES.onlineDocumentPageFetch.code,
@ -1376,7 +1376,7 @@ describe("online drive import edge branches", () => {
);
expect(response.status).toBe(200);
const body = await response.json();
expect(body.documents).toEqual([
expect(body.documents, JSON.stringify(body)).toEqual([
{ documentAssetId: expect.any(String), filename: "plain.txt" },
]);
expect(body.failed).toEqual([

20
knowledge-fs/packages/api/src/source-handlers.ts
View File

@ -1,5 +1,6 @@
import { isDeepStrictEqual } from "node:util";
import type { OpenAPIHono } from "@hono/zod-openapi";
import { documentMimeTypesForFilename } from "@knowledge/parsers";
import {
CANDIDATE_VISIBILITY_SCAN_BUDGET_EXCEEDED_MESSAGE,
@ -1436,25 +1437,8 @@ function slugPart(value: string): string {
return value.replace(/[^a-zA-Z0-9._-]+/gu, "-").replace(/^-+|-+$/gu, "");
}
const MIME_TYPES_BY_EXTENSION: Readonly<Record<string, string>> = {
csv: "text/csv",
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
htm: "text/html",
html: "text/html",
json: "application/json",
markdown: "text/markdown",
md: "text/markdown",
pdf: "application/pdf",
pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
txt: "text/plain",
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
xml: "application/xml",
};
export function mimeTypeForFilename(filename: string): string {
const extension = /\.([a-zA-Z0-9]+)$/u.exec(filename)?.[1]?.toLowerCase();
return (extension && MIME_TYPES_BY_EXTENSION[extension]) || "application/octet-stream";
return documentMimeTypesForFilename(filename)?.[0] ?? "application/octet-stream";
}
export interface ImportedPageState {

17
knowledge-fs/packages/api/src/source-mime-registry.test.ts Normal file
View File

@ -0,0 +1,17 @@
import { describe, expect, it } from "vitest";
import { mimeTypeForFilename } from "./source-handlers";
describe("source file MIME inference shares the upload registry", () => {
it.each([
["source.JSONL", "application/x-ndjson"],
[" mail.eml ", "message/rfc822"],
["legacy.doc", "application/msword"],
["deck.ppt", "application/vnd.ms-powerpoint"],
["caption.vtt", "text/vtt"],
["settings.properties", "text/x-java-properties"],
["data.unknown", "application/octet-stream"],
["README", "application/octet-stream"],
])("infers %s without widening upload types", (filename, expected) => {
expect(mimeTypeForFilename(filename)).toBe(expected);
});
});

15
knowledge-fs/packages/core/src/models.ts
View File

@ -929,6 +929,21 @@ export const DocumentMultimodalAssetVariantSchema = z.object({
export type DocumentMultimodalAssetVariant = z.infer<typeof DocumentMultimodalAssetVariantSchema>;
export const DocumentMultimodalAssetRefSchema = z.object({
// A retained download/preview reference is not necessarily safe to decode for model analysis.
// Absent on legacy artifacts; only explicit rejection disables image analysis.
analysisUnavailable: z
.object({
reason: z.enum([
"asset-count-budget",
"materialized-byte-budget",
"variant-pixel-budget",
"variant-deadline",
"variant-input-rejected",
"variant-unavailable",
]),
})
.strict()
.optional(),
contentType: z.string().min(1).optional(),
objectKey: ObjectStorageKeySchema.optional(),
sha256: Sha256Schema.optional(),

72
knowledge-fs/packages/parsers/src/archive-media-completeness.test.ts Normal file
View File

@ -0,0 +1,72 @@
import { zipSync } from "fflate";
import { describe, expect, it } from "vitest";
import { createUnstructuredParserClient } from "./index";
const parser = () =>
createUnstructuredParserClient({
endpoint: "https://parser.test",
fetch: async () =>
Response.json([{ type: "NarrativeText", text: "Readable body", metadata: {} }]),
});
const input = (entries: Record<string, Uint8Array>, requiresImages?: boolean) => ({
body: zipSync(entries),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: "x.docx",
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
parserHints: requiresImages === undefined ? {} : { requiresImages },
version: 1,
});
describe("archive media completeness", () => {
it("does not coalesce or reuse legacy-auto image output for explicitly disabled media", async () => {
const remote = parser();
const entries = { "word/media/photo.png": new Uint8Array([1, 2, 3]) };
const automatic = input(entries);
const disabled = { ...automatic, parserHints: { requiresImages: false } };
expect(remote.policyFingerprint?.(automatic)).not.toBe(remote.policyFingerprint?.(disabled));
const [withImages, withoutImages] = await Promise.all([
remote.parse(automatic),
remote.parse(disabled),
]);
expect(withImages.elements.some((element) => element.type === "image")).toBe(true);
expect(withoutImages.elements.some((element) => element.type === "image")).toBe(false);
expect(withImages.artifactHash).not.toBe(withoutImages.artifactHash);
});
it("reports unsupported media by source path without silently claiming complete", async () => {
const artifact = await parser().parse(
input({ "word/media/vector.svg": new TextEncoder().encode("<svg/>") }, true),
);
expect(artifact.elements.map((element) => element.text)).toEqual(["Readable body"]);
expect(artifact.metadata.parseCoverage).toMatchObject({
media: { status: "partial", reasons: ["unsupported-media-format"] },
});
expect(artifact.metadata.archiveMediaReport).toMatchObject({
skippedResources: [
{ archivePath: "word/media/vector.svg", reason: "unsupported-media-format" },
],
});
});
it("honors disabled image extraction without materializing archive images", async () => {
const artifact = await parser().parse(
input({ "word/media/photo.png": new Uint8Array([1, 2, 3]) }, false),
);
expect(artifact.elements.some((element) => element.type === "image")).toBe(false);
expect(artifact.metadata.parseCoverage).toMatchObject({ media: { status: "not-requested" } });
});
it("records Office charts and drawing definitions whose visuals cannot be extracted", async () => {
const artifact = await parser().parse(
input({ "word/charts/chart1.xml": new TextEncoder().encode("<chart/>") }),
);
expect(artifact.metadata.parseCoverage).toMatchObject({
media: { status: "partial", reasons: ["office-visual-structure-not-rendered"] },
});
});
it("reports oversized image omission and retains its archive reference", async () => {
const artifact = await parser().parse(
input({ "word/media/photo.png": new Uint8Array(10 * 1024 * 1024 + 1) }),
);
expect(artifact.metadata.archiveMediaReport).toMatchObject({
skippedResources: [{ archivePath: "word/media/photo.png", reason: "media-byte-budget" }],
});
});
});

73
knowledge-fs/packages/parsers/src/document-format-registry.test.ts Normal file
View File

@ -0,0 +1,73 @@
import { describe, expect, it } from "vitest";
import {
DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION,
documentMimeTypesForFilename,
resolveDocumentFormat,
} from "./document-format-registry";
import {
createNativeHtmlParser,
createNativeMarkdownParser,
createParserRouter,
createUnstructuredParserClient,
} from "./index";
describe("shared document format registry", () => {
it("only returns MIME aliases for actual registered filename extensions", () => {
expect(documentMimeTypesForFilename("file.docx")).toEqual([
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
]);
for (const filename of [
"file.constructor",
"file.__proto__",
"file.unknown",
"file.",
"README",
])
expect(documentMimeTypesForFilename(filename)).toBeUndefined();
});
it("preserves the existing upload allowlist exactly", () => {
expect(Object.keys(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION).sort()).toEqual(
"csv doc docx eml epub htm html json jsonl markdown md mdx msg odt pdf ppt pptx properties rtf text txt vtt xls xlsx xml".split(
" ",
),
);
});
it.each([
[" Report.JSONL ", "application/json", "jsonl"],
["x.csv", "application/vnd.ms-excel", "csv"],
["x.unknown", "text/html; charset=utf-8", "html"],
["x.yaml", "text/plain", "yaml"],
["README", "text/plain", "markdown"],
["x.pdf", "text/plain", "unstructured"],
["x.xyz", "", null],
["x.constructor", "", null],
["x.__proto__", "", null],
])("resolves %s deterministically", (filename, mimeType, expected) => {
expect(resolveDocumentFormat({ filename, mimeType })).toBe(expected);
});
it("never bypasses native resource limits by sending oversized markup remotely", async () => {
let requests = 0;
const parser = createParserRouter({
html: createNativeHtmlParser(),
markdown: createNativeMarkdownParser(),
maxNativeInputBytes: 3,
unstructured: createUnstructuredParserClient({
endpoint: "https://parser.test",
fetch: async () => {
requests += 1;
return Response.json([]);
},
}),
});
await expect(
parser.parse({
body: new TextEncoder().encode("too long"),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: "x.md",
mimeType: "text/markdown",
version: 1,
}),
).rejects.toMatchObject({ code: "provider_input" });
expect(requests).toBe(0);
});
});

119
knowledge-fs/packages/parsers/src/document-format-registry.ts Normal file
View File

@ -0,0 +1,119 @@
/** One source of truth for upload MIME aliases and parser family. Internal source formats do not widen uploads. */
export const DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION = {
csv: ["text/csv", "application/vnd.ms-excel"],
doc: ["application/msword"],
docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
eml: ["message/rfc822"],
epub: ["application/epub+zip"],
htm: ["text/html"],
html: ["text/html"],
json: ["application/json"],
jsonl: ["application/x-ndjson", "application/jsonl", "application/ndjson", "application/json"],
markdown: ["text/markdown", "text/x-markdown", "text/plain"],
md: ["text/markdown", "text/x-markdown", "text/plain"],
mdx: ["text/mdx", "text/markdown", "text/plain"],
msg: ["application/vnd.ms-outlook", "application/x-msg"],
odt: ["application/vnd.oasis.opendocument.text"],
pdf: ["application/pdf"],
ppt: ["application/vnd.ms-powerpoint", "application/mspowerpoint", "application/x-mspowerpoint"],
pptx: ["application/vnd.openxmlformats-officedocument.presentationml.presentation"],
properties: ["text/x-java-properties", "text/plain"],
rtf: ["application/rtf", "text/rtf", "application/x-rtf"],
text: ["text/plain"],
txt: ["text/plain"],
vtt: ["text/vtt", "text/plain"],
xls: [
"application/vnd.ms-excel",
"application/excel",
"application/x-excel",
"application/x-msexcel",
],
xlsx: ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
xml: ["application/xml", "text/xml"],
} as const satisfies Readonly<Record<string, readonly string[]>>;
export type DocumentFormat =
| "csv"
| "html"
| "json"
| "jsonl"
| "markdown"
| "properties"
| "unstructured"
| "vtt"
| "xml"
| "yaml";
const formatsByExtension: Readonly<Record<string, DocumentFormat>> = {
csv: "csv",
doc: "unstructured",
docx: "unstructured",
eml: "unstructured",
epub: "unstructured",
htm: "html",
html: "html",
json: "json",
jsonl: "jsonl",
markdown: "markdown",
md: "markdown",
mdx: "markdown",
msg: "unstructured",
odt: "unstructured",
pdf: "unstructured",
ppt: "unstructured",
pptx: "unstructured",
properties: "properties",
rtf: "unstructured",
text: "markdown",
txt: "markdown",
vtt: "vtt",
xls: "unstructured",
xlsx: "unstructured",
xml: "xml",
// Source connectors historically support these; upload admission deliberately remains unchanged.
ndjson: "jsonl",
yaml: "yaml",
yml: "yaml",
};
const formatsByMime = new Map<string, DocumentFormat>();
for (const [extension, mimes] of Object.entries(DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION)) {
const format = formatsByExtension[extension];
if (format) for (const mime of mimes) formatsByMime.set(mime, format);
}
// Ambiguous upload aliases must not change generic MIME routing (CSV may be declared Excel).
formatsByMime.set("text/plain", "markdown");
formatsByMime.set("text/markdown", "markdown");
formatsByMime.set("application/json", "json");
formatsByMime.set("application/xhtml+xml", "html");
formatsByMime.set("text/json", "json");
for (const mime of ["application/yaml", "text/yaml", "application/x-yaml"])
formatsByMime.set(mime, "yaml");
export function documentExtension(filename: string): string | undefined {
const normalized = filename.trim().toLowerCase();
const dot = normalized.lastIndexOf(".");
return dot >= 0 && dot < normalized.length - 1 ? normalized.slice(dot + 1) : undefined;
}
export function resolveDocumentFormat(input: {
readonly filename: string;
readonly mimeType: string;
}): DocumentFormat | null {
const extension = documentExtension(input.filename);
const byExtension =
extension && Object.hasOwn(formatsByExtension, extension)
? formatsByExtension[extension]
: undefined;
return (
byExtension ??
formatsByMime.get(input.mimeType.split(";", 1)[0]?.trim().toLowerCase() ?? "") ??
null
);
}
export function documentMimeTypesForFilename(filename: string): readonly string[] | undefined {
const extension = documentExtension(filename);
const registry: Readonly<Record<string, readonly string[]>> =
DOCUMENT_UPLOAD_MIME_TYPES_BY_EXTENSION;
return extension && Object.hasOwn(registry, extension) ? registry[extension] : undefined;
}

113
knowledge-fs/packages/parsers/src/document-table-bytes.test.ts Normal file
View File

@ -0,0 +1,113 @@
import { describe, expect, it, vi } from "vitest";
import {
createNativeHtmlParser,
createNativeMarkdownParser,
createUnstructuredParserClient,
} from "./index";
vi.mock("./parser-resource-budget", async (importOriginal) => {
const actual = await importOriginal<typeof import("./parser-resource-budget")>();
return {
...actual,
parserResourceLimits: { ...actual.parserResourceLimits, maxOutputBytes: 64 },
};
});
function input(body: string, extension = "html") {
return {
body: new TextEncoder().encode(body),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: `tables.${extension}`,
mimeType:
extension === "html"
? "text/html"
: extension === "pdf"
? "application/pdf"
: "text/markdown",
version: 1,
};
}
const label = "中".repeat(12);
const htmlTable = `<table><tr><th>${label}</th></tr></table>`;
const markdownTable = `| ${label} |\n| --- |`;
describe("document-local table projection bytes", () => {
it("charges header-only HTML tables against one byte budget", async () => {
await expect(
createNativeHtmlParser().parse(input(htmlTable + htmlTable)),
).rejects.toMatchObject({
code: "provider_input",
retryable: false,
message: expect.stringContaining("table projection bytes"),
});
});
it("charges headerless HTML data rows against one byte budget", async () => {
const table = `<table><tr><td>${label}</td></tr></table>`;
await expect(createNativeHtmlParser().parse(input(table + table))).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("table projection bytes"),
});
});
it.each(["md", "mdx"])(
"shares bytes across ordinary tables and HTML tokens in %s",
async (extension) => {
for (const body of [
`${markdownTable}\n\nBetween\n\n${markdownTable}`,
`${markdownTable}\n\n${htmlTable}`,
]) {
await expect(
createNativeMarkdownParser().parse(input(body, extension)),
).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("table projection bytes"),
});
}
},
);
it("charges every provider table before joining subsequent table output", async () => {
const table = { type: "Table", metadata: { text_as_html: htmlTable } };
const parser = createUnstructuredParserClient({
endpoint: "http://parser.invalid",
fetch: async () => Response.json([table, table]),
});
await expect(parser.parse(input("%PDF-1.4", "pdf"))).rejects.toMatchObject({
code: "provider_response_invalid",
retryable: false,
cause: { message: expect.stringContaining("table projection bytes") },
});
});
it("admits the exact cumulative byte boundary without charging header labels twice", async () => {
const table = `<table><tr><th>${"x".repeat(32)}</th></tr></table>`;
const artifact = await createNativeHtmlParser().parse(input(table + table));
expect(artifact.elements.map((element) => element.text)).toEqual([
"x".repeat(32),
"x".repeat(32),
]);
});
it.each(["html", "md", "mdx"])(
"keeps independent %s parses byte-budget isolated",
async (extension) => {
const parser = extension === "html" ? createNativeHtmlParser() : createNativeMarkdownParser();
const source = input(extension === "html" ? htmlTable : markdownTable, extension);
expect((await parser.parse(source)).elements[0]?.text).toBe(label);
expect((await parser.parse(source)).elements[0]?.text).toBe(label);
},
);
it("keeps independent provider transports byte-budget isolated", async () => {
const table = { type: "Table", metadata: { text_as_html: htmlTable } };
const fetch = vi.fn(async () => Response.json([table]));
const parser = createUnstructuredParserClient({ endpoint: "http://parser.invalid", fetch });
const source = input("%PDF-1.4", "pdf");
expect((await parser.parse(source)).elements[0]?.text).toBe(label);
expect((await parser.parse(source)).elements[0]?.text).toBe(label);
expect(fetch).toHaveBeenCalledTimes(2);
});
});

218
knowledge-fs/packages/parsers/src/html-table-expansion.test.ts Normal file
View File

@ -0,0 +1,218 @@
import { describe, expect, it, vi } from "vitest";
import {
createNativeHtmlParser,
createNativeMarkdownParser,
createUnstructuredParserClient,
} from "./index";
vi.mock("./parser-resource-budget", async (importOriginal) => {
const actual = await importOriginal<typeof import("./parser-resource-budget")>();
return {
...actual,
parserResourceLimits: {
...actual.parserResourceLimits,
maxOutputBytes: 256,
maxTableCells: 6,
maxTableColumns: 4,
},
};
});
function parseTable(table: string) {
return createNativeHtmlParser().parse({
body: new TextEncoder().encode(table),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: "table.html",
mimeType: "text/html",
version: 1,
});
}
describe("HTML table pre-expansion resource budget", () => {
it("shares the expansion budget across separate empty HTML tables", async () => {
const table = '<table><tr><td colspan="4"></td></tr></table>';
await expect(parseTable(table + table)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it("charges ragged-row padding against the document budget before dense projection", async () => {
const table = "<table><tr><td>A</td></tr><tr><td>B</td><td>C</td></tr></table>";
await expect(parseTable(table + table)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it.each(["md", "mdx"])(
"shares the expansion budget across separate %s HTML tokens",
async (extension) => {
const table = '<table><tr><td colspan="4"></td></tr></table>';
await expect(
createNativeMarkdownParser().parse({
body: new TextEncoder().encode(`${table}\n\nBetween\n\n${table}`),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: `table.${extension}`,
mimeType: "text/markdown",
version: 1,
}),
).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
},
);
it("shares the expansion budget across separate provider tables", async () => {
const table = {
type: "Table",
metadata: { text_as_html: '<table><tr><td colspan="4"></td></tr></table>' },
};
const parser = createUnstructuredParserClient({
endpoint: "http://parser.invalid",
fetch: async () => new Response(JSON.stringify([table, table])),
});
await expect(
parser.parse({
body: new TextEncoder().encode("%PDF-1.4\n"),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: "table.pdf",
mimeType: "application/pdf",
version: 1,
}),
).rejects.toMatchObject({
code: "provider_response_invalid",
retryable: false,
cause: { message: expect.stringContaining("expanded HTML table cell count") },
});
});
it("does not retain provider table budget between independent transports", async () => {
const table = {
type: "Table",
metadata: {
text_as_html: '<table><tr><td colspan="3" rowspan="2"></td></tr><tr></tr></table>',
},
};
const fetch = vi.fn(async () => new Response(JSON.stringify([table])));
const parser = createUnstructuredParserClient({ endpoint: "http://parser.invalid", fetch });
const input = {
body: new TextEncoder().encode("%PDF-1.4\n"),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: "table.pdf",
mimeType: "application/pdf",
version: 1,
};
await expect(parser.parse(input)).resolves.toMatchObject({ parser: "unstructured" });
await expect(parser.parse(input)).resolves.toMatchObject({ parser: "unstructured" });
expect(fetch).toHaveBeenCalledTimes(2);
});
it.each(["html", "md", "mdx"])(
"does not share the expansion budget across independent %s parses",
async (extension) => {
const parser = extension === "html" ? createNativeHtmlParser() : createNativeMarkdownParser();
const input = {
body: new TextEncoder().encode(
'<table><tr><td colspan="3" rowspan="2"></td></tr><tr></tr></table>',
),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: `table.${extension}`,
mimeType: extension === "html" ? "text/html" : "text/markdown",
version: 1,
};
await expect(parser.parse(input)).resolves.toMatchObject({ elements: [] });
await expect(parser.parse(input)).resolves.toMatchObject({ elements: [] });
},
);
it("rejects an overwide empty colspan before filtered rows can hide its allocation", async () => {
await expect(parseTable('<table><tr><td colspan="5"></td></tr></table>')).rejects.toMatchObject(
{
code: "provider_input",
retryable: false,
message: expect.stringContaining("HTML table column count"),
},
);
});
it("charges cumulative empty cell spans even when no row produces searchable text", async () => {
const html = '<table><tr><td colspan="4"></td></tr><tr><td colspan="4"></td></tr></table>';
await expect(parseTable(html)).rejects.toMatchObject({
code: "provider_input",
retryable: false,
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it("charges carried rowspans against the same cumulative cell budget", async () => {
const html = '<table><tr><td colspan="3" rowspan="3"></td></tr><tr></tr><tr></tr></table>';
await expect(parseTable(html)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it("charges the logical width of sparse carried rows, not just their nonempty cell count", async () => {
const html =
'<table><tr><td colspan="3"></td><td rowspan="3"></td></tr><tr></tr><tr></tr></table>';
await expect(parseTable(html)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it("bounds dense header traversal before flattening ragged rows", async () => {
const html = '<table><tr><th colspan="2">A</th></tr><tr><th colspan="4">B</th></tr></table>';
await expect(parseTable(html)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it("bounds dense headerless traversal before inferring column types", async () => {
const html = '<table><tr><td colspan="2">A</td></tr><tr><td colspan="4">B</td></tr></table>';
await expect(parseTable(html)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("expanded HTML table cell count"),
});
});
it("bounds repeated header label bytes before constructing joined columns", async () => {
const html = `<table><tr><th colspan="4">${"x".repeat(65)}</th></tr></table>`;
await expect(parseTable(html)).rejects.toMatchObject({
code: "provider_input",
message: expect.stringContaining("HTML table header bytes"),
});
});
it("preserves empty tables exactly at the expansion boundary", async () => {
const artifact = await parseTable(
'<table><tr><td colspan="3" rowspan="2"></td></tr><tr></tr></table>',
);
expect(artifact.elements).toEqual([]);
});
it("preserves admitted multirow headers and rowspan carry semantics", async () => {
const html =
'<table><tr><th rowspan="2">Name</th><th>Q1</th></tr><tr><th>Q2</th></tr><tr><td>A</td><td>1</td></tr></table>';
const artifact = await parseTable(html);
expect(artifact.elements[0]?.text).toBe("Name: A | Q1 / Q2: 1");
expect(artifact.elements[0]?.metadata.table).toMatchObject({
columns: ["Name", "Q1 / Q2"],
headerRowCount: 2,
recordCount: 1,
sourceRowCount: 3,
});
});
it("preserves colspan carry before a later source cell without double charging", async () => {
const html = '<table><tr><td rowspan="2" colspan="2">X</td></tr><tr><td>Y</td></tr></table>';
const artifact = await parseTable(html);
expect(artifact.elements[0]?.text).toBe(
"column_1: X | column_2: X | column_3:\ncolumn_1: X | column_2: X | column_3: Y",
);
});
});

1076
knowledge-fs/packages/parsers/src/index.ts
View File

File diff suppressed because it is too large Load Diff

211
knowledge-fs/packages/parsers/src/native-markup-fidelity.test.ts Normal file
View File

@ -0,0 +1,211 @@
import { describe, expect, it } from "vitest";
import { createNativeHtmlParser, createNativeMarkdownParser } from "./index";
const input = (body: string, extension: "html" | "md" | "mdx" = "md") => ({
body: new TextEncoder().encode(body),
documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44",
filename: `fixture.${extension}`,
mimeType: extension === "html" ? "text/html" : "text/markdown",
version: 1,
});
describe("native markup content fidelity", () => {
it("retains nested blockquotes and their images in section order", async () => {
const artifact = await createNativeMarkdownParser().parse(
input(
"# Rules\n\n> Quoted policy\n>\n> > Nested policy\n>\n> ![Evidence](proof.png)\n\nFinal policy",
),
);
expect(artifact.elements.map((element) => element.text)).toEqual([
"Rules",
"Quoted policy",
"Nested policy",
"Evidence",
"Final policy",
]);
expect(artifact.elements.every((element) => element.sectionPath[0] === "Rules")).toBe(true);
});
it.each(["md", "mdx"] as const)(
"keeps static HTML text in %s without indexing script/style bodies",
async (extension) => {
const artifact = await createNativeMarkdownParser().parse(
input(
"<Callout>Critical <strong>policy</strong><script>secretScript()</script><style>secretStyle</style></Callout>",
extension,
),
);
expect(artifact.elements.map((element) => element.text).join(" ")).toBe("Critical policy");
},
);
it("keeps text on both sides of Markdown images in source order without duplicating image syntax", async () => {
const artifact = await createNativeMarkdownParser().parse(
input(
"Before ![Diagram](a.png) after ![Photo](b.png) ending.\n\n![First](c.png) retained suffix.",
),
);
expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([
["paragraph", "Before"],
["image", "Diagram"],
["paragraph", "after"],
["image", "Photo"],
["paragraph", "ending."],
["image", "First"],
["paragraph", "retained suffix."],
]);
});
it("retains inline HTML text and excludes embedded executable content in Markdown paragraphs", async () => {
const artifact = await createNativeMarkdownParser().parse(
input("Before <span>important</span> <script>hidden()</script> after."),
);
expect(artifact.elements.map((element) => element.text)).toEqual(["Before important after."]);
});
it("preserves each placement caption when the same Markdown image URI is reused", async () => {
const artifact = await createNativeMarkdownParser().parse(
input('![First](same.png "One") ![Second](same.png "Two")'),
);
expect(
artifact.elements.map((element) => [
element.text,
element.metadata.caption,
element.metadata.title,
]),
).toEqual([
["First", "First", "One"],
["Second", "Second", "Two"],
]);
});
it("keeps images nested in Markdown headings and tables", async () => {
const artifact = await createNativeMarkdownParser().parse(
input(
"# Heading ![Badge](badge.png)\n\n| Name | Evidence |\n| --- | --- |\n| Ada | ![Chart](chart.png) |",
),
);
expect(
artifact.elements
.filter((element) => element.type === "image")
.map((element) => element.text),
).toEqual(["Badge", "Chart"]);
expect(artifact.elements[0]).toMatchObject({
sectionPath: ["Heading"],
text: "Heading",
type: "heading",
});
});
it("retains HTML heading/table images without duplicating normal text", async () => {
const artifact = await createNativeHtmlParser().parse(
input(
'<h1>Heading<img src="badge.png" alt="Badge"></h1><table><tr><th>Name</th><th>Evidence</th></tr><tr><td>Ada</td><td><img src="chart.png" alt="Chart"></td></tr></table>',
"html",
),
);
expect(artifact.elements.map((element) => element.type)).toEqual([
"heading",
"image",
"table",
"image",
]);
expect(
artifact.elements
.filter((element) => element.type === "image")
.map((element) => element.text),
).toEqual(["Badge", "Chart"]);
});
it("does not extract images hidden inside excluded HTML subtrees", async () => {
const artifact = await createNativeHtmlParser().parse(
input(
'<h1>Heading<noscript><img src="hidden.png" alt="hidden"></noscript></h1><table><tr><td>Kept<noscript><img src="also-hidden.png"></noscript></td></tr></table>',
"html",
),
);
expect(artifact.elements.map((element) => element.type)).toEqual(["heading", "table"]);
expect(artifact.elements.map((element) => element.text).join(" ")).not.toContain("hidden");
});
it("rejects excessive HTML node counts even when the tree is shallow", async () => {
await expect(
createNativeHtmlParser().parse(
input(`<div>${"<span></span>".repeat(250_001)}</div>`, "html"),
),
).rejects.toMatchObject({
code: "provider_input",
name: "ParserResourceLimitError",
retryable: false,
});
});
it("extracts images inside Markdown lists and links without losing surrounding list content", async () => {
const artifact = await createNativeMarkdownParser().parse(
input("- Before [![Linked](a.png)](https://example.test) after\n- Last item"),
);
expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([
["list", "Before"],
["image", "Linked"],
["list", "after\nLast item"],
]);
});
it("retains direct container text and nested inline images without duplicate text", async () => {
const artifact = await createNativeHtmlParser().parse(
input(
'<h1>Guide</h1><div>Bare <em>text</em><p>Before <a><img src="a.png" alt="Figure"></a> after</p>Trailing text</div>',
"html",
),
);
expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([
["heading", "Guide"],
["paragraph", "Bare text"],
["paragraph", "Before"],
["image", "Figure"],
["paragraph", "after"],
["paragraph", "Trailing text"],
]);
expect(artifact.elements.every((element) => element.sectionPath[0] === "Guide")).toBe(true);
});
it("retains list images, inline spacing, and line breaks without indexing scripts", async () => {
const artifact = await createNativeHtmlParser().parse(
input(
'<ul><li>Before<img src="a.png" alt="Figure">after<script>hidden()</script></li><li>Last<br>line</li></ul>',
"html",
),
);
expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([
["list", "Before"],
["image", "Figure"],
["list", "after\nLast\nline"],
]);
});
it("preserves all figure images and captions instead of selecting only the first image", async () => {
const artifact = await createNativeHtmlParser().parse(
input(
'<figure><img src="a.png"><img src="b.png"><figcaption>Shared caption</figcaption></figure>',
"html",
),
);
expect(artifact.elements.map((element) => [element.type, element.text])).toEqual([
["image", "Shared caption"],
["image", "Shared caption"],
]);
});
it("rejects overly deep HTML with a non-retryable resource error instead of overflowing the stack", async () => {
await expect(
createNativeHtmlParser().parse(
input(`${"<div>".repeat(5_000)}text${"</div>".repeat(5_000)}`, "html"),
),
).rejects.toMatchObject({
code: "provider_input",
name: "ParserResourceLimitError",
retryable: false,
});
});
});

91
knowledge-fs/packages/parsers/src/office-admission-integration.test.ts Normal file
View File

@ -0,0 +1,91 @@
import { strToU8, zipSync } from "fflate";
import { describe, expect, it, vi } from "vitest";
import { createUnstructuredParserClient } from "./index";
const identity = { documentAssetId: "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", version: 1 };
const spreadsheetMime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
describe("Office resource admission at the provider boundary", () => {
it("rejects tiny sparse workbooks before the provider or retries and releases the admission slot", async () => {
const fetch = vi.fn(async () => Response.json([{ type: "NarrativeText", text: "retained" }]));
const parser = createUnstructuredParserClient({
endpoint: "https://parser.invalid",
fetch,
maxRetries: 3,
maxConcurrency: 1,
});
const rejected = {
...identity,
filename: "sparse.xlsx",
mimeType: spreadsheetMime,
body: zipSync({
"xl/worksheets/sheet1.xml": strToU8(
'<worksheet><sheetData><row r="1048576"><c r="XFD1048576"><v>1</v></c></row></sheetData></worksheet>',
),
}),
};
await expect(parser.parse(rejected)).rejects.toMatchObject({
code: "provider_input",
retryable: false,
requestOutcomeAmbiguous: false,
});
expect(fetch).not.toHaveBeenCalled();
const artifact = await parser.parse({
...rejected,
version: 2,
body: zipSync({
"xl/worksheets/sheet1.xml": strToU8(
'<worksheet><sheetData><row r="1"><c r="A1"><v>1</v></c></row></sheetData></worksheet>',
),
}),
});
expect(artifact.elements[0]?.text).toBe("retained");
expect(fetch).toHaveBeenCalledOnce();
});
it.each(["docx", "pptx", "odt", "epub"])(
"rejects oversized XML structure in %s before sending it",
async (extension) => {
const fetch = vi.fn(async () => Response.json([]));
const parser = createUnstructuredParserClient({ endpoint: "https://parser.invalid", fetch });
await expect(
parser.parse({
...identity,
filename: `deep.${extension}`,
mimeType: "application/octet-stream",
body: zipSync({
"content.xml": strToU8("<document>".repeat(129) + "</document>".repeat(129)),
}),
}),
).rejects.toMatchObject({ code: "provider_input", retryable: false });
expect(fetch).not.toHaveBeenCalled();
},
);
it("keeps ordinary archive bytes and provider extraction options unchanged", async () => {
const body = zipSync({
"word/document.xml": strToU8(
'<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>hello</w:t></w:r></w:p></w:body></w:document>',
),
});
const fetch = vi.fn(async (request: RequestInfo | URL) => {
if (!(request instanceof Request)) throw new Error("Expected Request");
const form = await request.formData();
const file = form.get("files");
if (!(file instanceof File)) throw new Error("Expected file");
expect(new Uint8Array(await file.arrayBuffer())).toEqual(body);
expect(form.get("strategy")).toBe("auto");
return Response.json([{ type: "NarrativeText", text: "hello" }]);
});
const parser = createUnstructuredParserClient({ endpoint: "https://parser.invalid", fetch });
const artifact = await parser.parse({
...identity,
body,
filename: "ordinary.docx",
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
});
expect(artifact.elements[0]?.text).toBe("hello");
expect(fetch).toHaveBeenCalledOnce();
});
});

596
knowledge-fs/packages/parsers/src/office-parser-preflight.test.ts Normal file
View File

@ -0,0 +1,596 @@
import { Zip, ZipDeflate, ZipPassThrough, strToU8, zipSync } from "fflate";
import { describe, expect, it, vi } from "vitest";
import { OfficeArchiveAdmissionError, assertOfficeArchiveSafe } from "./office-parser-preflight";
const spreadsheetMime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
function spreadsheetInput(xml: string, extraEntries: Record<string, Uint8Array> = {}) {
return {
body: zipSync({
"[Content_Types].xml": strToU8(
'<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>',
),
"xl/workbook.xml": strToU8(
'<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheets><sheet name="Sheet1" sheetId="1"/></sheets></workbook>',
),
"xl/worksheets/sheet1.xml": strToU8(xml),
...extraEntries,
}),
filename: "workbook.xlsx",
mimeType: spreadsheetMime,
};
}
function sheetXml(contents: string, dimension = "A1:B2"): string {
return `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><dimension ref="${dimension}"/><sheetData>${contents}</sheetData></worksheet>`;
}
describe("assertOfficeArchiveSafe", () => {
it("rejects a tiny sparse workbook before its rectangular cell span can be materialized", async () => {
const input = spreadsheetInput(
sheetXml(
'<row r="1"><c r="A1" t="inlineStr"><is><t>first</t></is></c></row><row r="1048576"><c r="XFD1048576" t="inlineStr"><is><t>last</t></is></c></row>',
"A1:XFD1048576",
),
);
expect(input.body.byteLength).toBeLessThan(2000);
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow(OfficeArchiveAdmissionError);
});
it("preserves an ordinary spreadsheet without changing source bytes", async () => {
const input = spreadsheetInput(
sheetXml(
'<row r="1"><c r="A1" t="inlineStr"><is><t>Name</t></is></c><c r="B1"><v>5</v></c></row>',
),
);
const original = input.body.slice();
await expect(assertOfficeArchiveSafe(input)).resolves.toBeUndefined();
expect(input.body).toEqual(original);
});
it.each(["xl/alternate.xml", "custom/sheet-part"])(
"detects worksheet content even if relationships select an unconventional part %s",
async (part) => {
const input = spreadsheetInput(sheetXml(""), {
[part]: strToU8(
sheetXml('<row r="90000"><c r="XFD90000"><v>1</v></c></row>', "A1:XFD90000"),
),
});
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("dense-cell budget");
},
);
it("does not let leading whitespace hide a renamed XML worksheet", async () => {
const input = spreadsheetInput(sheetXml(""), {
"custom/sheet-part": strToU8(" ".repeat(4096) + sheetXml("", "A1:XFD1048576")),
});
// Store the member without compression so its prefix spans several admission chunks.
input.body = zipSync(
{
"xl/workbook.xml": strToU8("<workbook/>"),
"custom/sheet-part": strToU8(" ".repeat(4096) + sheetXml("", "A1:XFD1048576")),
},
{ level: 0 },
);
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("dense-cell budget");
});
it("checks Office contents even when both filename and MIME type are misleading", async () => {
const input = spreadsheetInput(sheetXml("", "A1:XFD1048576"));
await expect(
assertOfficeArchiveSafe({ ...input, filename: "plain.txt", mimeType: "text/plain" }),
).rejects.toThrow(OfficeArchiveAdmissionError);
});
it.each([
[
"document.docx",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"word/document.xml",
"document",
],
[
"slides.pptx",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"ppt/presentation.xml",
"presentation",
],
["document.odt", "application/vnd.oasis.opendocument.text", "content.xml", "document-content"],
["book.epub", "application/epub+zip", "META-INF/container.xml", "container"],
])("accepts bounded ordinary %s archives", async (filename, mimeType, part, root) => {
const body = zipSync({
[part]: strToU8(`<${root}><text>Ordinary content &amp; text</text></${root}>`),
});
await expect(assertOfficeArchiveSafe({ body, filename, mimeType })).resolves.toBeUndefined();
});
it("preserves an EPUB's harmless HTML doctype", async () => {
await expect(
assertOfficeArchiveSafe({
body: zipSync({
"META-INF/container.xml": strToU8("<container/>"),
"OPS/chapter.xhtml": strToU8(
'<!DOCTYPE html><html xmlns="http://www.w3.org/1999/xhtml"><body><p>Chapter</p></body></html>',
),
}),
filename: "book.epub",
mimeType: "application/epub+zip",
}),
).resolves.toBeUndefined();
});
it.each([
["-//W3C//DTD XHTML 1.1//EN", "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"],
["-//W3C//DTD XHTML 1.0 Strict//EN", "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"],
[
"-//W3C//DTD XHTML 1.0 Transitional//EN",
"https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd",
],
])(
"preserves the standard EPUB XHTML PUBLIC doctype %s without resolving it",
async (publicId, systemId) => {
const body = zipSync({
"META-INF/container.xml": strToU8("<container/>"),
"OPS/chapter.xhtml": strToU8(
`<!DOCTYPE html PUBLIC "${publicId}" "${systemId}"><html><body><p>Chapter</p></body></html>`,
),
});
await expect(
assertOfficeArchiveSafe({ body, filename: "book.epub", mimeType: "application/epub+zip" }),
).resolves.toBeUndefined();
},
);
it.each([
'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "https://example.invalid/evil.dtd">',
'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd" [<!ENTITY evil "payload">]>',
'<!DOCTYPE html SYSTEM "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">',
])("does not admit custom or internally extended XHTML DTDs", async (doctype) => {
const body = zipSync({
"META-INF/container.xml": strToU8("<container/>"),
"OPS/chapter.xhtml": strToU8(`${doctype}<html><body/></html>`),
});
await expect(
assertOfficeArchiveSafe({ body, filename: "book.epub", mimeType: "application/epub+zip" }),
).rejects.toThrow("external entities");
});
it.each([
[
'<sheet name="One" r:id="r1"/>',
'<Relationship Id="r1" Target="https://example.invalid/sheet" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"/>',
"safe internal",
],
[
'<sheet name="One" r:id="r1"/>',
'<Relationship Id="r1" Target="worksheets/sheet1.xml" TargetMode="External" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"/>',
"external",
],
['<sheet name="One" r:id="r1"/>', "", "missing"],
['<sheet name="One" r:id="r1" s:id="r2"/>', "", "ambiguous"],
[
'<sheet name="One" r:id="r1"/>',
'<Relationship Id="r1" Target="../../outside.xml" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"/>',
"escapes",
],
])("rejects unsafe worksheet relationships", async (sheets, relations, message) => {
const input = spreadsheetInput(sheetXml(""), {
"xl/workbook.xml": strToU8(
`<workbook xmlns:r="urn:rel" xmlns:s="urn:rel"><sheets>${sheets}</sheets></workbook>`,
),
"xl/_rels/workbook.xml.rels": strToU8(`<Relationships>${relations}</Relationships>`),
});
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow(message);
});
it("bounds logical sheet count rather than only ZIP part count", async () => {
const input = spreadsheetInput(sheetXml(""), {
"xl/workbook.xml": strToU8(
'<workbook><sheets><sheet name="One"/><sheet name="Two"/></sheets></workbook>',
),
});
await expect(assertOfficeArchiveSafe(input, { maxWorksheets: 1 })).rejects.toThrow(
"too many worksheets",
);
});
it("keeps column formatting indices inside the format limit", async () => {
await expect(
assertOfficeArchiveSafe(
spreadsheetInput('<worksheet><cols><col min="1" max="20000"/></cols></worksheet>'),
),
).rejects.toThrow("column formatting");
});
it.each(["utf16le", "utf16be"])(
"recognizes %s encoded spreadsheet dimensions",
async (encoding) => {
const bytes = Buffer.from(
`\ufeff<?xml version="1.0" encoding="UTF-16"?>${sheetXml("", "A1:XFD1048576")}`,
"utf16le",
);
if (encoding === "utf16be") bytes.swap16();
const input = spreadsheetInput(sheetXml(""), { "xl/worksheets/sheet1.xml": bytes });
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("dense-cell budget");
},
);
it("rejects a UTF-32 part instead of misreading its NUL-padded tags as unimportant XML", async () => {
const xml = sheetXml("", "A1:XFD1048576");
const bytes = Uint8Array.from(
[...xml].flatMap((character) => [character.charCodeAt(0), 0, 0, 0]),
);
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml(""), { "custom/worksheet": bytes })),
).rejects.toThrow(OfficeArchiveAdmissionError);
});
it.each(["utf16le", "utf16be"])(
"recognizes BOM-less %s XML with a whitespace prefix",
async (encoding) => {
const bytes = Buffer.from(` \n${sheetXml("", "A1:XFD1048576")}`, "utf16le");
if (encoding === "utf16be") bytes.swap16();
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml(""), { "custom/worksheet": bytes })),
).rejects.toThrow(OfficeArchiveAdmissionError);
},
);
it("handles inferred coordinates and legal column format bounds", async () => {
const xml =
'<worksheet><cols><col min="1" max="3"/></cols><sheetData><row><c/><c/></row><row><c/></row></sheetData><mergeCells><mergeCell ref="A1:C2"/></mergeCells></worksheet>';
await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).resolves.toBeUndefined();
});
it("admits a long narrow table inside the dense-cell budget", async () => {
const input = spreadsheetInput(
sheetXml(
'<row r="1"><c r="A1"><v>1</v></c></row><row r="90000"><c r="A90000"><v>2</v></c></row>',
"A1:A90000",
),
);
await expect(assertOfficeArchiveSafe(input)).resolves.toBeUndefined();
});
it("does not treat a default column style as populated spreadsheet columns", async () => {
const xml =
'<worksheet><dimension ref="A1:A90000"/><cols><col min="1" max="16384" width="12"/></cols><sheetData><row r="90000"><c r="A90000"><v>1</v></c></row></sheetData></worksheet>';
await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).resolves.toBeUndefined();
});
it.each([
['<row r="0"/>', "positive coordinate"],
['<row r="1.5"/>', "positive coordinate"],
['<row r="99999999999999999999999"/>', "positive coordinate"],
['<row r="1"><c r="B2"/></row>', "coordinates disagree"],
['<c r="A1"/>', "outside a row"],
['<row r="1"><row r="2"/></row>', "nested rows"],
['<row r="1"/><row r="1"/>', "increasing"],
['<row r="1"><c r="B1"/><c r="A1"/></row>', "increasing"],
])("rejects unsafe or ambiguous spreadsheet coordinates: %s", async (contents, message) => {
await expect(assertOfficeArchiveSafe(spreadsheetInput(sheetXml(contents)))).rejects.toThrow(
message,
);
});
it.each(["A0", "A1:B2:C3", "a1", "AAAA1", "A1 B2"])(
"rejects invalid worksheet dimension %s",
async (dimension) => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("", dimension))),
).rejects.toThrow("invalid");
},
);
it("bounds inferred coordinates when row and cell references are omitted", async () => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("<row><c/><c/><c/></row>")), {
maxSheetCellSpan: 4,
}),
).rejects.toThrow("dense-cell budget");
});
it("bounds rectangular area accumulated across individually safe sheets", async () => {
const input = spreadsheetInput(sheetXml("", "A1:C3"), {
"xl/worksheets/sheet2.xml": strToU8(sheetXml("", "A1:C3")),
});
await expect(
assertOfficeArchiveSafe(input, { maxSheetCellSpan: 10, maxWorkbookCellSpan: 15 }),
).rejects.toThrow("workbook exceeds");
});
it("counts worksheet relationships that load the same physical part more than once", async () => {
const input = spreadsheetInput(sheetXml("", "A1:C3"), {
"xl/workbook.xml": strToU8(
'<workbook xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="One" sheetId="1" r:id="rId1"/><sheet name="Two" sheetId="2" r:id="rId2"/></sheets></workbook>',
),
"xl/_rels/workbook.xml.rels": strToU8(
'<Relationships><Relationship Id="rId1" Target="worksheets/sheet1.xml" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"/><Relationship Id="rId2" Target="/xl/worksheets/sheet1.xml" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"/></Relationships>',
),
});
await expect(assertOfficeArchiveSafe(input, { maxWorkbookCellSpan: 15 })).rejects.toThrow(
"workbook exceeds",
);
await expect(
assertOfficeArchiveSafe(input, { maxWorkbookCellSpan: 20 }),
).resolves.toBeUndefined();
});
it("bounds actual cell count independently of occupied area", async () => {
await expect(
assertOfficeArchiveSafe(
spreadsheetInput(sheetXml('<row r="1"><c r="A1"/><c r="B1"/><c r="C1"/></row>', "A1:C1")),
{ maxWorkbookCells: 2 },
),
).rejects.toThrow("too many cells");
});
it("bounds shared string count including rich strings", async () => {
const input = spreadsheetInput(sheetXml(""), {
"xl/sharedStrings.xml": strToU8("<sst><si><t>a</t></si><si><r><t>b</t></r></si></sst>"),
});
await expect(assertOfficeArchiveSafe(input, { maxSharedStrings: 1 })).rejects.toThrow(
"shared strings",
);
});
it.each([
'<!DOCTYPE worksheet [<!ENTITY payload "abc">]><worksheet>&payload;</worksheet>',
'<!DOCTYPE worksheet SYSTEM "https://example.invalid/schema.dtd"><worksheet/>',
])("rejects DTDs without resolving entities or fetching URLs", async (xml) => {
await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).rejects.toThrow(
"external entities",
);
});
it("bounds XML depth", async () => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput("<worksheet><a><b><c/></b></a></worksheet>"), {
maxXmlDepth: 3,
}),
).rejects.toThrow("structural complexity");
});
it.each([
`<worksheet oversized="${"a".repeat(16385)}"/>`,
`<worksheet ${Array.from({ length: 129 }, (_, index) => `attr${index}="x"`).join(" ")}/>`,
])("bounds XML attributes", async (xml) => {
await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).rejects.toThrow(
"attributes exceed",
);
});
it("retains bounded binary media without attempting UTF-8 or XML parsing", async () => {
const input = spreadsheetInput(sheetXml(""), {
"xl/media/image.png": Uint8Array.from([0x89, 0x50, 0x4e, 0x47, 0xff, 0x00]),
"custom/readme": strToU8(" not XML"),
"custom/small": strToU8("a"),
"custom/empty": new Uint8Array(0),
});
await expect(assertOfficeArchiveSafe(input)).resolves.toBeUndefined();
});
it("rejects a source that has damaged UTF-8 in a declared XML member", async () => {
await expect(
assertOfficeArchiveSafe(
spreadsheetInput(sheetXml(""), {
"custom/data.xml": Uint8Array.from([0xff, 0x00, 0x81, 0x81]),
}),
),
).rejects.toThrow("cannot be inspected safely");
});
it("does not permit local sizes to differ from central-directory sizes", async () => {
const input = spreadsheetInput(sheetXml(""));
new DataView(input.body.buffer).setUint32(22, 1, true);
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("local entries disagree");
});
it("bounds actual inflation even when both size headers understate a member", async () => {
const input = {
body: zipSync({ "xl/workbook.xml": strToU8(`<workbook>${"a".repeat(4000)}</workbook>`) }),
filename: "book.xlsx",
mimeType: spreadsheetMime,
};
const central = Buffer.from(input.body).indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
const view = new DataView(input.body.buffer);
view.setUint32(22, 10, true);
view.setUint32(central + 24, 10, true);
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("actual expansion");
});
it("detects overstated actual member sizes", async () => {
const input = {
body: zipSync({ "xl/workbook.xml": strToU8("<workbook/>") }),
filename: "book.xlsx",
mimeType: spreadsheetMime,
};
const central = Buffer.from(input.body).indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
const view = new DataView(input.body.buffer);
view.setUint32(22, 20, true);
view.setUint32(central + 24, 20, true);
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("actual entry size");
});
it("rejects unsupported ZIP compression before starting a decoder", async () => {
const input = spreadsheetInput(sheetXml(""));
const central = Buffer.from(input.body).indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02]));
new DataView(input.body.buffer).setUint16(central + 10, 99, true);
await expect(assertOfficeArchiveSafe(input)).rejects.toThrow("compression is unsupported");
});
it("bounds XML nodes across parts", async () => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxXmlNodes: 2 }),
).rejects.toThrow("structural complexity");
});
it.each(["<worksheet><row></worksheet>", "<worksheet>", "", "<worksheet/><worksheet/>"])(
"rejects incomplete or structurally ambiguous XML %s",
async (xml) => {
await expect(assertOfficeArchiveSafe(spreadsheetInput(xml))).rejects.toThrow(
OfficeArchiveAdmissionError,
);
},
);
it("rejects unsupported XML encodings explicitly", async () => {
await expect(
assertOfficeArchiveSafe(
spreadsheetInput('<?xml version="1.0" encoding="ISO-8859-1"?><worksheet/>'),
),
).rejects.toThrow("encoding is unsupported");
});
it("bounds per-part declared XML bytes before decompression", async () => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxXmlEntryBytes: 20 }),
).rejects.toThrow("XML part");
});
it("bounds aggregate actual XML bytes", async () => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxXmlBytes: 200 }),
).rejects.toThrow("XML expansion");
});
it("bounds archive bytes including non-XML media", async () => {
const input = spreadsheetInput(sheetXml(""), { "xl/media/image.png": new Uint8Array(600) });
await expect(assertOfficeArchiveSafe(input, { maxArchiveBytes: 700 })).rejects.toThrow(
"actual expansion",
);
});
it("bounds entries before allocating extracted data", async () => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxEntries: 2 }),
).rejects.toThrow("too many entries");
});
it.each([
"../outside.xml",
"word/../outside.xml",
"word\\document.xml",
"/absolute.xml",
"nul\0.xml",
"word/./document.xml",
])("rejects ambiguous archive path %s", async (name) => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml(""), { [name]: strToU8("<a/>") })),
).rejects.toThrow("entry names");
});
it("rejects case-ambiguous part names", async () => {
await expect(
assertOfficeArchiveSafe(
spreadsheetInput(sheetXml(""), { "XL/WORKBOOK.XML": strToU8("<workbook/>") }),
),
).rejects.toThrow("entry names");
});
it.each([ZipDeflate, ZipPassThrough])(
"accepts lawful streamed ZIP data descriptors (%s)",
async (Member) => {
const chunks: Uint8Array[] = [];
const archive = new Zip((error, chunk) => {
if (error) throw error;
chunks.push(chunk);
});
const member = new Member("xl/worksheets/sheet1.xml");
archive.add(member);
const xml = strToU8(sheetXml('<row r="1"><c r="A1"><v>1</v></c></row>'));
member.push(xml.subarray(0, 15), false);
member.push(xml.subarray(15), true);
archive.end();
await expect(
assertOfficeArchiveSafe({
body: Buffer.concat(chunks),
filename: "streamed.xlsx",
mimeType: spreadsheetMime,
}),
).resolves.toBeUndefined();
},
);
it("retains cancellation reason without wrapping it as an input failure", async () => {
const controller = new AbortController();
const reason = new Error("lease lost");
controller.abort(reason);
await expect(
assertOfficeArchiveSafe({ ...spreadsheetInput(sheetXml("")), signal: controller.signal }),
).rejects.toBe(reason);
});
it("yields while inspecting so cancellation can stop further decompression", async () => {
const controller = new AbortController();
const reason = new Error("client disconnected");
const body = zipSync(
{
"xl/workbook.xml": strToU8("<workbook/>"),
"xl/media/image.png": new Uint8Array(128 * 1024),
},
{ level: 0 },
);
const timer = setTimeout(() => controller.abort(reason), 0);
try {
await expect(
assertOfficeArchiveSafe({
body,
filename: "large.xlsx",
mimeType: spreadsheetMime,
signal: controller.signal,
}),
).rejects.toBe(reason);
} finally {
clearTimeout(timer);
}
});
it("bounds inspection time independently of provider timeouts", async () => {
const input = spreadsheetInput(sheetXml(""));
const clock = vi.spyOn(Date, "now").mockReturnValueOnce(0).mockReturnValue(2);
try {
await expect(assertOfficeArchiveSafe(input, { timeoutMs: 1 })).rejects.toThrow(
"processing time",
);
} finally {
clock.mockRestore();
}
});
it("ignores non-Office inputs and unrelated ZIP archives", async () => {
await expect(
assertOfficeArchiveSafe({
body: strToU8("hello"),
filename: "note.txt",
mimeType: "text/plain",
}),
).resolves.toBeUndefined();
await expect(
assertOfficeArchiveSafe({
body: zipSync({ "notes.txt": strToU8("hello") }),
filename: "archive.zip",
mimeType: "application/zip",
}),
).resolves.toBeUndefined();
});
it.each([new Uint8Array([1, 2, 3]), new Uint8Array([0x50, 0x4b]), zipSync({})])(
"rejects Office containers that cannot be inspected",
async (body) => {
await expect(
assertOfficeArchiveSafe({ body, filename: "broken.xlsx", mimeType: spreadsheetMime }),
).rejects.toThrow(OfficeArchiveAdmissionError);
},
);
it.each([0, -1, 1.5, Number.NaN])(
"rejects invalid programmer-supplied limits %s",
async (value) => {
await expect(
assertOfficeArchiveSafe(spreadsheetInput(sheetXml("")), { maxEntries: value }),
).rejects.toThrow("positive safe integer");
},
);
});

588
knowledge-fs/packages/parsers/src/office-parser-preflight.ts Normal file
View File

@ -0,0 +1,588 @@
import { Unzip, type UnzipFileInfo, UnzipInflate, unzipSync } from "fflate";
import { Parser } from "htmlparser2";
import { classifyUnstructuredWorkload } from "./unstructured-workload-policy";
export interface OfficeArchivePreflightInput {
readonly body: Uint8Array;
readonly filename: string;
readonly mimeType: string;
readonly signal?: AbortSignal;
}
export interface OfficeArchivePreflightLimits {
readonly maxArchiveBytes: number;
readonly maxEntries: number;
readonly maxXmlBytes: number;
readonly maxXmlEntryBytes: number;
readonly maxXmlDepth: number;
readonly maxXmlNodes: number;
readonly maxSheetRows: number;
readonly maxSheetColumns: number;
readonly maxSheetCellSpan: number;
readonly maxWorkbookCellSpan: number;
readonly maxWorkbookCells: number;
readonly maxSharedStrings: number;
readonly maxWorksheets: number;
readonly timeoutMs: number;
}
/**
* The cell-span caps bound the *dense* Pandas/NetworkX work in the pinned provider. They are not
* Excel format limits. A very sparse or unusually large valid workbook can intentionally fail
* admission; no cells are silently discarded and the original document is never rewritten.
*/
export const defaultOfficeArchivePreflightLimits: OfficeArchivePreflightLimits = Object.freeze({
maxArchiveBytes: 512 * 1024 * 1024,
maxEntries: 4096,
maxXmlBytes: 64 * 1024 * 1024,
maxXmlEntryBytes: 16 * 1024 * 1024,
maxXmlDepth: 128,
maxXmlNodes: 1_000_000,
maxSheetRows: 100_000,
maxSheetColumns: 16_384,
maxSheetCellSpan: 250_000,
maxWorkbookCellSpan: 500_000,
maxWorkbookCells: 500_000,
maxSharedStrings: 200_000,
maxWorksheets: 256,
timeoutMs: 30_000,
});
export class OfficeArchiveAdmissionError extends Error {
readonly code = "office_archive_input";
constructor(message: string, options?: ErrorOptions) {
super(message, options);
this.name = "OfficeArchiveAdmissionError";
}
}
interface ArchiveBudget {
bytes: number;
cells: number;
sharedStrings: number;
workbookCellSpan: number;
xmlBytes: number;
xmlNodes: number;
worksheets: number;
readonly worksheetAreas: Map<string, number>;
readonly worksheetReferences: { readonly source: string; readonly id: string }[];
readonly worksheetRelationships: Map<string, Map<string, string>>;
}
const compressedChunkBytes = 1024;
const cooperativeYieldBytes = 64 * 1024;
const maxXmlAttributes = 128;
const maxXmlAttributeChars = 16 * 1024;
/**
* Resource admission for OOXML/ODF/EPUB; not a document parser or a general-purpose ZIP validator.
* The existing bounded classifier establishes a classic central-directory bound. fflate reads
* that directory without inflating, then streams actual members in small compressed chunks.
* Local names/sizes must match the directory and actual expansion is counted before XML parsing.
* No expanded archive or XML DOM is retained, and no external entities/resources are loaded.
*/
export async function assertOfficeArchiveSafe(
input: OfficeArchivePreflightInput,
options: Partial<OfficeArchivePreflightLimits> = {},
): Promise<void> {
const claimedOffice = claimsOfficeArchive(input);
if (!claimedOffice && !hasZipSignature(input.body)) return;
input.signal?.throwIfAborted();
const limits = { ...defaultOfficeArchivePreflightLimits, ...options };
for (const [name, value] of Object.entries(limits)) {
if (!Number.isSafeInteger(value) || value < 1) {
throw new Error(`Office archive preflight ${name} must be a positive safe integer`);
}
}
const start = Date.now();
const assertActive = () => {
input.signal?.throwIfAborted();
if (Date.now() - start > limits.timeoutMs) {
reject("Office archive inspection exceeded its bounded processing time");
}
};
try {
const classification = classifyUnstructuredWorkload(input);
if (classification.kind === "rejected" || classification.reason === "archive-invalid") {
// Non-Office archives retain their existing router policy. The outer classifier still
// rejects proven ZIP expansion hazards, independently of this Office-specific check.
if (!claimedOffice) return;
reject("Office archive metadata is unsafe or cannot be inspected safely");
}
const directory = new Map<string, UnzipFileInfo>();
unzipSync(input.body, {
filter: (entry) => {
if (directory.size >= limits.maxEntries) reject("Office archive has too many entries");
if (directory.has(entry.name)) reject("Office archive contains duplicate entry names");
directory.set(entry.name, entry);
return false;
},
});
if (!claimedOffice && ![...directory.keys()].some(isOfficePart)) return;
if (directory.size === 0) reject("Office archive contains no inspectable entries");
const normalizedNames = new Set<string>();
for (const entry of directory.values()) {
assertActive();
const normalizedName = entry.name.toLowerCase();
if (!safeArchivePath(entry.name) || normalizedNames.has(normalizedName)) {
reject("Office archive contains ambiguous or unsafe entry names");
}
normalizedNames.add(normalizedName);
if (entry.compression !== 0 && entry.compression !== 8) {
reject("Office archive compression is unsupported for safe inspection");
}
if (entry.originalSize > limits.maxArchiveBytes)
reject("Office archive expansion is too large");
if (isXmlPart(entry.name) && entry.originalSize > limits.maxXmlEntryBytes) {
reject("Office XML part exceeds the inspection byte limit");
}
}
const budget: ArchiveBudget = {
bytes: 0,
cells: 0,
sharedStrings: 0,
workbookCellSpan: 0,
xmlBytes: 0,
xmlNodes: 0,
worksheets: 0,
worksheetAreas: new Map(),
worksheetReferences: [],
worksheetRelationships: new Map(),
};
const observed = new Set<string>();
const completed = new Set<string>();
const unzip = new Unzip((file) => {
assertActive();
const expected = directory.get(file.name);
if (
!expected ||
observed.has(file.name) ||
file.compression !== expected.compression ||
(file.size !== undefined && file.size !== expected.size) ||
(file.originalSize !== undefined && file.originalSize !== expected.originalSize)
) {
reject("Office archive local entries disagree with its central directory");
}
observed.add(file.name);
let entryBytes = 0;
// OPC relationships may target XML parts without a conventional path or extension. Sniff
// each member's prefix while streaming; never let a renamed worksheet bypass cell budgets.
const xml = createXmlInspector(file.name, limits, budget);
file.ondata = (error, chunk, final) => {
assertActive();
if (error) throw error;
entryBytes += chunk.byteLength;
budget.bytes += chunk.byteLength;
if (entryBytes > expected.originalSize || budget.bytes > limits.maxArchiveBytes) {
reject("Office archive actual expansion exceeds its declared size or byte limit");
}
xml.push(chunk, final);
if (final) {
if (entryBytes !== expected.originalSize) {
reject("Office archive actual entry size disagrees with its central directory");
}
completed.add(file.name);
}
};
file.start();
});
unzip.register(UnzipInflate);
let lastYieldAt = start;
for (let offset = 0; offset < input.body.byteLength; offset += compressedChunkBytes) {
assertActive();
const end = Math.min(input.body.byteLength, offset + compressedChunkBytes);
unzip.push(input.body.subarray(offset, end), end === input.body.byteLength);
if (end % cooperativeYieldBytes === 0 || Date.now() - lastYieldAt >= 8) {
await new Promise<void>((resolve) => setTimeout(resolve, 0));
lastYieldAt = Date.now();
}
}
assertActive();
if (observed.size !== directory.size || completed.size !== directory.size) {
reject("Office archive has incomplete or inconsistent local entries");
}
let referencedCellSpan = 0;
for (const reference of budget.worksheetReferences) {
const target = budget.worksheetRelationships.get(reference.source)?.get(reference.id);
const area = target === undefined ? undefined : budget.worksheetAreas.get(target);
if (area === undefined)
reject("Spreadsheet worksheet relationship is missing or unsupported");
referencedCellSpan += area;
if (referencedCellSpan > limits.maxWorkbookCellSpan) {
reject(
"Spreadsheet workbook exceeds the safe dense-cell budget through worksheet references",
);
}
}
} catch (error) {
if (input.signal?.aborted) throw input.signal.reason;
if (error instanceof OfficeArchiveAdmissionError) throw error;
throw new OfficeArchiveAdmissionError(
"Office archive is invalid or cannot be inspected safely",
{
cause: error,
},
);
}
}
function claimsOfficeArchive(input: OfficeArchivePreflightInput): boolean {
const filename = input.filename.trim().toLowerCase();
const mime = input.mimeType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
return (
/\.(?:docx|pptx|xlsx|odt|epub)$/u.test(filename) ||
/(?:wordprocessingml|presentationml|spreadsheetml)/u.test(mime) ||
mime === "application/vnd.oasis.opendocument.text" ||
mime === "application/epub+zip"
);
}
function hasZipSignature(body: Uint8Array): boolean {
return body[0] === 0x50 && body[1] === 0x4b;
}
function isOfficePart(name: string): boolean {
return /^(?:word\/document\.xml|ppt\/presentation\.xml|xl\/workbook\.xml|xl\/worksheets\/[^/]+\.xml|meta-inf\/container\.xml|content\.xml)$/iu.test(
name,
);
}
function isXmlPart(name: string): boolean {
return /\.(?:xml|rels|opf|ncx|xhtml)$/iu.test(name);
}
function safeArchivePath(name: string): boolean {
return (
!name.startsWith("/") &&
!name.includes("\\") &&
!name.includes("\0") &&
!name.split("/").some((part) => part === ".." || part === ".")
);
}
function reject(message: string): never {
throw new OfficeArchiveAdmissionError(message);
}
function localXmlName(name: string): string {
return name.slice(name.lastIndexOf(":") + 1);
}
function isAllowedHtmlDoctype(path: string, declaration: string): boolean {
if (!/\.(?:xhtml|html|htm)$/iu.test(path)) return false;
if (/^!DOCTYPE\s+html\s*$/iu.test(declaration)) return true;
// EPUB 2 uses these standard XHTML declarations. They identify vocabulary only: this SAX
// inspector never resolves DTDs, and EPUB chapter content is passed to Pandoc's HTML reader.
// Full matching deliberately excludes internal subsets, entities, and custom external DTDs.
const match = /^!DOCTYPE\s+html\s+PUBLIC\s+(["'])([^"']+)\1\s+(["'])([^"']+)\3\s*$/iu.exec(
declaration,
);
if (!match) return false;
const standardDtds: Readonly<Record<string, string>> = {
"-//W3C//DTD XHTML 1.0 Strict//EN": "www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd",
"-//W3C//DTD XHTML 1.0 Transitional//EN": "www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd",
"-//W3C//DTD XHTML 1.0 Frameset//EN": "www.w3.org/TR/xhtml1/DTD/xhtml1-frameset.dtd",
"-//W3C//DTD XHTML 1.1//EN": "www.w3.org/TR/xhtml11/DTD/xhtml11.dtd",
};
const expected = standardDtds[match[2] ?? ""];
return (
expected !== undefined &&
/^https?:\/\//u.test(match[4] ?? "") &&
match[4]?.replace(/^https?:\/\//u, "") === expected
);
}
function createXmlInspector(
path: string,
limits: OfficeArchivePreflightLimits,
budget: ArchiveBudget,
): { push(chunk: Uint8Array, final: boolean): void } {
let worksheet = /^xl\/worksheets\/[^/]+\.xml$/iu.test(path);
let sharedStrings = /^xl\/sharedstrings\.xml$/iu.test(path);
let workbook = false;
const relationshipSource = sourcePartForRelationshipFile(path);
let activeXml = isXmlPart(path);
let ignoredBinary = false;
let memberBytes = 0;
let countedXmlBytes = 0;
let decoder: TextDecoder | undefined;
let pendingPrefix = new Uint8Array(0);
let depth = 0;
let roots = 0;
let attributeCount = 0;
let lastOpenEndIndex = -1;
let row = 0;
let column = 0;
let maxRow = 0;
let maxColumn = 0;
let inRow = false;
const useCoordinate = (nextRow: number, nextColumn: number) => {
maxRow = Math.max(maxRow, nextRow);
maxColumn = Math.max(maxColumn, nextColumn);
if (
maxRow > limits.maxSheetRows ||
maxColumn > limits.maxSheetColumns ||
maxRow * maxColumn > limits.maxSheetCellSpan
) {
reject("Spreadsheet worksheet dimensions exceed the safe dense-cell budget");
}
};
const useRange = (value: string) => {
const parts = value.split(":");
if (parts.length > 2) reject("Spreadsheet contains an invalid cell range");
for (const part of parts) {
const coordinate = cellCoordinate(part);
useCoordinate(coordinate.row, coordinate.column);
}
};
const parser = new Parser(
{
onopentagname() {
depth += 1;
if (depth === 1) roots += 1;
budget.xmlNodes += 1;
attributeCount = 0;
if (depth > limits.maxXmlDepth || budget.xmlNodes > limits.maxXmlNodes || roots > 1) {
reject("Office XML exceeds its structural complexity budget");
}
},
onattribute(_name, value) {
attributeCount += 1;
if (attributeCount > maxXmlAttributes || value.length > maxXmlAttributeChars) {
reject("Office XML attributes exceed their inspection budget");
}
},
onprocessinginstruction(name, data) {
if (name.startsWith("!") && !isAllowedHtmlDoctype(path, data)) {
reject("Office XML declarations and external entities are not allowed");
}
if (name === "?xml") {
const encoding = /\bencoding\s*=\s*["']([^"']+)["']/iu.exec(data)?.[1];
if (encoding && !/^(?:utf-?8|utf-?16(?:le|be)?)$/iu.test(encoding)) {
reject("Office XML encoding is unsupported for safe inspection");
}
}
},
onopentag(name, attributes) {
lastOpenEndIndex = parser.endIndex;
const tag = localXmlName(name);
if (depth === 1) {
worksheet ||= tag === "worksheet";
sharedStrings ||= tag === "sst";
workbook = tag === "workbook";
}
if (workbook && tag === "sheet") {
budget.worksheets += 1;
if (budget.worksheets > limits.maxWorksheets)
reject("Spreadsheet has too many worksheets");
const ids = Object.entries(attributes).filter(([key]) => localXmlName(key) === "id");
if (ids.length > 1) reject("Spreadsheet worksheet references are ambiguous");
const id = ids[0]?.[1];
if (id) budget.worksheetReferences.push({ id, source: path });
}
if (
relationshipSource !== undefined &&
tag === "Relationship" &&
attributes.Type?.endsWith("/relationships/worksheet")
) {
const id = attributes.Id;
const target = attributes.Target;
if (!id || !target || attributes.TargetMode?.toLowerCase() === "external") {
reject("Spreadsheet worksheet relationship is missing or external");
}
const resolved = resolveInternalPartTarget(relationshipSource, target);
const relations =
budget.worksheetRelationships.get(relationshipSource) ?? new Map<string, string>();
if (relations.has(id) || relations.size >= limits.maxWorksheets) {
reject("Spreadsheet worksheet relationships are ambiguous or excessive");
}
relations.set(id, resolved);
budget.worksheetRelationships.set(relationshipSource, relations);
}
if (sharedStrings && tag === "si") {
budget.sharedStrings += 1;
if (budget.sharedStrings > limits.maxSharedStrings) {
reject("Spreadsheet shared strings exceed the safe item budget");
}
}
if (!worksheet) return;
if ((tag === "dimension" || tag === "mergeCell") && attributes.ref !== undefined) {
useRange(attributes.ref);
} else if (tag === "row") {
if (inRow) reject("Spreadsheet contains nested rows");
const nextRow = attributes.r === undefined ? row + 1 : positiveXmlInteger(attributes.r);
if (nextRow <= row) reject("Spreadsheet row coordinates must be strictly increasing");
row = nextRow;
inRow = true;
column = 0;
useCoordinate(row, 0);
} else if (tag === "c") {
if (!inRow) reject("Spreadsheet contains a cell outside a row");
budget.cells += 1;
if (budget.cells > limits.maxWorkbookCells) reject("Spreadsheet has too many cells");
if (attributes.r === undefined) {
column += 1;
} else {
const coordinate = cellCoordinate(attributes.r);
if (coordinate.row !== row) reject("Spreadsheet cell and row coordinates disagree");
if (coordinate.column <= column)
reject("Spreadsheet cell coordinates must be strictly increasing");
column = coordinate.column;
}
useCoordinate(row, column);
} else if (tag === "col") {
// Column formatting does not make otherwise empty columns part of Pandas' data frame.
// Validate its indices without increasing the occupied rectangular cell span.
for (const value of [attributes.min, attributes.max]) {
if (value !== undefined && positiveXmlInteger(value) > limits.maxSheetColumns) {
reject("Spreadsheet column formatting exceeds the safe column budget");
}
}
}
},
onclosetag(name, implied) {
if (implied && parser.endIndex !== lastOpenEndIndex) {
reject("Office XML contains unbalanced elements");
}
if (worksheet && localXmlName(name) === "row") inRow = false;
depth -= 1;
},
},
{ decodeEntities: true, xmlMode: true },
);
return {
push(chunk, final) {
if (ignoredBinary) return;
memberBytes += chunk.byteLength;
let bytes = chunk;
if (!decoder) {
if (pendingPrefix.byteLength > 0) {
bytes = new Uint8Array(pendingPrefix.byteLength + chunk.byteLength);
bytes.set(pendingPrefix);
bytes.set(chunk, pendingPrefix.byteLength);
}
if (bytes.byteLength < 4 && !final) {
pendingPrefix = bytes.slice();
return;
}
pendingPrefix = new Uint8Array(0);
if (!activeXml && !couldBeXmlPrefix(bytes)) {
ignoredBinary = true;
return;
}
const encoding =
(bytes[0] === 0xff && bytes[1] === 0xfe) || (isAsciiXmlPrefix(bytes[0]) && bytes[1] === 0)
? "utf-16le"
: (bytes[0] === 0xfe && bytes[1] === 0xff) ||
(bytes[0] === 0 && isAsciiXmlPrefix(bytes[1]))
? "utf-16be"
: "utf-8";
decoder = new TextDecoder(encoding, { fatal: true });
}
let text: string;
try {
text = decoder.decode(bytes, { stream: !final });
} catch (error) {
if (activeXml) throw error;
ignoredBinary = true;
return;
}
if (!activeXml) {
text = text.replace(/^[\t\r\n ]+/u, "");
if (!text) return;
if (!text.startsWith("<")) {
ignoredBinary = true;
return;
}
activeXml = true;
}
budget.xmlBytes += memberBytes - countedXmlBytes;
countedXmlBytes = memberBytes;
if (memberBytes > limits.maxXmlEntryBytes || budget.xmlBytes > limits.maxXmlBytes) {
reject("Office XML expansion exceeds the inspection byte limit");
}
if (text.includes("\0"))
reject("Office XML contains an unsupported encoding or NUL character");
parser.write(text);
if (final) {
if (depth !== 0 || roots !== 1)
reject("Office XML is incomplete or has no document element");
parser.end();
if (worksheet) {
budget.worksheetAreas.set(path, maxRow * maxColumn);
budget.workbookCellSpan += maxRow * maxColumn;
if (budget.workbookCellSpan > limits.maxWorkbookCellSpan) {
reject("Spreadsheet workbook exceeds the safe dense-cell budget");
}
}
}
},
};
}
function couldBeXmlPrefix(bytes: Uint8Array): boolean {
if (
(bytes[0] === 0xff && bytes[1] === 0xfe) ||
(bytes[0] === 0xfe && bytes[1] === 0xff) ||
(bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) ||
(bytes[0] === 0 && isAsciiXmlPrefix(bytes[1])) ||
(isAsciiXmlPrefix(bytes[0]) && bytes[1] === 0)
)
return true;
for (const byte of bytes) {
if (byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d) continue;
return byte === 0x3c;
}
return bytes.byteLength > 0;
}
function isAsciiXmlPrefix(byte: number | undefined): boolean {
return byte === 0x3c || byte === 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d;
}
function sourcePartForRelationshipFile(path: string): string | undefined {
const match = /^(.*\/)?_rels\/([^/]+)\.rels$/u.exec(path);
return match ? `${match[1] ?? ""}${match[2]}` : undefined;
}
function resolveInternalPartTarget(source: string, target: string): string {
if (!target || /[\\\0?#]/u.test(target) || /^[a-z][a-z0-9+.-]*:/iu.test(target))
reject("Spreadsheet worksheet target is not a safe internal part");
const segments = target.startsWith("/") ? [] : source.split("/").slice(0, -1);
for (const segment of target.split("/")) {
if (!segment || segment === ".") continue;
if (segment === "..") {
if (segments.length === 0) reject("Spreadsheet worksheet target escapes the archive");
segments.pop();
} else segments.push(segment);
}
const resolved = segments.join("/");
if (!resolved) reject("Spreadsheet worksheet target is empty");
return resolved;
}
function positiveXmlInteger(value: string): number {
if (!/^\d+$/u.test(value)) reject("Spreadsheet contains an invalid positive coordinate");
const parsed = Number(value);
if (!Number.isSafeInteger(parsed) || parsed < 1) {
reject("Spreadsheet contains an invalid positive coordinate");
}
return parsed;
}
function cellCoordinate(value: string): { row: number; column: number } {
const match = /^\$?([A-Z]{1,3})\$?(\d+)$/u.exec(value);
if (!match) reject("Spreadsheet contains an invalid cell coordinate");
let column = 0;
for (const letter of match[1] as string) column = column * 26 + letter.charCodeAt(0) - 64;
return { column, row: positiveXmlInteger(match[2] as string) };
}

16
knowledge-fs/packages/parsers/src/parse-coverage.test.ts Normal file
View File

@ -0,0 +1,16 @@
import { describe, expect, it } from "vitest";
import { createArchiveMediaReportCollector } from "./parse-coverage";
describe("bounded parse coverage reports", () => {
it("bounds diagnostic references and explicitly reports truncation", () => {
const report = createArchiveMediaReportCollector(true);
report.skip("x".repeat(1025), "unsupported-media-format");
for (let index = 0; index < 4096; index += 1)
report.skip(`word/media/${index}.svg`, "unsupported-media-format");
const metadata = report.metadata();
expect(metadata.archiveMediaReport.skippedResources).toHaveLength(4096);
expect(metadata.archiveMediaReport.skippedResources[0]?.archivePath).toHaveLength(1024);
expect(metadata.archiveMediaReport.omittedReferences).toBe(1);
expect(metadata.parseCoverage.media.reasons).toContain("resource-reference-truncated");
});
});

38
knowledge-fs/packages/parsers/src/parse-coverage.ts Normal file
View File

@ -0,0 +1,38 @@
export interface ParseCoveragePart {
readonly status: "complete" | "partial" | "unknown" | "not-requested";
readonly reasons: readonly string[];
}
/** Diagnostic references are bounded independently from decoded images and never inline bytes. */
export function createArchiveMediaReportCollector(requiresImages: boolean | undefined) {
const skippedResources: { archivePath: string; reason: string }[] = [];
const reasons = new Set<string>();
let omittedReferences = 0;
let observedResources = 0;
const maxDiagnosticReferences = 4096;
return {
observe() {
observedResources += 1;
},
skip(archivePath: string, reason: string) {
reasons.add(reason);
if (skippedResources.length < maxDiagnosticReferences) {
skippedResources.push({ archivePath: archivePath.slice(0, 1024), reason });
if (archivePath.length > 1024) reasons.add("resource-reference-truncated");
} else omittedReferences += 1;
},
metadata() {
const unknown = { reasons: ["provider-coverage-not-reported"], status: "unknown" } as const;
const media: ParseCoveragePart =
requiresImages === false
? { reasons: ["image-extraction-disabled"], status: "not-requested" }
: reasons.size > 0
? { reasons: [...reasons].sort(), status: "partial" }
: unknown;
return {
archiveMediaReport: { observedResources, omittedReferences, skippedResources },
parseCoverage: { media, tables: unknown, text: unknown },
};
},
};
}

Some files were not shown because too many files have changed in this diff Show More