From 77a5de2fc9cc126bfcd8d260207e504f6e027708 Mon Sep 17 00:00:00 2001 From: Jyong Date: Wed, 26 Aug 2026 03:57:35 -0400 Subject: [PATCH] fix(knowledge-fs): persist document images across formats --- .../inner_api/knowledge_fs/storage.py | 51 ++++ api/knowledge-fs-contract.lock.json | 2 +- api/services/knowledge_fs/remote_images.py | 187 ++++++++++++++ .../inner_api/knowledge_fs/test_storage.py | 56 ++++ .../test_knowledge_fs_remote_images.py | 170 ++++++++++++ .../core-services/knowledge-fs.env.example | 1 + ...-26-office-remote-image-materialization.md | 35 +++ .../document-compilation-runtime-options.ts | 6 + knowledge-fs/apps/api/src/index.ts | 11 +- .../kubernetes/dify-integration-baseline.yaml | 1 + knowledge-fs/infra/local/.env.example | 1 + knowledge-fs/infra/local/compose.yaml | 1 + .../adapters/src/dify-remote-image.test.ts | 112 ++++++++ .../adapters/src/dify-remote-image.ts | 208 +++++++++++++++ knowledge-fs/packages/adapters/src/index.ts | 1 + knowledge-fs/packages/adapters/src/node.ts | 11 + .../src/document-compilation-pipeline.test.ts | 109 ++++++++ .../api/src/document-compilation-pipeline.ts | 12 +- .../api/src/document-compilation-worker.ts | 11 +- ...ocument-multimodal-asset-extractor.test.ts | 244 ++++++++++++++++++ .../document-multimodal-asset-extractor.ts | 81 +++++- .../api/src/document-write-handlers.ts | 4 + .../packages/api/src/gateway-options.ts | 2 + knowledge-fs/packages/api/src/index.ts | 4 + .../packages/parsers/src/parser.test.ts | 50 ++++ knowledge-fs/scripts/compose-apps.test.mjs | 1 + 26 files changed, 1364 insertions(+), 8 deletions(-) create mode 100644 api/services/knowledge_fs/remote_images.py create mode 100644 api/tests/unit_tests/services/test_knowledge_fs_remote_images.py create mode 100644 knowledge-fs/.harness/changes/2026-08-26-office-remote-image-materialization.md create mode 100644 knowledge-fs/packages/adapters/src/dify-remote-image.test.ts create mode 100644 knowledge-fs/packages/adapters/src/dify-remote-image.ts diff --git a/api/controllers/inner_api/knowledge_fs/storage.py b/api/controllers/inner_api/knowledge_fs/storage.py index 1dbde70a132..a8c048e1afc 100644 --- a/api/controllers/inner_api/knowledge_fs/storage.py +++ b/api/controllers/inner_api/knowledge_fs/storage.py @@ -30,6 +30,10 @@ from services.knowledge_fs.object_storage import ( KnowledgeFSObjectStorageUnavailableError, ) from services.knowledge_fs.query_images import KnowledgeFSQueryImageError, load_query_image +from services.knowledge_fs.remote_images import ( + KnowledgeFSRemoteImageError, + load_remote_image, +) _METADATA_HEADER = "X-Knowledge-FS-Metadata" _CHECKSUM_HEADER = "X-Knowledge-FS-Checksum-Sha256" @@ -95,6 +99,12 @@ class KnowledgeFSQueryImageQuery(BaseModel): subject_id: str = Field(min_length=1) +class KnowledgeFSRemoteImageQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + url: str = Field(min_length=1, max_length=8_192) + + register_response_schema_models( inner_api_ns, KnowledgeFSObjectMetadataResponse, @@ -142,6 +152,28 @@ class KnowledgeFSQueryImageApi(Resource): return response +@inner_api_ns.route("/knowledge-fs/remote-image") +class KnowledgeFSRemoteImageApi(Resource): + """Resolve one bounded document image through Dify's SSRF-protected network client.""" + + @knowledge_fs_inner_api_only + @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSRemoteImageQuery)) + @inner_api_ns.produces(["image/gif", "image/jpeg", "image/png", "image/webp"]) + def get(self) -> Response: + try: + query = KnowledgeFSRemoteImageQuery.model_validate(request.args.to_dict(flat=True)) + image = load_remote_image(query.url) + except ValidationError as exc: + raise _invalid_request_error() from exc + except KnowledgeFSRemoteImageError as exc: + raise _remote_image_http_error(exc) from exc + + response = Response(image.body, content_type=image.mime_type) + response.content_length = image.byte_size + response.headers["X-Knowledge-FS-Remote-Image-Sha256"] = image.sha256 + return response + + @inner_api_ns.route("/knowledge-fs/storage/object") class KnowledgeFSObjectApi(Resource): """Read, write, or delete one logical KnowledgeFS object.""" @@ -357,3 +389,22 @@ def _not_found_error() -> KnowledgeFSObjectStorageHttpError: description="KnowledgeFS object was not found.", status_code=HTTPStatus.NOT_FOUND, ) + + +def _remote_image_http_error(error: KnowledgeFSRemoteImageError) -> KnowledgeFSObjectStorageHttpError: + status_by_code = { + "REMOTE_IMAGE_BLOCKED": HTTPStatus.FORBIDDEN, + "REMOTE_IMAGE_CONTENT_UNSUPPORTED": HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + "REMOTE_IMAGE_EMPTY": HTTPStatus.UNPROCESSABLE_ENTITY, + "REMOTE_IMAGE_NOT_FOUND": HTTPStatus.NOT_FOUND, + "REMOTE_IMAGE_RATE_LIMITED": HTTPStatus.TOO_MANY_REQUESTS, + "REMOTE_IMAGE_REQUEST_REJECTED": HTTPStatus.UNPROCESSABLE_ENTITY, + "REMOTE_IMAGE_TOO_LARGE": HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE": HTTPStatus.BAD_GATEWAY, + "REMOTE_IMAGE_URL_INVALID": HTTPStatus.BAD_REQUEST, + } + return KnowledgeFSObjectStorageHttpError( + error_code=error.code.lower(), + description=str(error), + status_code=status_by_code.get(error.code, HTTPStatus.BAD_GATEWAY), + ) diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 084dc95a6c8..90359644465 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,6 +1,6 @@ { "schemaVersion": 5, - "subtreeTree": "c6d1bb478d08368b88c87b9178dd9db46add8719", + "subtreeTree": "2badcef0264f09fd0e0d33577866b8cbb4278a8b", "openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796", "capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", diff --git a/api/services/knowledge_fs/remote_images.py b/api/services/knowledge_fs/remote_images.py new file mode 100644 index 00000000000..ca73fa058f9 --- /dev/null +++ b/api/services/knowledge_fs/remote_images.py @@ -0,0 +1,187 @@ +"""SSRF-safe, bounded retrieval for KnowledgeFS document image references.""" + +from __future__ import annotations + +import hashlib +import urllib.parse +from dataclasses import dataclass + +import httpx + +from core.file import remote_fetcher +from core.helper import ssrf_proxy +from core.tools.errors import ToolSSRFError + +KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024 +KNOWLEDGE_FS_REMOTE_IMAGE_MAX_URL_CHARS = 8_192 +_REMOTE_IMAGE_ACCEPT = "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1" + + +class KnowledgeFSRemoteImageError(ValueError): + """Safe, classified failure returned across the trusted KnowledgeFS bridge.""" + + def __init__(self, code: str, message: str, *, retryable: bool) -> None: + super().__init__(message) + self.code = code + self.retryable = retryable + + +@dataclass(frozen=True) +class KnowledgeFSResolvedRemoteImage: + byte_size: int + mime_type: str + body: bytes + sha256: str + + +def load_remote_image(url: str) -> KnowledgeFSResolvedRemoteImage: + """Resolve one user-authored image URL through Dify's SSRF-protected file client.""" + + normalized_url = _validate_remote_image_url(url) + try: + response = remote_fetcher.make_request( + "GET", + normalized_url, + follow_redirects=True, + headers={ + "Accept": _REMOTE_IMAGE_ACCEPT, + "Accept-Encoding": "identity", + }, + max_retries=1, + stream_response=True, + ) + except ToolSSRFError as exc: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_BLOCKED", + "Remote image access was blocked by network safety policy.", + retryable=False, + ) from exc + except (httpx.TimeoutException, httpx.RequestError, ssrf_proxy.MaxRetriesExceededError) as exc: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE", + "Remote image service is temporarily unavailable.", + retryable=True, + ) from exc + + try: + _assert_remote_status(response.status_code) + declared_size = _content_length(response) + if declared_size is not None and declared_size > KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_TOO_LARGE", + "Remote image exceeds the configured size limit.", + retryable=False, + ) + try: + buffered = ssrf_proxy.buffer_response( + response, + max_response_bytes=KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES, + ) + except ssrf_proxy.ResponseTooLargeError as exc: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_TOO_LARGE", + "Remote image exceeds the configured size limit.", + retryable=False, + ) from exc + except ssrf_proxy.UnsupportedResponseEncodingError as exc: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_CONTENT_UNSUPPORTED", + "Remote image response encoding is not supported.", + retryable=False, + ) from exc + except Exception: + response.close() + raise + + body = buffered.content + if not body: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_EMPTY", + "Remote image is empty.", + retryable=False, + ) + mime_type = _detect_image_mime_type(body) + return KnowledgeFSResolvedRemoteImage( + byte_size=len(body), + mime_type=mime_type, + body=body, + sha256=hashlib.sha256(body).hexdigest(), + ) + + +def _validate_remote_image_url(url: str) -> str: + normalized = url.strip() + if not normalized or len(normalized) > KNOWLEDGE_FS_REMOTE_IMAGE_MAX_URL_CHARS: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_URL_INVALID", + "Remote image URL is invalid.", + retryable=False, + ) + parsed = urllib.parse.urlsplit(normalized) + if ( + parsed.scheme.lower() not in {"http", "https"} + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + ): + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_URL_INVALID", + "Remote image URL is invalid.", + retryable=False, + ) + return normalized + + +def _assert_remote_status(status_code: int) -> None: + if 200 <= status_code < 300: + return + if status_code in {404, 410}: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_NOT_FOUND", + "Remote image was not found.", + retryable=False, + ) + if status_code == 429: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_RATE_LIMITED", + "Remote image service rate limited the request.", + retryable=True, + ) + if status_code in {408, 425} or status_code >= 500: + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE", + "Remote image service is temporarily unavailable.", + retryable=True, + ) + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_REQUEST_REJECTED", + "Remote image request was rejected.", + retryable=False, + ) + + +def _content_length(response: httpx.Response) -> int | None: + value = response.headers.get("content-length") + if value is None: + return None + try: + parsed = int(value) + except ValueError: + return None + return parsed if parsed >= 0 else None + + +def _detect_image_mime_type(body: bytes) -> str: + if body.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if body.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if body.startswith((b"GIF87a", b"GIF89a")): + return "image/gif" + if len(body) >= 12 and body.startswith(b"RIFF") and body[8:12] == b"WEBP": + return "image/webp" + raise KnowledgeFSRemoteImageError( + "REMOTE_IMAGE_CONTENT_UNSUPPORTED", + "Remote image content is not supported.", + retryable=False, + ) diff --git a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py index b5befa950d2..abc5aeaaa25 100644 --- a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py +++ b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py @@ -15,6 +15,7 @@ from controllers.inner_api.knowledge_fs.storage import ( KnowledgeFSObjectMetadataApi, KnowledgeFSObjectStorageHttpError, KnowledgeFSQueryImageApi, + KnowledgeFSRemoteImageApi, ) from services.knowledge_fs.object_storage import ( KnowledgeFSObjectList, @@ -23,6 +24,7 @@ from services.knowledge_fs.object_storage import ( KnowledgeFSObjectStorageUnavailableError, ) from services.knowledge_fs.query_images import KnowledgeFSResolvedQueryImage +from services.knowledge_fs.remote_images import KnowledgeFSRemoteImageError, KnowledgeFSResolvedRemoteImage @pytest.fixture @@ -40,6 +42,60 @@ def _metadata_header(metadata: dict[str, str]) -> str: return urlsafe_b64encode(json.dumps(metadata).encode()).decode().rstrip("=") +@patch("controllers.inner_api.knowledge_fs.storage.load_remote_image") +def test_remote_image_endpoint_returns_only_sniffed_bounded_image_bytes( + load_remote_image: MagicMock, + app: Flask, +) -> None: + body = b"\x89PNG\r\n\x1a\nremote" + load_remote_image.return_value = KnowledgeFSResolvedRemoteImage( + byte_size=len(body), + mime_type="image/png", + body=body, + sha256="a" * 64, + ) + handler = KnowledgeFSRemoteImageApi() + + with app.test_request_context("/?url=https%3A%2F%2Fcdn.example.test%2Fimage.png"): + result = inspect.unwrap(handler.get)(handler) + + assert result.get_data() == body + assert result.content_type == "image/png" + assert result.headers["X-Knowledge-FS-Remote-Image-Sha256"] == "a" * 64 + load_remote_image.assert_called_once_with("https://cdn.example.test/image.png") + + +@pytest.mark.parametrize( + ("error", "expected_status"), + [ + (KnowledgeFSRemoteImageError("REMOTE_IMAGE_BLOCKED", "blocked", retryable=False), 403), + (KnowledgeFSRemoteImageError("REMOTE_IMAGE_NOT_FOUND", "missing", retryable=False), 404), + (KnowledgeFSRemoteImageError("REMOTE_IMAGE_TOO_LARGE", "large", retryable=False), 413), + ( + KnowledgeFSRemoteImageError("REMOTE_IMAGE_CONTENT_UNSUPPORTED", "unsupported", retryable=False), + 415, + ), + (KnowledgeFSRemoteImageError("REMOTE_IMAGE_RATE_LIMITED", "limited", retryable=True), 429), + (KnowledgeFSRemoteImageError("REMOTE_IMAGE_UPSTREAM_UNAVAILABLE", "offline", retryable=True), 502), + ], +) +@patch("controllers.inner_api.knowledge_fs.storage.load_remote_image") +def test_remote_image_endpoint_maps_safe_failure_statuses( + load_remote_image: MagicMock, + error: KnowledgeFSRemoteImageError, + expected_status: int, + app: Flask, +) -> None: + load_remote_image.side_effect = error + handler = KnowledgeFSRemoteImageApi() + + with app.test_request_context("/?url=https%3A%2F%2Fcdn.example.test%2Fimage.png"): + with pytest.raises(KnowledgeFSObjectStorageHttpError) as exc_info: + inspect.unwrap(handler.get)(handler) + + assert exc_info.value.code == expected_status + + @patch("controllers.inner_api.knowledge_fs.storage.load_query_image") def test_query_image_endpoint_returns_only_actor_owned_bounded_bytes( load_query_image: MagicMock, diff --git a/api/tests/unit_tests/services/test_knowledge_fs_remote_images.py b/api/tests/unit_tests/services/test_knowledge_fs_remote_images.py new file mode 100644 index 00000000000..1efcdead599 --- /dev/null +++ b/api/tests/unit_tests/services/test_knowledge_fs_remote_images.py @@ -0,0 +1,170 @@ +"""Unit tests for SSRF-safe KnowledgeFS remote-image loading.""" + +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from core.helper.ssrf_proxy import ResponseTooLargeError, UnsupportedResponseEncodingError +from core.tools.errors import ToolSSRFError +from services.knowledge_fs.remote_images import ( + KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES, + KnowledgeFSRemoteImageError, + load_remote_image, +) + + +def _response(status: int, body: bytes = b"", headers: dict[str, str] | None = None) -> httpx.Response: + return httpx.Response( + status, + content=body, + headers=headers, + request=httpx.Request("GET", "https://cdn.example.test/image"), + ) + + +@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request") +def test_load_remote_image_uses_ssrf_fetcher_and_sniffs_supported_content( + make_request: MagicMock, +) -> None: + body = b"\x89PNG\r\n\x1a\nimage" + make_request.return_value = _response(200, body, {"content-type": "application/octet-stream"}) + + result = load_remote_image("https://cdn.example.test/image") + + assert result.body == body + assert result.mime_type == "image/png" + assert result.sha256 == "3c7474b4239ada3342d87f25ec8849eb8473ee35c5471452482686098b49e81b" + make_request.assert_called_once_with( + "GET", + "https://cdn.example.test/image", + follow_redirects=True, + headers={ + "Accept": "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1", + "Accept-Encoding": "identity", + }, + max_retries=1, + stream_response=True, + ) + + +@pytest.mark.parametrize( + ("body", "mime_type"), + [ + (b"\xff\xd8\xffimage", "image/jpeg"), + (b"GIF89aimage", "image/gif"), + (b"RIFF\x00\x00\x00\x00WEBPimage", "image/webp"), + ], +) +@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request") +def test_load_remote_image_sniffs_other_supported_formats( + make_request: MagicMock, + body: bytes, + mime_type: str, +) -> None: + make_request.return_value = _response(200, body, {"content-length": "invalid"}) + + assert load_remote_image("https://cdn.example.test/image").mime_type == mime_type + + +@pytest.mark.parametrize( + "url", + [ + "", + "https://user:secret@cdn.example.test/private.png", + "file:///etc/passwd", + "data:image/png;base64,AQ==", + "relative.png", + "https://cdn.example.test/" + "x" * 8_192, + ], +) +def test_load_remote_image_rejects_non_http_urls(url: str) -> None: + with pytest.raises(KnowledgeFSRemoteImageError) as exc_info: + load_remote_image(url) + + assert exc_info.value.code == "REMOTE_IMAGE_URL_INVALID" + assert exc_info.value.retryable is False + + +@pytest.mark.parametrize( + ("status", "code", "retryable"), + [ + (404, "REMOTE_IMAGE_NOT_FOUND", False), + (400, "REMOTE_IMAGE_REQUEST_REJECTED", False), + (429, "REMOTE_IMAGE_RATE_LIMITED", True), + (503, "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE", True), + ], +) +@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request") +def test_load_remote_image_classifies_upstream_statuses( + make_request: MagicMock, + status: int, + code: str, + retryable: bool, +) -> None: + make_request.return_value = _response(status) + + with pytest.raises(KnowledgeFSRemoteImageError) as exc_info: + load_remote_image("https://cdn.example.test/image.png") + + assert exc_info.value.code == code + assert exc_info.value.retryable is retryable + + +@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request") +def test_load_remote_image_rejects_oversized_and_unsupported_bodies(make_request: MagicMock) -> None: + make_request.return_value = _response( + 200, + b"x", + {"content-length": str(KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES + 1)}, + ) + with pytest.raises(KnowledgeFSRemoteImageError) as too_large: + load_remote_image("https://cdn.example.test/large.png") + assert too_large.value.code == "REMOTE_IMAGE_TOO_LARGE" + + make_request.return_value = _response(200, b"not-an-image") + with pytest.raises(KnowledgeFSRemoteImageError) as unsupported: + load_remote_image("https://cdn.example.test/not-image") + assert unsupported.value.code == "REMOTE_IMAGE_CONTENT_UNSUPPORTED" + + make_request.return_value = _response(200, b"", {"content-length": "-1"}) + with pytest.raises(KnowledgeFSRemoteImageError) as empty: + load_remote_image("https://cdn.example.test/empty.png") + assert empty.value.code == "REMOTE_IMAGE_EMPTY" + + +@patch("services.knowledge_fs.remote_images.ssrf_proxy.buffer_response") +@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request") +def test_load_remote_image_maps_stream_limit_and_ssrf_rejections( + make_request: MagicMock, + buffer_response: MagicMock, +) -> None: + make_request.return_value = _response(200, b"pending") + buffer_response.side_effect = ResponseTooLargeError("large") + with pytest.raises(KnowledgeFSRemoteImageError) as too_large: + load_remote_image("https://cdn.example.test/large.png") + assert too_large.value.code == "REMOTE_IMAGE_TOO_LARGE" + + make_request.side_effect = ToolSSRFError("blocked") + with pytest.raises(KnowledgeFSRemoteImageError) as blocked: + load_remote_image("http://127.0.0.1/private.png") + assert blocked.value.code == "REMOTE_IMAGE_BLOCKED" + assert blocked.value.retryable is False + + make_request.return_value = _response(200, b"pending") + make_request.side_effect = None + buffer_response.side_effect = UnsupportedResponseEncodingError("gzip") + with pytest.raises(KnowledgeFSRemoteImageError) as unsupported_encoding: + load_remote_image("https://cdn.example.test/compressed.png") + assert unsupported_encoding.value.code == "REMOTE_IMAGE_CONTENT_UNSUPPORTED" + + +@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request") +def test_load_remote_image_maps_transport_timeouts(make_request: MagicMock) -> None: + make_request.side_effect = httpx.ReadTimeout("stalled") + + with pytest.raises(KnowledgeFSRemoteImageError) as unavailable: + load_remote_image("https://cdn.example.test/stalled.png") + + assert unavailable.value.code == "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE" + assert unavailable.value.retryable is True diff --git a/docker/envs/core-services/knowledge-fs.env.example b/docker/envs/core-services/knowledge-fs.env.example index 60ec90f2a58..624b09e1fd9 100644 --- a/docker/envs/core-services/knowledge-fs.env.example +++ b/docker/envs/core-services/knowledge-fs.env.example @@ -71,3 +71,4 @@ UNSTRUCTURED_MAX_RESPONSE_BYTES=33554432 # Bound authenticated Dify object-storage calls so cleanup cannot hang a compilation lease forever. DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS=60000 +DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS=30000 diff --git a/knowledge-fs/.harness/changes/2026-08-26-office-remote-image-materialization.md b/knowledge-fs/.harness/changes/2026-08-26-office-remote-image-materialization.md new file mode 100644 index 00000000000..20e4252e74b --- /dev/null +++ b/knowledge-fs/.harness/changes/2026-08-26-office-remote-image-materialization.md @@ -0,0 +1,35 @@ +# Office and remote document image materialization + +## What changed + +- Kept embedded OpenXML/ODF/EPUB images on the existing archive-media path and added a regression for legacy `.doc` image payload extraction. +- Added a Dify-authenticated, SSRF-protected remote-image bridge for parser-produced HTTP(S) image references. +- Materialized remote images from Markdown, HTML, Office, and other parser formats through the same bounded KnowledgeFS object-storage and thumbnail path as embedded images. +- Included the final image-byte digest in synchronous compilation lineage so a changed visual artifact cannot reuse a stale multimodal publication identity. +- Added deployment configuration for the bounded inner remote-image request timeout. + +## Why + +- Some parsers return linked document images as HTTP(S) references instead of embedded bytes. Those references previously remained in manifests without an `objectKey`, so the document page could not render them reliably. +- The behavior must be format-independent: DOCX/DOC/PPTX/XLSX images need the same stable storage contract as Markdown and HTML images. + +## Safety boundaries + +- Only PNG, JPEG, GIF, and WebP payloads up to 10 MiB are accepted after content sniffing. +- External requests use Dify's existing signed-file resolver and SSRF-protected network client; credentials in URLs and unsafe network targets are rejected. +- Missing, blocked, oversized, or unsupported linked images remain as source references and do not block embedded Office images or fail the entire document. +- Transient bridge failures remain retryable, while each KnowledgeFS-to-Dify request has a configurable 30-second default timeout. + +## Verification + +- `pnpm --dir knowledge-fs --filter @knowledge/parsers test:coverage` (63 passed; package coverage above 90% in every dimension) +- `pnpm --dir knowledge-fs --filter @knowledge/adapters test:coverage` (117 passed; package coverage above 90% in every dimension) +- `pnpm --dir knowledge-fs --filter @knowledge/api-app test` (262 passed) +- `pnpm --dir knowledge-fs --filter @knowledge/api test:coverage` (all tests passed; branch coverage 89.27%, identical to clean-HEAD baseline and below the repository's pre-existing 90% gate) +- Focused KnowledgeFS Python tests (31 passed; remote-image service line/branch coverage 98.8%/96.2%) +- KnowledgeFS TypeScript typechecks, Biome, Ruff, deployment compose tests, and contract-lock verification + +## Risks and follow-up + +- Linked images are fetched sequentially under the existing per-document extraction cap to keep memory bounded. Large documents with many slow external hosts may take longer than documents whose images are embedded. +- Existing revisions whose manifests have no `objectKey` are not mutated in place; reindexing or creating a new revision is required to materialize their missing linked images. diff --git a/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts b/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts index d6e14cafd3a..3466eacf7f7 100644 --- a/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts +++ b/knowledge-fs/apps/api/src/document-compilation-runtime-options.ts @@ -165,6 +165,7 @@ export interface CreateApiDocumentCompilationRuntimeOptions { | "documentMultimodalMaxExtractedAssets" | "documentMultimodalMaxLocalAssetBytes" | "documentMultimodalMaxPdfRasterizedAssets" + | "documentMultimodalRemoteAssetFetcher" | "documentPdfRasterizer" > & { readonly documentMultimodalMaxConcurrency?: number | undefined }) | undefined; @@ -604,6 +605,11 @@ export function createApiDocumentCompilationRuntime({ multimodalMaxPdfRasterizedAssets: multimodal.documentMultimodalMaxPdfRasterizedAssets, } : {}), + ...(multimodal?.documentMultimodalRemoteAssetFetcher + ? { + multimodalRemoteAssetFetcher: multimodal.documentMultimodalRemoteAssetFetcher, + } + : {}), multimodalManifests: repositories.multimodalManifests, ...(createModelBudget ? { modelBudget: createModelBudget() } : {}), objectStorage: adapter.objectStorage, diff --git a/knowledge-fs/apps/api/src/index.ts b/knowledge-fs/apps/api/src/index.ts index f61ea73a1d5..112c54c228c 100644 --- a/knowledge-fs/apps/api/src/index.ts +++ b/knowledge-fs/apps/api/src/index.ts @@ -1,6 +1,9 @@ import { randomUUID } from "node:crypto"; -import { createNodePlatformAdapter } from "@knowledge/adapters/node"; +import { + createNodePlatformAdapter, + createNodeRemoteDocumentImageFetcher, +} from "@knowledge/adapters/node"; import { type KnowledgeSpaceEmbeddingResolver, createDatabaseDeletionObjectWriteAdmission, @@ -121,6 +124,7 @@ const researchTaskDirectStream = createApiResearchTaskDirectStreamAssembly({ }); const adapter = createNodePlatformAdapter(); +const documentRemoteAssetFetcher = createNodeRemoteDocumentImageFetcher(); const queryImageResolver = createApiQueryImageResolver({ env: process.env }); const queryImageExpansionProvider = createApiQueryImageExpansionProvider(process.env); const operationalMetrics = createApiKnowledgeFsOperationalMetrics({ @@ -165,7 +169,10 @@ const visualEmbeddingOptions = createApiVisualEmbeddingOptions({ modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate, objectStorage: adapter.objectStorage, }); -const multimodalOptions = createApiMultimodalOptions(); +const multimodalOptions = { + ...createApiMultimodalOptions(), + documentMultimodalRemoteAssetFetcher: documentRemoteAssetFetcher, +}; const multimodalAnswerOptions = createApiMultimodalAnswerOptions({ objectStorage: adapter.objectStorage, }); diff --git a/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml b/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml index b623b7eba58..6110b4f21ac 100644 --- a/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml +++ b/knowledge-fs/infra/kubernetes/dify-integration-baseline.yaml @@ -27,6 +27,7 @@ data: DURABLE_DELETION_ENABLED: "off" DIFY_INNER_API_URL: http://api:5001 DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS: "60000" + DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS: "30000" DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES: "8388608" DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS: "60000" DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: "8388608" diff --git a/knowledge-fs/infra/local/.env.example b/knowledge-fs/infra/local/.env.example index fadbb825a1c..b134557a382 100644 --- a/knowledge-fs/infra/local/.env.example +++ b/knowledge-fs/infra/local/.env.example @@ -22,6 +22,7 @@ UNSTRUCTURED_RETRY_DELAY_MS= DIFY_INNER_API_URL=http://host.docker.internal:5001 DIFY_INNER_API_KEY= DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS=60000 +DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS=30000 DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES=8388608 DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS=60000 DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES=8388608 diff --git a/knowledge-fs/infra/local/compose.yaml b/knowledge-fs/infra/local/compose.yaml index 820c83621fb..5018ca76b25 100644 --- a/knowledge-fs/infra/local/compose.yaml +++ b/knowledge-fs/infra/local/compose.yaml @@ -39,6 +39,7 @@ services: DIFY_INNER_API_KEY: ${DIFY_INNER_API_KEY:-} DIFY_INNER_API_URL: ${DIFY_INNER_API_URL:-http://host.docker.internal:5001} DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS: ${DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS:-60000} + DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS: ${DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS:-30000} DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES:-8388608} DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS: ${DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS:-60000} DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES:-8388608} diff --git a/knowledge-fs/packages/adapters/src/dify-remote-image.test.ts b/knowledge-fs/packages/adapters/src/dify-remote-image.test.ts new file mode 100644 index 00000000000..cd8e26bff7c --- /dev/null +++ b/knowledge-fs/packages/adapters/src/dify-remote-image.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it, vi } from "vitest"; + +import { DifyRemoteImageRequestError, createDifyRemoteImageFetcher } from "./dify-remote-image"; + +const pngBytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]); + +describe("Dify remote image fetcher", () => { + it("uses the authenticated SSRF-safe inner endpoint and returns bounded bytes", async () => { + const fetch = vi.fn().mockResolvedValue( + new Response(pngBytes, { + headers: { + "Content-Length": String(pngBytes.byteLength), + "Content-Type": "image/png", + }, + }), + ); + const resolver = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch, + }); + + await expect( + resolver.fetch({ maxBytes: 1024, url: "https://cdn.example.test/a b.png?x=1&y=2" }), + ).resolves.toEqual({ body: pngBytes, contentType: "image/png" }); + + const requestUrl = fetch.mock.calls[0]?.[0].toString() ?? ""; + expect(requestUrl).toContain("/inner/api/knowledge-fs/remote-image?"); + expect(new URL(requestUrl).searchParams.get("url")).toBe( + "https://cdn.example.test/a%20b.png?x=1&y=2", + ); + expect(new Headers(fetch.mock.calls[0]?.[1]?.headers).get("X-Inner-Api-Key")).toBe("inner-key"); + }); + + it.each([400, 403, 404, 413, 415, 422])( + "leaves terminally unavailable remote images inline for status %s", + async (status) => { + const resolver = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: vi.fn().mockResolvedValue(new Response(null, { status })), + }); + + await expect( + resolver.fetch({ maxBytes: 1024, url: "https://cdn.example.test/missing.png" }), + ).resolves.toBeNull(); + }, + ); + + it("classifies transient responses, transport errors, and timeouts as retryable", async () => { + const unavailable = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: vi + .fn() + .mockResolvedValue(new Response(null, { status: 503 })), + }); + await expect( + unavailable.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }), + ).rejects.toMatchObject({ retryable: true, status: 503 }); + + const offline = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: vi.fn().mockRejectedValue(new Error("offline")), + }); + await expect( + offline.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }), + ).rejects.toBeInstanceOf(DifyRemoteImageRequestError); + + const stalled = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: async (input, init) => { + const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined); + return await new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + }, + requestTimeoutMs: 10, + }); + await expect( + stalled.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }), + ).rejects.toMatchObject({ retryable: true }); + }); + + it("rejects oversized or invalid successful responses", async () => { + const oversized = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: vi.fn().mockResolvedValue( + new Response(new Uint8Array([1, 2, 3]), { + headers: { "Content-Length": "3", "Content-Type": "image/png" }, + }), + ), + }); + await expect( + oversized.fetch({ maxBytes: 2, url: "https://cdn.example.test/image.png" }), + ).rejects.toThrow("exceeds maxBytes=2"); + + const invalidType = createDifyRemoteImageFetcher({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: vi + .fn() + .mockResolvedValue(new Response(pngBytes, { headers: { "Content-Type": "text/plain" } })), + }); + await expect( + invalidType.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }), + ).rejects.toThrow("content type is invalid"); + }); +}); diff --git a/knowledge-fs/packages/adapters/src/dify-remote-image.ts b/knowledge-fs/packages/adapters/src/dify-remote-image.ts new file mode 100644 index 00000000000..79ab82221f7 --- /dev/null +++ b/knowledge-fs/packages/adapters/src/dify-remote-image.ts @@ -0,0 +1,208 @@ +export interface RemoteDocumentImageFetchInput { + readonly maxBytes: number; + readonly signal?: AbortSignal | undefined; + readonly url: string; +} + +export interface ResolvedRemoteDocumentImage { + readonly body: Uint8Array; + readonly contentType: string; +} + +export interface RemoteDocumentImageFetcher { + fetch(input: RemoteDocumentImageFetchInput): Promise; +} + +export interface DifyRemoteImageFetcherOptions { + readonly apiKey: string; + readonly baseUrl: string; + readonly fetch?: typeof globalThis.fetch; + readonly requestTimeoutMs?: number; +} + +const defaultRequestTimeoutMs = 30_000; +const terminalUnavailableStatuses = new Set([400, 403, 404, 413, 415, 422]); +const supportedContentTypes = new Set(["image/gif", "image/jpeg", "image/png", "image/webp"]); + +export class DifyRemoteImageRequestError extends Error { + readonly code = "dify_remote_image_request_failed"; + readonly retryable: boolean; + readonly status?: number; + + constructor( + message: string, + options: { + readonly cause?: unknown; + readonly retryable: boolean; + readonly status?: number; + }, + ) { + super(message, options.cause === undefined ? undefined : { cause: options.cause }); + this.name = "DifyRemoteImageRequestError"; + this.retryable = options.retryable; + if (options.status !== undefined) this.status = options.status; + } +} + +export function createDifyRemoteImageFetcher({ + apiKey, + baseUrl, + fetch = globalThis.fetch, + requestTimeoutMs = defaultRequestTimeoutMs, +}: DifyRemoteImageFetcherOptions): RemoteDocumentImageFetcher { + const normalizedBaseUrl = requiredBaseUrl(baseUrl); + const normalizedApiKey = requiredString(apiKey, "Dify inner API key"); + positiveSafeInteger(requestTimeoutMs, "requestTimeoutMs"); + + return { + async fetch({ maxBytes, signal, url }) { + positiveSafeInteger(maxBytes, "maxBytes"); + const remoteUrl = validRemoteUrl(url); + const timeout = AbortSignal.timeout(requestTimeoutMs); + const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout; + let response: Response; + try { + response = await fetch( + new URL( + `/inner/api/knowledge-fs/remote-image?${new URLSearchParams({ url: remoteUrl }).toString()}`, + normalizedBaseUrl, + ), + { + headers: { "X-Inner-Api-Key": normalizedApiKey }, + signal: requestSignal, + }, + ); + } catch (error) { + if (signal?.aborted) throw signal.reason; + throw new DifyRemoteImageRequestError("Dify remote image request failed", { + cause: error, + retryable: true, + }); + } + + if (terminalUnavailableStatuses.has(response.status)) { + await response.body?.cancel(); + return null; + } + if (!response.ok) { + await response.body?.cancel(); + throw new DifyRemoteImageRequestError( + `Dify remote image request failed with status ${response.status}`, + { retryable: isRetryableStatus(response.status), status: response.status }, + ); + } + + const contentType = normalizedImageContentType(response.headers.get("content-type")); + if (!contentType) { + await response.body?.cancel(); + throw new DifyRemoteImageRequestError("Dify remote image content type is invalid", { + retryable: false, + }); + } + const body = await readBoundedBody(response, maxBytes, signal); + return { body, contentType }; + }, + }; +} + +async function readBoundedBody( + response: Response, + maxBytes: number, + callerSignal: AbortSignal | undefined, +): Promise { + const declaredLength = Number.parseInt(response.headers.get("content-length") ?? "", 10); + if (Number.isFinite(declaredLength) && declaredLength > maxBytes) { + await response.body?.cancel(); + throw new DifyRemoteImageRequestError(`Dify remote image exceeds maxBytes=${maxBytes}`, { + retryable: false, + }); + } + if (!response.body) return new Uint8Array(); + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + throw new DifyRemoteImageRequestError(`Dify remote image exceeds maxBytes=${maxBytes}`, { + retryable: false, + }); + } + chunks.push(value); + } + } catch (error) { + if (callerSignal?.aborted) throw callerSignal.reason; + if (error instanceof DifyRemoteImageRequestError) throw error; + throw new DifyRemoteImageRequestError("Dify remote image response failed", { + cause: error, + retryable: true, + }); + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + + const body = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; +} + +function normalizedImageContentType(value: string | null): string | null { + const normalized = value?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + return supportedContentTypes.has(normalized) ? normalized : null; +} + +function validRemoteUrl(value: string): string { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error("Remote image URL is invalid"); + } + if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) { + throw new Error("Remote image URL is invalid"); + } + return url.href; +} + +function isRetryableStatus(status: number): boolean { + return status === 408 || status === 425 || status === 429 || status >= 500; +} + +function requiredBaseUrl(value: string): string { + const normalized = requiredString(value, "Dify inner API URL"); + let parsed: URL; + try { + parsed = new URL(normalized); + } catch { + throw new Error("Dify inner API URL is invalid"); + } + if ( + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || + parsed.username || + parsed.password + ) { + throw new Error("Dify inner API URL is invalid"); + } + return parsed.href.endsWith("/") ? parsed.href : `${parsed.href}/`; +} + +function requiredString(value: string, name: string): string { + const normalized = value.trim(); + if (!normalized) throw new Error(`${name} is required`); + return normalized; +} + +function positiveSafeInteger(value: number, name: string): void { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${name} must be a positive safe integer`); + } +} diff --git a/knowledge-fs/packages/adapters/src/index.ts b/knowledge-fs/packages/adapters/src/index.ts index 6ae1aca1943..efe1a7da922 100644 --- a/knowledge-fs/packages/adapters/src/index.ts +++ b/knowledge-fs/packages/adapters/src/index.ts @@ -2,6 +2,7 @@ export * from "./cache"; export * from "./cloudflare-job-queue"; export * from "./database"; export * from "./dify-object-storage"; +export * from "./dify-remote-image"; export * from "./job-queue"; export * from "./memory-object-storage"; export * from "./migration-runner"; diff --git a/knowledge-fs/packages/adapters/src/node.ts b/knowledge-fs/packages/adapters/src/node.ts index 4931087d125..488abc28c12 100644 --- a/knowledge-fs/packages/adapters/src/node.ts +++ b/knowledge-fs/packages/adapters/src/node.ts @@ -3,6 +3,7 @@ import { type PlatformAdapter, collectPlatformHealth } from "@knowledge/core"; import { createMemoryCacheAdapter } from "./cache"; import { createSchemaDatabaseAdapter } from "./database"; import { createDifyObjectStorageAdapter } from "./dify-object-storage"; +import { createDifyRemoteImageFetcher } from "./dify-remote-image"; import { createInlineJobQueueAdapter } from "./job-queue"; import { type PgBossClient, createPgBossJobQueueAdapter } from "./pg-boss-job-queue"; import { @@ -52,6 +53,16 @@ export function createNodePlatformAdapter( return adapter; } +export function createNodeRemoteDocumentImageFetcher(options: NodePlatformAdapterOptions = {}) { + const env = options.env ?? process.env; + return createDifyRemoteImageFetcher({ + apiKey: env.DIFY_INNER_API_KEY?.trim() || defaultDifyInnerApiKey, + baseUrl: env.DIFY_INNER_API_URL?.trim() || defaultDifyInnerApiUrl, + ...(options.difyStorageFetch ? { fetch: options.difyStorageFetch } : {}), + requestTimeoutMs: parsePositiveInteger(env.DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS, 30_000), + }); +} + function createNodeDatabaseAdapter(env: RuntimeEnv, databasePool?: PostgresPoolLike) { const configuredUrl = env.DATABASE_URL?.trim(); diff --git a/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts b/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts index c0f0851926c..e66edc41b60 100644 --- a/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts +++ b/knowledge-fs/packages/api/src/document-compilation-pipeline.test.ts @@ -26,6 +26,115 @@ const firstArtifactId = "30000000-0000-4000-8000-000000000001"; const retryArtifactId = "30000000-0000-4000-8000-000000000002"; describe("compileDocumentArtifact canonical artifact", () => { + it("materializes parser-provided remote image refs before persisting the manifest", async () => { + const adapter = createNodePlatformAdapter({ env: {} }); + const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 }); + const artifactSegments = createInMemoryArtifactSegmentRepository({ + maxBatchSize: 10, + maxListLimit: 10, + maxSegments: 10, + }); + const documentMultimodalManifests = createInMemoryDocumentMultimodalManifestRepository({ + maxManifests: 4, + }); + const outlines = createInMemoryDocumentOutlineRepository({ maxOutlines: 4 }); + const knowledgePaths = createInMemoryKnowledgePathRepository({ + maxListLimit: 20, + maxPaths: 20, + }); + const remoteBody = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]); + const remoteFetches: unknown[] = []; + + const artifact = await compileDocumentArtifact( + { + asset: { ...documentAsset(), filename: "linked-images.docx", mimeType: "application/docx" }, + body: new Uint8Array([1, 2, 3]), + knowledgeSpaceId, + permissionScope: [], + tenantId: "tenant-1", + traceId: randomUUID(), + }, + { + artifacts, + artifactSegments, + documentMultimodalManifests, + documentMultimodalRemoteAssetFetcher: { + fetch: async (input) => { + remoteFetches.push(input); + return { body: remoteBody, contentType: "image/png" }; + }, + }, + documentParser: { + kind: "unstructured", + parse: async (input) => + ParseArtifactSchema.parse({ + artifactHash: "c".repeat(64), + contentType: "mixed", + createdAt: "2026-07-13T00:00:00.000Z", + documentAssetId: input.documentAssetId, + elements: [ + { + id: "linked-image", + metadata: { + assetRef: { uri: "https://cdn.example.test/office-linked.png" }, + }, + sectionPath: [], + type: "image", + }, + ], + id: firstArtifactId, + metadata: {}, + parser: "unstructured", + version: input.version, + }), + }, + generateArtifactSegmentId: randomUUID, + generateKnowledgePathId: randomUUID, + knowledgePaths, + now: () => "2026-07-13T00:00:00.000Z", + objectStorage: adapter.objectStorage, + outlineBuilder: createDocumentOutlineBuilder({ + generateId: randomUUID, + maxElements: 10, + maxNodes: 10, + maxSummaryChars: 1_000, + now: () => "2026-07-13T00:00:00.000Z", + }), + outlines, + synchronousUploadReindexer: null, + traces: createNoopTraceRecorder(), + }, + ); + + expect(remoteFetches).toEqual([ + { maxBytes: 10 * 1024 * 1024, url: "https://cdn.example.test/office-linked.png" }, + ]); + expect(artifact.elements[0]?.metadata).toMatchObject({ + assetRef: { + contentType: "image/png", + objectKey: expect.any(String), + source: "remote-url", + }, + }); + expect(artifact.elements[0]?.metadata).not.toHaveProperty("assetRef.uri"); + expect(artifact.artifactHash).not.toBe("c".repeat(64)); + expect(artifact.metadata).toMatchObject({ + multimodalMaterialization: { + assetCount: 1, + sourceArtifactHash: "c".repeat(64), + }, + }); + await expect( + documentMultimodalManifests.getByDocumentVersion({ documentAssetId, version: 1 }), + ).resolves.toMatchObject({ + items: [ + expect.objectContaining({ + assetRef: expect.objectContaining({ objectKey: expect.any(String) }), + }), + ], + }); + }); + it("uses the first persisted artifact id for every derived write on parser retry", async () => { const adapter = createNodePlatformAdapter({ env: {} }); const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 }); diff --git a/knowledge-fs/packages/api/src/document-compilation-pipeline.ts b/knowledge-fs/packages/api/src/document-compilation-pipeline.ts index 5d434333b88..03326e1fe07 100644 --- a/knowledge-fs/packages/api/src/document-compilation-pipeline.ts +++ b/knowledge-fs/packages/api/src/document-compilation-pipeline.ts @@ -18,7 +18,9 @@ import { buildDocumentOutlineKnowledgePath, buildDocumentSectionKnowledgePaths, } from "./document-knowledge-paths"; +import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact"; import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor"; +import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor"; import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder"; import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository"; import { @@ -70,6 +72,7 @@ export interface CompileDocumentArtifactDeps { readonly documentMultimodalMaxExtractedAssets?: number | undefined; readonly documentMultimodalMaxLocalAssetBytes?: number | undefined; readonly documentMultimodalMaxPdfRasterizedAssets?: number | undefined; + readonly documentMultimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined; readonly documentMultimodalManifests: DocumentMultimodalManifestRepository; readonly documentParser: ParserAdapter; readonly documentPdfRasterizer?: DocumentPdfRasterizer | undefined; @@ -111,6 +114,7 @@ export async function compileDocumentArtifact( documentMultimodalMaxExtractedAssets, documentMultimodalMaxLocalAssetBytes, documentMultimodalMaxPdfRasterizedAssets, + documentMultimodalRemoteAssetFetcher, documentMultimodalManifests, documentParser, documentPdfRasterizer, @@ -190,13 +194,17 @@ export async function compileDocumentArtifact( ? { imageVariantGenerator: documentMultimodalImageVariantGenerator } : {}), objectStorage, + ...(documentMultimodalRemoteAssetFetcher + ? { remoteAssetFetcher: documentMultimodalRemoteAssetFetcher } + : {}), tenantId, }), ); + const materializedArtifact = finalizeDocumentMultimodalArtifact(assetExtractionResult.artifact); const artifactToPersist = ParseArtifactSchema.parse({ - ...assetExtractionResult.artifact, + ...materializedArtifact, metadata: { - ...assetExtractionResult.artifact.metadata, + ...materializedArtifact.metadata, ...(assetExtractionResult.extractedCount > 0 ? { multimodalAssetExtractionCount: assetExtractionResult.extractedCount } : {}), diff --git a/knowledge-fs/packages/api/src/document-compilation-worker.ts b/knowledge-fs/packages/api/src/document-compilation-worker.ts index 4824578ca10..e92a7d1244f 100644 --- a/knowledge-fs/packages/api/src/document-compilation-worker.ts +++ b/knowledge-fs/packages/api/src/document-compilation-worker.ts @@ -49,7 +49,10 @@ import { } from "./document-knowledge-paths"; import type { DocumentModelBudget } from "./document-model-budget"; import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact"; -import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor"; +import { + type DocumentRemoteAssetFetcher, + extractDocumentMultimodalAssets, +} from "./document-multimodal-asset-extractor"; import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder"; import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository"; import type { DocumentOutlineBuilder } from "./document-outline-builder"; @@ -111,6 +114,7 @@ export interface DocumentCompilationWorkerOptions { readonly multimodalMaxConcurrency?: number | undefined; readonly multimodalMaxLocalAssetBytes?: number | undefined; readonly multimodalMaxPdfRasterizedAssets?: number | undefined; + readonly multimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined; readonly multimodalManifests: DocumentMultimodalManifestRepository; readonly modelBudget?: DocumentModelBudget | undefined; readonly objectStorage: PlatformAdapter["objectStorage"]; @@ -247,6 +251,7 @@ export function createDocumentCompilationWorker({ multimodalMaxConcurrency = 2, multimodalMaxLocalAssetBytes, multimodalMaxPdfRasterizedAssets, + multimodalRemoteAssetFetcher, multimodalManifests, modelBudget, objectStorage, @@ -469,6 +474,10 @@ export function createDocumentCompilationWorker({ ? { imageVariantGenerator: multimodalImageVariantGenerator } : {}), objectStorage: multimodalObjectStorage, + ...(multimodalRemoteAssetFetcher + ? { remoteAssetFetcher: multimodalRemoteAssetFetcher } + : {}), + ...(signal ? { signal } : {}), tenantId: input.tenantId, writeOwnerId: multimodalWriteOwnerId, }); diff --git a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts index da61de892d4..0219db21a2c 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.test.ts @@ -11,6 +11,229 @@ const documentAssetId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43"; const parseArtifactId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44"; describe("extractDocumentMultimodalAssets", () => { + it("downloads and stores remote images independently of the source document format", async () => { + const adapter = createNodePlatformAdapter({ env: {} }); + const remoteBody = new Uint8Array([ + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 2, 0, 0, 0, 3, 8, 6, 0, + 0, 0, 0, 0, 0, 0, + ]); + const fetchCalls: unknown[] = []; + + const result = await extractDocumentMultimodalAssets({ + artifact: { + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId, + elements: [ + { + id: "markdown-image", + metadata: { assetRef: { uri: "https://cdn.example.test/markdown.png" } }, + sectionPath: [], + type: "image", + }, + { + id: "office-linked-image", + metadata: { assetRef: { uri: "https://cdn.example.test/office.png" } }, + sectionPath: [], + type: "image", + }, + ], + id: parseArtifactId, + metadata: {}, + parser: "native-markdown", + version: 1, + }, + knowledgeSpaceId, + maxRemoteAssetBytes: 1024, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { + fetch: async (input) => { + fetchCalls.push(input); + return { body: remoteBody, contentType: "image/png; charset=binary" }; + }, + }, + tenantId: "tenant-1", + }); + + expect(result.extractedCount).toBe(2); + expect(fetchCalls).toEqual([ + { maxBytes: 1024, url: "https://cdn.example.test/markdown.png" }, + { maxBytes: 1024, url: "https://cdn.example.test/office.png" }, + ]); + for (const element of result.artifact.elements) { + expect(element.metadata).toMatchObject({ + assetRef: { + contentType: "image/png", + height: 3, + objectKey: expect.any(String), + source: "remote-url", + width: 2, + }, + }); + expect(element.metadata).not.toHaveProperty("assetRef.uri"); + } + expect(result.artifact.metadata).toMatchObject({ + multimodalAssets: { extractedCount: 2, sources: ["remote-url"] }, + }); + }); + + it("leaves unavailable remote images inline without blocking embedded Office images", async () => { + const adapter = createNodePlatformAdapter({ env: {} }); + const controller = new AbortController(); + const remoteFetches: unknown[] = []; + + const result = await extractDocumentMultimodalAssets({ + artifact: { + artifactHash: "a".repeat(64), + contentType: "mixed", + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId, + elements: [ + { + id: "remote-missing", + metadata: { assetRef: { uri: "https://cdn.example.test/missing.png" } }, + sectionPath: [], + type: "image", + }, + { + id: "docx-embedded", + metadata: { + archivePath: "word/media/image1.png", + assetRef: { uri: "data:image/png;base64,AQIDBA==" }, + }, + sectionPath: [], + type: "image", + }, + ], + id: parseArtifactId, + metadata: {}, + parser: "unstructured", + version: 1, + }, + knowledgeSpaceId, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { + fetch: async (input) => { + remoteFetches.push(input); + return null; + }, + }, + signal: controller.signal, + tenantId: "tenant-1", + }); + + expect(result.extractedCount).toBe(1); + expect(remoteFetches).toEqual([ + { + maxBytes: 10 * 1024 * 1024, + signal: controller.signal, + url: "https://cdn.example.test/missing.png", + }, + ]); + expect(result.artifact.elements[0]?.metadata).toEqual({ + assetRef: { uri: "https://cdn.example.test/missing.png" }, + }); + expect(result.artifact.elements[1]?.metadata).toMatchObject({ + archivePath: "word/media/image1.png", + assetRef: { objectKey: expect.any(String), source: "data-uri" }, + }); + }); + + it("rejects invalid limits and dishonest remote fetcher responses", async () => { + const adapter = createNodePlatformAdapter({ env: {} }); + const artifact = remoteImageArtifact("https://cdn.example.test/image.png"); + + await expect( + extractDocumentMultimodalAssets({ + artifact, + knowledgeSpaceId, + maxRemoteAssetBytes: 0, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { fetch: async () => null }, + tenantId: "tenant-1", + }), + ).rejects.toThrow("Document multimodal remote asset max bytes must be at least 1"); + + await expect( + extractDocumentMultimodalAssets({ + artifact, + knowledgeSpaceId, + maxRemoteAssetBytes: 1, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { + fetch: async () => ({ body: new Uint8Array([1, 2]), contentType: "image/png" }), + }, + tenantId: "tenant-1", + }), + ).rejects.toThrow("Document multimodal remote asset exceeds maxRemoteAssetBytes=1"); + + await expect( + extractDocumentMultimodalAssets({ + artifact, + knowledgeSpaceId, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { + fetch: async () => ({ body: new Uint8Array([1]), contentType: "text/plain" }), + }, + tenantId: "tenant-1", + }), + ).rejects.toThrow("Document multimodal remote asset content type is unsupported"); + + await expect( + extractDocumentMultimodalAssets({ + artifact, + knowledgeSpaceId, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { + fetch: async () => ({ body: new Uint8Array(), contentType: "image/png" }), + }, + tenantId: "tenant-1", + }), + ).rejects.toThrow("Document multimodal remote asset is empty"); + }); + + it("does not fetch remote refs after the extraction cap or URLs containing credentials", async () => { + const adapter = createNodePlatformAdapter({ env: {} }); + const fetchCalls: unknown[] = []; + const artifact = remoteImageArtifact("https://cdn.example.test/second.png"); + artifact.elements.unshift({ + id: "embedded-first", + metadata: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } }, + sectionPath: [], + type: "image", + }); + artifact.elements.push({ + id: "credential-url", + metadata: { assetRef: { uri: "https://user:secret@cdn.example.test/private.png" } }, + sectionPath: [], + type: "image", + }); + + const result = await extractDocumentMultimodalAssets({ + artifact, + knowledgeSpaceId, + maxExtractedAssets: 1, + objectStorage: adapter.objectStorage, + remoteAssetFetcher: { + fetch: async (input) => { + fetchCalls.push(input); + return null; + }, + }, + tenantId: "tenant-1", + }); + + expect(result).toMatchObject({ extractedCount: 1, skippedForCapCount: 1 }); + expect(fetchCalls).toEqual([]); + expect(result.artifact.elements[1]?.metadata).toEqual({ + assetRef: { uri: "https://cdn.example.test/second.png" }, + }); + expect(result.artifact.elements[2]?.metadata).toEqual({ + assetRef: { uri: "https://user:secret@cdn.example.test/private.png" }, + }); + }); + it("stores embedded image data URIs and rewrites image asset refs", async () => { const adapter = createNodePlatformAdapter({ env: {} }); const dataUri = "data:image/png;base64,AQIDBA=="; @@ -413,3 +636,24 @@ describe("extractDocumentMultimodalAssets", () => { ).resolves.toEqual(new Uint8Array([5, 6, 7, 8])); }); }); + +function remoteImageArtifact(uri: string) { + return { + artifactHash: "a".repeat(64), + contentType: "mixed" as const, + createdAt: "2026-06-23T00:00:00.000Z", + documentAssetId, + elements: [ + { + id: "remote-image", + metadata: { assetRef: { uri } }, + sectionPath: [], + type: "image" as const, + }, + ], + id: parseArtifactId, + metadata: {}, + parser: "unstructured" as const, + version: 1, + }; +} diff --git a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts index 63572868468..4a503839fae 100644 --- a/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts +++ b/knowledge-fs/packages/api/src/document-multimodal-asset-extractor.ts @@ -22,12 +22,23 @@ export interface ExtractDocumentMultimodalAssetsInput { readonly maxEmbeddedAssetBytes?: number | undefined; readonly maxExtractedAssets?: number | undefined; readonly maxLocalAssetBytes?: number | undefined; + readonly maxRemoteAssetBytes?: number | undefined; readonly imageVariantGenerator?: DocumentImageVariantGenerator | undefined; readonly objectStorage: PlatformAdapter["objectStorage"]; + readonly remoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined; + readonly signal?: AbortSignal | undefined; readonly tenantId: string; readonly writeOwnerId?: string | undefined; } +export interface DocumentRemoteAssetFetcher { + fetch(input: { + readonly maxBytes: number; + readonly signal?: AbortSignal | undefined; + readonly url: string; + }): Promise<{ readonly body: Uint8Array; readonly contentType: string } | null>; +} + export interface ExtractDocumentMultimodalAssetsResult { readonly artifact: ParseArtifact; readonly extractedCount: number; @@ -39,7 +50,7 @@ interface DataUriImage { readonly body: Uint8Array; readonly contentType: string; readonly dimensions?: ImageDimensions | undefined; - readonly source: "data-uri" | "local-file"; + readonly source: "data-uri" | "local-file" | "remote-url"; } interface ImageDimensions { @@ -51,6 +62,13 @@ const dataUriPattern = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)$/iu; const defaultMaxEmbeddedAssetBytes = 10 * 1024 * 1024; const defaultMaxExtractedAssets = 1_000; const defaultMaxLocalAssetBytes = 50 * 1024 * 1024; +const defaultMaxRemoteAssetBytes = 10 * 1024 * 1024; +const supportedRemoteImageContentTypes = new Set([ + "image/gif", + "image/jpeg", + "image/png", + "image/webp", +]); export async function extractDocumentMultimodalAssets({ allowLocalAssetPaths = [], @@ -59,8 +77,11 @@ export async function extractDocumentMultimodalAssets({ maxEmbeddedAssetBytes = defaultMaxEmbeddedAssetBytes, maxExtractedAssets = defaultMaxExtractedAssets, maxLocalAssetBytes = defaultMaxLocalAssetBytes, + maxRemoteAssetBytes = defaultMaxRemoteAssetBytes, imageVariantGenerator, objectStorage, + remoteAssetFetcher, + signal, tenantId, writeOwnerId, }: ExtractDocumentMultimodalAssetsInput): Promise { @@ -76,6 +97,10 @@ export async function extractDocumentMultimodalAssets({ throw new Error("Document multimodal max extracted assets must be at least 1"); } + if (!Number.isSafeInteger(maxRemoteAssetBytes) || maxRemoteAssetBytes < 1) { + throw new Error("Document multimodal remote asset max bytes must be at least 1"); + } + let extractedCount = 0; let skippedForCapCount = 0; const extractionSources = new Set(); @@ -90,7 +115,7 @@ export async function extractDocumentMultimodalAssets({ const assetRef = isPlainObject(element.metadata.assetRef) ? element.metadata.assetRef : null; const uri = typeof assetRef?.uri === "string" ? assetRef.uri.trim() : ""; - const image = + let image = parseDataUriImage(uri, maxEmbeddedAssetBytes) ?? (await readLocalImageAsset({ allowedRoots: allowedLocalRoots, @@ -99,6 +124,40 @@ export async function extractDocumentMultimodalAssets({ uri, })); + if (!image && remoteAssetFetcher && isRemoteHttpUri(uri)) { + if (extractedCount >= maxExtractedAssets) { + skippedForCapCount += 1; + elements.push(element); + continue; + } + const fetched = await remoteAssetFetcher.fetch({ + maxBytes: maxRemoteAssetBytes, + ...(signal ? { signal } : {}), + url: uri, + }); + if (fetched) { + if (fetched.body.byteLength > maxRemoteAssetBytes) { + throw new Error( + `Document multimodal remote asset exceeds maxRemoteAssetBytes=${maxRemoteAssetBytes}`, + ); + } + const contentType = normalizeRemoteImageContentType(fetched.contentType); + if (!contentType) { + throw new Error("Document multimodal remote asset content type is unsupported"); + } + if (fetched.body.byteLength === 0) { + throw new Error("Document multimodal remote asset is empty"); + } + const dimensions = readImageDimensions(fetched.body, contentType); + image = { + body: fetched.body, + contentType, + ...(dimensions ? { dimensions } : {}), + source: "remote-url", + }; + } + } + if (!assetRef || !image) { elements.push(element); continue; @@ -204,6 +263,24 @@ export async function extractDocumentMultimodalAssets({ }; } +function isRemoteHttpUri(uri: string): boolean { + try { + const parsed = new URL(uri); + return ( + (parsed.protocol === "http:" || parsed.protocol === "https:") && + !parsed.username && + !parsed.password + ); + } catch { + return false; + } +} + +function normalizeRemoteImageContentType(value: string): string | null { + const normalized = value.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + return supportedRemoteImageContentTypes.has(normalized) ? normalized : null; +} + async function storeGeneratedImageVariants({ assetId, elementId, diff --git a/knowledge-fs/packages/api/src/document-write-handlers.ts b/knowledge-fs/packages/api/src/document-write-handlers.ts index ee01157e261..480865b1b16 100644 --- a/knowledge-fs/packages/api/src/document-write-handlers.ts +++ b/knowledge-fs/packages/api/src/document-write-handlers.ts @@ -32,6 +32,7 @@ import type { DocumentCompilationJobStateMachine } from "./document-compilation- import { compileDocumentArtifact } from "./document-compilation-pipeline"; import type { DocumentImageVariantGenerator } from "./document-image-variant-generator"; import { buildDocumentKnowledgePath } from "./document-knowledge-paths"; +import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor"; import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository"; import type { DocumentOutlineBuilder } from "./document-outline-builder"; import type { DocumentOutlineRepository } from "./document-outline-repository"; @@ -149,6 +150,7 @@ export interface RegisterDocumentWriteHandlersOptions { readonly documentMultimodalMaxExtractedAssets?: number | undefined; readonly documentMultimodalMaxLocalAssetBytes?: number | undefined; readonly documentMultimodalMaxPdfRasterizedAssets?: number | undefined; + readonly documentMultimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined; readonly documentMultimodalManifests: DocumentMultimodalManifestRepository; readonly documentParser: ParserAdapter; readonly documentPdfRasterizer?: DocumentPdfRasterizer | undefined; @@ -220,6 +222,7 @@ export function registerDocumentWriteHandlers({ documentMultimodalMaxExtractedAssets, documentMultimodalMaxLocalAssetBytes, documentMultimodalMaxPdfRasterizedAssets, + documentMultimodalRemoteAssetFetcher, documentMultimodalManifests, documentParser, documentPdfRasterizer, @@ -1437,6 +1440,7 @@ export function registerDocumentWriteHandlers({ documentMultimodalMaxExtractedAssets, documentMultimodalMaxLocalAssetBytes, documentMultimodalMaxPdfRasterizedAssets, + documentMultimodalRemoteAssetFetcher, documentMultimodalManifests, documentParser, documentPdfRasterizer, diff --git a/knowledge-fs/packages/api/src/gateway-options.ts b/knowledge-fs/packages/api/src/gateway-options.ts index 79a3f585009..a4fc853baf4 100644 --- a/knowledge-fs/packages/api/src/gateway-options.ts +++ b/knowledge-fs/packages/api/src/gateway-options.ts @@ -28,6 +28,7 @@ import type { } from "./document-chunk-repository"; import type { DocumentCompilationJobStateMachine } from "./document-compilation-job"; import type { DocumentImageVariantGenerator } from "./document-image-variant-generator"; +import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor"; import type { DocumentMultimodalManifestEnhancer } from "./document-multimodal-manifest-enhancer"; import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository"; import type { DocumentOutlineRepository } from "./document-outline-repository"; @@ -211,6 +212,7 @@ export interface KnowledgeGatewayOptions { documentMultimodalLocalAssetAllowlist?: readonly string[]; documentMultimodalMaxLocalAssetBytes?: number; documentMultimodalMaxPdfRasterizedAssets?: number; + documentMultimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher; documentPdfRasterizer?: DocumentPdfRasterizer; documentOutlineSummaryEnhancer?: DocumentOutlineSummaryEnhancer; documentOutlines?: DocumentOutlineRepository; diff --git a/knowledge-fs/packages/api/src/index.ts b/knowledge-fs/packages/api/src/index.ts index e1e10bd6403..e282216395b 100644 --- a/knowledge-fs/packages/api/src/index.ts +++ b/knowledge-fs/packages/api/src/index.ts @@ -105,6 +105,7 @@ export * from "./document-asset-repository"; export * from "./document-asset-embedding-profile-guard"; export * from "./durable-deletion-repository"; export * from "./document-image-variant-generator"; +export * from "./document-multimodal-asset-extractor"; export * from "./document-multimodal-enrichment-providers"; export * from "./document-multimodal-candidate-resolver"; export * from "./document-multimodal-evaluation"; @@ -735,6 +736,7 @@ export function createKnowledgeGateway({ documentMultimodalMaxExtractedAssets, documentMultimodalMaxLocalAssetBytes, documentMultimodalMaxPdfRasterizedAssets, + documentMultimodalRemoteAssetFetcher, documentPdfRasterizer, documentOutlineSummaryEnhancer, documentOutlines, @@ -2012,6 +2014,7 @@ export function createKnowledgeGateway({ ...(documentMultimodalMaxExtractedAssets ? { documentMultimodalMaxExtractedAssets } : {}), ...(documentMultimodalImageVariantGenerator ? { documentMultimodalImageVariantGenerator } : {}), ...(documentMultimodalMaxLocalAssetBytes ? { documentMultimodalMaxLocalAssetBytes } : {}), + ...(documentMultimodalRemoteAssetFetcher ? { documentMultimodalRemoteAssetFetcher } : {}), ...(documentMultimodalMaxPdfRasterizedAssets ? { documentMultimodalMaxPdfRasterizedAssets } : {}), @@ -2071,6 +2074,7 @@ export function createKnowledgeGateway({ documentMultimodalMaxExtractedAssets, documentMultimodalMaxLocalAssetBytes, documentMultimodalMaxPdfRasterizedAssets, + documentMultimodalRemoteAssetFetcher, documentMultimodalManifests: multimodalManifestRepository, documentParser, documentPdfRasterizer, diff --git a/knowledge-fs/packages/parsers/src/parser.test.ts b/knowledge-fs/packages/parsers/src/parser.test.ts index 47cc1ac28ad..65e3b815502 100644 --- a/knowledge-fs/packages/parsers/src/parser.test.ts +++ b/knowledge-fs/packages/parsers/src/parser.test.ts @@ -1405,6 +1405,56 @@ describe("parser adapters", () => { }); }); + it("requests and preserves provider image payloads for legacy DOC files", async () => { + const parser = createUnstructuredParserClient({ + endpoint: "https://unstructured.example.test", + fetch: async (request) => { + const form = await (request instanceof Request ? request : new Request(request)).formData(); + + expect(form.get("strategy")).toBe("hi_res"); + expect(form.getAll("extract_image_block_types")).toEqual(["Image"]); + expect(form.get("extract_image_block_to_payload")).toBe("true"); + + return new Response( + JSON.stringify([ + { + metadata: { + image_base64: "AQIDBA==", + image_mime_type: "image/png", + page_number: 1, + }, + type: "Image", + }, + ]), + { headers: { "content-type": "application/json" }, status: 200 }, + ); + }, + generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c54", + now: () => createdAt, + }); + + const artifact = await parser.parse({ + body: new Uint8Array([0xd0, 0xcf, 0x11, 0xe0]), + documentAssetId, + filename: "legacy.doc", + mimeType: "application/msword", + parserHints: { requiresImages: true }, + version: 1, + }); + + expect(artifact.elements).toEqual([ + expect.objectContaining({ + metadata: expect.objectContaining({ + assetRef: { + contentType: "image/png", + uri: "data:image/png;base64,AQIDBA==", + }, + }), + type: "image", + }), + ]); + }); + it("preserves nested Unstructured title paths from parent ids and category depth", async () => { const parser = createUnstructuredParserClient({ endpoint: "https://unstructured.example.test", diff --git a/knowledge-fs/scripts/compose-apps.test.mjs b/knowledge-fs/scripts/compose-apps.test.mjs index 85caa4a89e0..7b3eab67c0f 100644 --- a/knowledge-fs/scripts/compose-apps.test.mjs +++ b/knowledge-fs/scripts/compose-apps.test.mjs @@ -328,6 +328,7 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", ( "UNSTRUCTURED_REQUEST_TIMEOUT_MS", "UNSTRUCTURED_MAX_RESPONSE_BYTES", "DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS", + "DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS", ]); assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME=on$/m); assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_PDF_RASTERIZER=poppler$/m);