mirror of
https://github.com/langgenius/dify.git
synced 2026-09-04 07:53:20 +08:00
fix(knowledge-fs): persist document images across formats
This commit is contained in:
Jyong
parent
207640bcd1
commit
77a5de2fc9
@ -30,6 +30,10 @@ from services.knowledge_fs.object_storage import (
|
||||
KnowledgeFSObjectStorageUnavailableError,
|
||||
)
|
||||
from services.knowledge_fs.query_images import KnowledgeFSQueryImageError, load_query_image
|
||||
from services.knowledge_fs.remote_images import (
|
||||
KnowledgeFSRemoteImageError,
|
||||
load_remote_image,
|
||||
)
|
||||
|
||||
_METADATA_HEADER = "X-Knowledge-FS-Metadata"
|
||||
_CHECKSUM_HEADER = "X-Knowledge-FS-Checksum-Sha256"
|
||||
@ -95,6 +99,12 @@ class KnowledgeFSQueryImageQuery(BaseModel):
|
||||
subject_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class KnowledgeFSRemoteImageQuery(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
url: str = Field(min_length=1, max_length=8_192)
|
||||
|
||||
|
||||
register_response_schema_models(
|
||||
inner_api_ns,
|
||||
KnowledgeFSObjectMetadataResponse,
|
||||
@ -142,6 +152,28 @@ class KnowledgeFSQueryImageApi(Resource):
|
||||
return response
|
||||
|
||||
|
||||
@inner_api_ns.route("/knowledge-fs/remote-image")
|
||||
class KnowledgeFSRemoteImageApi(Resource):
|
||||
"""Resolve one bounded document image through Dify's SSRF-protected network client."""
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.doc(params=query_params_from_model(KnowledgeFSRemoteImageQuery))
|
||||
@inner_api_ns.produces(["image/gif", "image/jpeg", "image/png", "image/webp"])
|
||||
def get(self) -> Response:
|
||||
try:
|
||||
query = KnowledgeFSRemoteImageQuery.model_validate(request.args.to_dict(flat=True))
|
||||
image = load_remote_image(query.url)
|
||||
except ValidationError as exc:
|
||||
raise _invalid_request_error() from exc
|
||||
except KnowledgeFSRemoteImageError as exc:
|
||||
raise _remote_image_http_error(exc) from exc
|
||||
|
||||
response = Response(image.body, content_type=image.mime_type)
|
||||
response.content_length = image.byte_size
|
||||
response.headers["X-Knowledge-FS-Remote-Image-Sha256"] = image.sha256
|
||||
return response
|
||||
|
||||
|
||||
@inner_api_ns.route("/knowledge-fs/storage/object")
|
||||
class KnowledgeFSObjectApi(Resource):
|
||||
"""Read, write, or delete one logical KnowledgeFS object."""
|
||||
@ -357,3 +389,22 @@ def _not_found_error() -> KnowledgeFSObjectStorageHttpError:
|
||||
description="KnowledgeFS object was not found.",
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
)
|
||||
|
||||
|
||||
def _remote_image_http_error(error: KnowledgeFSRemoteImageError) -> KnowledgeFSObjectStorageHttpError:
|
||||
status_by_code = {
|
||||
"REMOTE_IMAGE_BLOCKED": HTTPStatus.FORBIDDEN,
|
||||
"REMOTE_IMAGE_CONTENT_UNSUPPORTED": HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
|
||||
"REMOTE_IMAGE_EMPTY": HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
"REMOTE_IMAGE_NOT_FOUND": HTTPStatus.NOT_FOUND,
|
||||
"REMOTE_IMAGE_RATE_LIMITED": HTTPStatus.TOO_MANY_REQUESTS,
|
||||
"REMOTE_IMAGE_REQUEST_REJECTED": HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
"REMOTE_IMAGE_TOO_LARGE": HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
|
||||
"REMOTE_IMAGE_UPSTREAM_UNAVAILABLE": HTTPStatus.BAD_GATEWAY,
|
||||
"REMOTE_IMAGE_URL_INVALID": HTTPStatus.BAD_REQUEST,
|
||||
}
|
||||
return KnowledgeFSObjectStorageHttpError(
|
||||
error_code=error.code.lower(),
|
||||
description=str(error),
|
||||
status_code=status_by_code.get(error.code, HTTPStatus.BAD_GATEWAY),
|
||||
)
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 5,
|
||||
"subtreeTree": "c6d1bb478d08368b88c87b9178dd9db46add8719",
|
||||
"subtreeTree": "2badcef0264f09fd0e0d33577866b8cbb4278a8b",
|
||||
"openapiSha256": "2cf348c68bbe65dd51bbde9a0a4f91398beeebd79e89e9288c9386b26ae09796",
|
||||
"capabilityV2AuthManifestSha256": "e322a2fa779d1f40b95c54c1021cffecaec77abbd7b34899573dcdf4ff353109",
|
||||
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
|
||||
|
||||
187
api/services/knowledge_fs/remote_images.py
Normal file
187
api/services/knowledge_fs/remote_images.py
Normal file
@ -0,0 +1,187 @@
|
||||
"""SSRF-safe, bounded retrieval for KnowledgeFS document image references."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from core.file import remote_fetcher
|
||||
from core.helper import ssrf_proxy
|
||||
from core.tools.errors import ToolSSRFError
|
||||
|
||||
KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024
|
||||
KNOWLEDGE_FS_REMOTE_IMAGE_MAX_URL_CHARS = 8_192
|
||||
_REMOTE_IMAGE_ACCEPT = "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1"
|
||||
|
||||
|
||||
class KnowledgeFSRemoteImageError(ValueError):
|
||||
"""Safe, classified failure returned across the trusted KnowledgeFS bridge."""
|
||||
|
||||
def __init__(self, code: str, message: str, *, retryable: bool) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KnowledgeFSResolvedRemoteImage:
|
||||
byte_size: int
|
||||
mime_type: str
|
||||
body: bytes
|
||||
sha256: str
|
||||
|
||||
|
||||
def load_remote_image(url: str) -> KnowledgeFSResolvedRemoteImage:
|
||||
"""Resolve one user-authored image URL through Dify's SSRF-protected file client."""
|
||||
|
||||
normalized_url = _validate_remote_image_url(url)
|
||||
try:
|
||||
response = remote_fetcher.make_request(
|
||||
"GET",
|
||||
normalized_url,
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"Accept": _REMOTE_IMAGE_ACCEPT,
|
||||
"Accept-Encoding": "identity",
|
||||
},
|
||||
max_retries=1,
|
||||
stream_response=True,
|
||||
)
|
||||
except ToolSSRFError as exc:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_BLOCKED",
|
||||
"Remote image access was blocked by network safety policy.",
|
||||
retryable=False,
|
||||
) from exc
|
||||
except (httpx.TimeoutException, httpx.RequestError, ssrf_proxy.MaxRetriesExceededError) as exc:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_UPSTREAM_UNAVAILABLE",
|
||||
"Remote image service is temporarily unavailable.",
|
||||
retryable=True,
|
||||
) from exc
|
||||
|
||||
try:
|
||||
_assert_remote_status(response.status_code)
|
||||
declared_size = _content_length(response)
|
||||
if declared_size is not None and declared_size > KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_TOO_LARGE",
|
||||
"Remote image exceeds the configured size limit.",
|
||||
retryable=False,
|
||||
)
|
||||
try:
|
||||
buffered = ssrf_proxy.buffer_response(
|
||||
response,
|
||||
max_response_bytes=KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES,
|
||||
)
|
||||
except ssrf_proxy.ResponseTooLargeError as exc:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_TOO_LARGE",
|
||||
"Remote image exceeds the configured size limit.",
|
||||
retryable=False,
|
||||
) from exc
|
||||
except ssrf_proxy.UnsupportedResponseEncodingError as exc:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_CONTENT_UNSUPPORTED",
|
||||
"Remote image response encoding is not supported.",
|
||||
retryable=False,
|
||||
) from exc
|
||||
except Exception:
|
||||
response.close()
|
||||
raise
|
||||
|
||||
body = buffered.content
|
||||
if not body:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_EMPTY",
|
||||
"Remote image is empty.",
|
||||
retryable=False,
|
||||
)
|
||||
mime_type = _detect_image_mime_type(body)
|
||||
return KnowledgeFSResolvedRemoteImage(
|
||||
byte_size=len(body),
|
||||
mime_type=mime_type,
|
||||
body=body,
|
||||
sha256=hashlib.sha256(body).hexdigest(),
|
||||
)
|
||||
|
||||
|
||||
def _validate_remote_image_url(url: str) -> str:
|
||||
normalized = url.strip()
|
||||
if not normalized or len(normalized) > KNOWLEDGE_FS_REMOTE_IMAGE_MAX_URL_CHARS:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_URL_INVALID",
|
||||
"Remote image URL is invalid.",
|
||||
retryable=False,
|
||||
)
|
||||
parsed = urllib.parse.urlsplit(normalized)
|
||||
if (
|
||||
parsed.scheme.lower() not in {"http", "https"}
|
||||
or not parsed.hostname
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
):
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_URL_INVALID",
|
||||
"Remote image URL is invalid.",
|
||||
retryable=False,
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _assert_remote_status(status_code: int) -> None:
|
||||
if 200 <= status_code < 300:
|
||||
return
|
||||
if status_code in {404, 410}:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_NOT_FOUND",
|
||||
"Remote image was not found.",
|
||||
retryable=False,
|
||||
)
|
||||
if status_code == 429:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_RATE_LIMITED",
|
||||
"Remote image service rate limited the request.",
|
||||
retryable=True,
|
||||
)
|
||||
if status_code in {408, 425} or status_code >= 500:
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_UPSTREAM_UNAVAILABLE",
|
||||
"Remote image service is temporarily unavailable.",
|
||||
retryable=True,
|
||||
)
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_REQUEST_REJECTED",
|
||||
"Remote image request was rejected.",
|
||||
retryable=False,
|
||||
)
|
||||
|
||||
|
||||
def _content_length(response: httpx.Response) -> int | None:
|
||||
value = response.headers.get("content-length")
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
parsed = int(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed >= 0 else None
|
||||
|
||||
|
||||
def _detect_image_mime_type(body: bytes) -> str:
|
||||
if body.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return "image/png"
|
||||
if body.startswith(b"\xff\xd8\xff"):
|
||||
return "image/jpeg"
|
||||
if body.startswith((b"GIF87a", b"GIF89a")):
|
||||
return "image/gif"
|
||||
if len(body) >= 12 and body.startswith(b"RIFF") and body[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
raise KnowledgeFSRemoteImageError(
|
||||
"REMOTE_IMAGE_CONTENT_UNSUPPORTED",
|
||||
"Remote image content is not supported.",
|
||||
retryable=False,
|
||||
)
|
||||
@ -15,6 +15,7 @@ from controllers.inner_api.knowledge_fs.storage import (
|
||||
KnowledgeFSObjectMetadataApi,
|
||||
KnowledgeFSObjectStorageHttpError,
|
||||
KnowledgeFSQueryImageApi,
|
||||
KnowledgeFSRemoteImageApi,
|
||||
)
|
||||
from services.knowledge_fs.object_storage import (
|
||||
KnowledgeFSObjectList,
|
||||
@ -23,6 +24,7 @@ from services.knowledge_fs.object_storage import (
|
||||
KnowledgeFSObjectStorageUnavailableError,
|
||||
)
|
||||
from services.knowledge_fs.query_images import KnowledgeFSResolvedQueryImage
|
||||
from services.knowledge_fs.remote_images import KnowledgeFSRemoteImageError, KnowledgeFSResolvedRemoteImage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@ -40,6 +42,60 @@ def _metadata_header(metadata: dict[str, str]) -> str:
|
||||
return urlsafe_b64encode(json.dumps(metadata).encode()).decode().rstrip("=")
|
||||
|
||||
|
||||
@patch("controllers.inner_api.knowledge_fs.storage.load_remote_image")
|
||||
def test_remote_image_endpoint_returns_only_sniffed_bounded_image_bytes(
|
||||
load_remote_image: MagicMock,
|
||||
app: Flask,
|
||||
) -> None:
|
||||
body = b"\x89PNG\r\n\x1a\nremote"
|
||||
load_remote_image.return_value = KnowledgeFSResolvedRemoteImage(
|
||||
byte_size=len(body),
|
||||
mime_type="image/png",
|
||||
body=body,
|
||||
sha256="a" * 64,
|
||||
)
|
||||
handler = KnowledgeFSRemoteImageApi()
|
||||
|
||||
with app.test_request_context("/?url=https%3A%2F%2Fcdn.example.test%2Fimage.png"):
|
||||
result = inspect.unwrap(handler.get)(handler)
|
||||
|
||||
assert result.get_data() == body
|
||||
assert result.content_type == "image/png"
|
||||
assert result.headers["X-Knowledge-FS-Remote-Image-Sha256"] == "a" * 64
|
||||
load_remote_image.assert_called_once_with("https://cdn.example.test/image.png")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_status"),
|
||||
[
|
||||
(KnowledgeFSRemoteImageError("REMOTE_IMAGE_BLOCKED", "blocked", retryable=False), 403),
|
||||
(KnowledgeFSRemoteImageError("REMOTE_IMAGE_NOT_FOUND", "missing", retryable=False), 404),
|
||||
(KnowledgeFSRemoteImageError("REMOTE_IMAGE_TOO_LARGE", "large", retryable=False), 413),
|
||||
(
|
||||
KnowledgeFSRemoteImageError("REMOTE_IMAGE_CONTENT_UNSUPPORTED", "unsupported", retryable=False),
|
||||
415,
|
||||
),
|
||||
(KnowledgeFSRemoteImageError("REMOTE_IMAGE_RATE_LIMITED", "limited", retryable=True), 429),
|
||||
(KnowledgeFSRemoteImageError("REMOTE_IMAGE_UPSTREAM_UNAVAILABLE", "offline", retryable=True), 502),
|
||||
],
|
||||
)
|
||||
@patch("controllers.inner_api.knowledge_fs.storage.load_remote_image")
|
||||
def test_remote_image_endpoint_maps_safe_failure_statuses(
|
||||
load_remote_image: MagicMock,
|
||||
error: KnowledgeFSRemoteImageError,
|
||||
expected_status: int,
|
||||
app: Flask,
|
||||
) -> None:
|
||||
load_remote_image.side_effect = error
|
||||
handler = KnowledgeFSRemoteImageApi()
|
||||
|
||||
with app.test_request_context("/?url=https%3A%2F%2Fcdn.example.test%2Fimage.png"):
|
||||
with pytest.raises(KnowledgeFSObjectStorageHttpError) as exc_info:
|
||||
inspect.unwrap(handler.get)(handler)
|
||||
|
||||
assert exc_info.value.code == expected_status
|
||||
|
||||
|
||||
@patch("controllers.inner_api.knowledge_fs.storage.load_query_image")
|
||||
def test_query_image_endpoint_returns_only_actor_owned_bounded_bytes(
|
||||
load_query_image: MagicMock,
|
||||
|
||||
170
api/tests/unit_tests/services/test_knowledge_fs_remote_images.py
Normal file
170
api/tests/unit_tests/services/test_knowledge_fs_remote_images.py
Normal file
@ -0,0 +1,170 @@
|
||||
"""Unit tests for SSRF-safe KnowledgeFS remote-image loading."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from core.helper.ssrf_proxy import ResponseTooLargeError, UnsupportedResponseEncodingError
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from services.knowledge_fs.remote_images import (
|
||||
KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES,
|
||||
KnowledgeFSRemoteImageError,
|
||||
load_remote_image,
|
||||
)
|
||||
|
||||
|
||||
def _response(status: int, body: bytes = b"", headers: dict[str, str] | None = None) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
status,
|
||||
content=body,
|
||||
headers=headers,
|
||||
request=httpx.Request("GET", "https://cdn.example.test/image"),
|
||||
)
|
||||
|
||||
|
||||
@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request")
|
||||
def test_load_remote_image_uses_ssrf_fetcher_and_sniffs_supported_content(
|
||||
make_request: MagicMock,
|
||||
) -> None:
|
||||
body = b"\x89PNG\r\n\x1a\nimage"
|
||||
make_request.return_value = _response(200, body, {"content-type": "application/octet-stream"})
|
||||
|
||||
result = load_remote_image("https://cdn.example.test/image")
|
||||
|
||||
assert result.body == body
|
||||
assert result.mime_type == "image/png"
|
||||
assert result.sha256 == "3c7474b4239ada3342d87f25ec8849eb8473ee35c5471452482686098b49e81b"
|
||||
make_request.assert_called_once_with(
|
||||
"GET",
|
||||
"https://cdn.example.test/image",
|
||||
follow_redirects=True,
|
||||
headers={
|
||||
"Accept": "image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.1",
|
||||
"Accept-Encoding": "identity",
|
||||
},
|
||||
max_retries=1,
|
||||
stream_response=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "mime_type"),
|
||||
[
|
||||
(b"\xff\xd8\xffimage", "image/jpeg"),
|
||||
(b"GIF89aimage", "image/gif"),
|
||||
(b"RIFF\x00\x00\x00\x00WEBPimage", "image/webp"),
|
||||
],
|
||||
)
|
||||
@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request")
|
||||
def test_load_remote_image_sniffs_other_supported_formats(
|
||||
make_request: MagicMock,
|
||||
body: bytes,
|
||||
mime_type: str,
|
||||
) -> None:
|
||||
make_request.return_value = _response(200, body, {"content-length": "invalid"})
|
||||
|
||||
assert load_remote_image("https://cdn.example.test/image").mime_type == mime_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"",
|
||||
"https://user:secret@cdn.example.test/private.png",
|
||||
"file:///etc/passwd",
|
||||
"data:image/png;base64,AQ==",
|
||||
"relative.png",
|
||||
"https://cdn.example.test/" + "x" * 8_192,
|
||||
],
|
||||
)
|
||||
def test_load_remote_image_rejects_non_http_urls(url: str) -> None:
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as exc_info:
|
||||
load_remote_image(url)
|
||||
|
||||
assert exc_info.value.code == "REMOTE_IMAGE_URL_INVALID"
|
||||
assert exc_info.value.retryable is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status", "code", "retryable"),
|
||||
[
|
||||
(404, "REMOTE_IMAGE_NOT_FOUND", False),
|
||||
(400, "REMOTE_IMAGE_REQUEST_REJECTED", False),
|
||||
(429, "REMOTE_IMAGE_RATE_LIMITED", True),
|
||||
(503, "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE", True),
|
||||
],
|
||||
)
|
||||
@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request")
|
||||
def test_load_remote_image_classifies_upstream_statuses(
|
||||
make_request: MagicMock,
|
||||
status: int,
|
||||
code: str,
|
||||
retryable: bool,
|
||||
) -> None:
|
||||
make_request.return_value = _response(status)
|
||||
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as exc_info:
|
||||
load_remote_image("https://cdn.example.test/image.png")
|
||||
|
||||
assert exc_info.value.code == code
|
||||
assert exc_info.value.retryable is retryable
|
||||
|
||||
|
||||
@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request")
|
||||
def test_load_remote_image_rejects_oversized_and_unsupported_bodies(make_request: MagicMock) -> None:
|
||||
make_request.return_value = _response(
|
||||
200,
|
||||
b"x",
|
||||
{"content-length": str(KNOWLEDGE_FS_REMOTE_IMAGE_MAX_BYTES + 1)},
|
||||
)
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as too_large:
|
||||
load_remote_image("https://cdn.example.test/large.png")
|
||||
assert too_large.value.code == "REMOTE_IMAGE_TOO_LARGE"
|
||||
|
||||
make_request.return_value = _response(200, b"not-an-image")
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as unsupported:
|
||||
load_remote_image("https://cdn.example.test/not-image")
|
||||
assert unsupported.value.code == "REMOTE_IMAGE_CONTENT_UNSUPPORTED"
|
||||
|
||||
make_request.return_value = _response(200, b"", {"content-length": "-1"})
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as empty:
|
||||
load_remote_image("https://cdn.example.test/empty.png")
|
||||
assert empty.value.code == "REMOTE_IMAGE_EMPTY"
|
||||
|
||||
|
||||
@patch("services.knowledge_fs.remote_images.ssrf_proxy.buffer_response")
|
||||
@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request")
|
||||
def test_load_remote_image_maps_stream_limit_and_ssrf_rejections(
|
||||
make_request: MagicMock,
|
||||
buffer_response: MagicMock,
|
||||
) -> None:
|
||||
make_request.return_value = _response(200, b"pending")
|
||||
buffer_response.side_effect = ResponseTooLargeError("large")
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as too_large:
|
||||
load_remote_image("https://cdn.example.test/large.png")
|
||||
assert too_large.value.code == "REMOTE_IMAGE_TOO_LARGE"
|
||||
|
||||
make_request.side_effect = ToolSSRFError("blocked")
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as blocked:
|
||||
load_remote_image("http://127.0.0.1/private.png")
|
||||
assert blocked.value.code == "REMOTE_IMAGE_BLOCKED"
|
||||
assert blocked.value.retryable is False
|
||||
|
||||
make_request.return_value = _response(200, b"pending")
|
||||
make_request.side_effect = None
|
||||
buffer_response.side_effect = UnsupportedResponseEncodingError("gzip")
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as unsupported_encoding:
|
||||
load_remote_image("https://cdn.example.test/compressed.png")
|
||||
assert unsupported_encoding.value.code == "REMOTE_IMAGE_CONTENT_UNSUPPORTED"
|
||||
|
||||
|
||||
@patch("services.knowledge_fs.remote_images.remote_fetcher.make_request")
|
||||
def test_load_remote_image_maps_transport_timeouts(make_request: MagicMock) -> None:
|
||||
make_request.side_effect = httpx.ReadTimeout("stalled")
|
||||
|
||||
with pytest.raises(KnowledgeFSRemoteImageError) as unavailable:
|
||||
load_remote_image("https://cdn.example.test/stalled.png")
|
||||
|
||||
assert unavailable.value.code == "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE"
|
||||
assert unavailable.value.retryable is True
|
||||
@ -71,3 +71,4 @@ UNSTRUCTURED_MAX_RESPONSE_BYTES=33554432
|
||||
|
||||
# Bound authenticated Dify object-storage calls so cleanup cannot hang a compilation lease forever.
|
||||
DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS=60000
|
||||
DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS=30000
|
||||
|
||||
@ -0,0 +1,35 @@
|
||||
# Office and remote document image materialization
|
||||
|
||||
## What changed
|
||||
|
||||
- Kept embedded OpenXML/ODF/EPUB images on the existing archive-media path and added a regression for legacy `.doc` image payload extraction.
|
||||
- Added a Dify-authenticated, SSRF-protected remote-image bridge for parser-produced HTTP(S) image references.
|
||||
- Materialized remote images from Markdown, HTML, Office, and other parser formats through the same bounded KnowledgeFS object-storage and thumbnail path as embedded images.
|
||||
- Included the final image-byte digest in synchronous compilation lineage so a changed visual artifact cannot reuse a stale multimodal publication identity.
|
||||
- Added deployment configuration for the bounded inner remote-image request timeout.
|
||||
|
||||
## Why
|
||||
|
||||
- Some parsers return linked document images as HTTP(S) references instead of embedded bytes. Those references previously remained in manifests without an `objectKey`, so the document page could not render them reliably.
|
||||
- The behavior must be format-independent: DOCX/DOC/PPTX/XLSX images need the same stable storage contract as Markdown and HTML images.
|
||||
|
||||
## Safety boundaries
|
||||
|
||||
- Only PNG, JPEG, GIF, and WebP payloads up to 10 MiB are accepted after content sniffing.
|
||||
- External requests use Dify's existing signed-file resolver and SSRF-protected network client; credentials in URLs and unsafe network targets are rejected.
|
||||
- Missing, blocked, oversized, or unsupported linked images remain as source references and do not block embedded Office images or fail the entire document.
|
||||
- Transient bridge failures remain retryable, while each KnowledgeFS-to-Dify request has a configurable 30-second default timeout.
|
||||
|
||||
## Verification
|
||||
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/parsers test:coverage` (63 passed; package coverage above 90% in every dimension)
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/adapters test:coverage` (117 passed; package coverage above 90% in every dimension)
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/api-app test` (262 passed)
|
||||
- `pnpm --dir knowledge-fs --filter @knowledge/api test:coverage` (all tests passed; branch coverage 89.27%, identical to clean-HEAD baseline and below the repository's pre-existing 90% gate)
|
||||
- Focused KnowledgeFS Python tests (31 passed; remote-image service line/branch coverage 98.8%/96.2%)
|
||||
- KnowledgeFS TypeScript typechecks, Biome, Ruff, deployment compose tests, and contract-lock verification
|
||||
|
||||
## Risks and follow-up
|
||||
|
||||
- Linked images are fetched sequentially under the existing per-document extraction cap to keep memory bounded. Large documents with many slow external hosts may take longer than documents whose images are embedded.
|
||||
- Existing revisions whose manifests have no `objectKey` are not mutated in place; reindexing or creating a new revision is required to materialize their missing linked images.
|
||||
@ -165,6 +165,7 @@ export interface CreateApiDocumentCompilationRuntimeOptions {
|
||||
| "documentMultimodalMaxExtractedAssets"
|
||||
| "documentMultimodalMaxLocalAssetBytes"
|
||||
| "documentMultimodalMaxPdfRasterizedAssets"
|
||||
| "documentMultimodalRemoteAssetFetcher"
|
||||
| "documentPdfRasterizer"
|
||||
> & { readonly documentMultimodalMaxConcurrency?: number | undefined })
|
||||
| undefined;
|
||||
@ -604,6 +605,11 @@ export function createApiDocumentCompilationRuntime({
|
||||
multimodalMaxPdfRasterizedAssets: multimodal.documentMultimodalMaxPdfRasterizedAssets,
|
||||
}
|
||||
: {}),
|
||||
...(multimodal?.documentMultimodalRemoteAssetFetcher
|
||||
? {
|
||||
multimodalRemoteAssetFetcher: multimodal.documentMultimodalRemoteAssetFetcher,
|
||||
}
|
||||
: {}),
|
||||
multimodalManifests: repositories.multimodalManifests,
|
||||
...(createModelBudget ? { modelBudget: createModelBudget() } : {}),
|
||||
objectStorage: adapter.objectStorage,
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import { createNodePlatformAdapter } from "@knowledge/adapters/node";
|
||||
import {
|
||||
createNodePlatformAdapter,
|
||||
createNodeRemoteDocumentImageFetcher,
|
||||
} from "@knowledge/adapters/node";
|
||||
import {
|
||||
type KnowledgeSpaceEmbeddingResolver,
|
||||
createDatabaseDeletionObjectWriteAdmission,
|
||||
@ -121,6 +124,7 @@ const researchTaskDirectStream = createApiResearchTaskDirectStreamAssembly({
|
||||
});
|
||||
|
||||
const adapter = createNodePlatformAdapter();
|
||||
const documentRemoteAssetFetcher = createNodeRemoteDocumentImageFetcher();
|
||||
const queryImageResolver = createApiQueryImageResolver({ env: process.env });
|
||||
const queryImageExpansionProvider = createApiQueryImageExpansionProvider(process.env);
|
||||
const operationalMetrics = createApiKnowledgeFsOperationalMetrics({
|
||||
@ -165,7 +169,10 @@ const visualEmbeddingOptions = createApiVisualEmbeddingOptions({
|
||||
modelRequestGate: ingestionModelRuntimeOptions.modelRequestGate,
|
||||
objectStorage: adapter.objectStorage,
|
||||
});
|
||||
const multimodalOptions = createApiMultimodalOptions();
|
||||
const multimodalOptions = {
|
||||
...createApiMultimodalOptions(),
|
||||
documentMultimodalRemoteAssetFetcher: documentRemoteAssetFetcher,
|
||||
};
|
||||
const multimodalAnswerOptions = createApiMultimodalAnswerOptions({
|
||||
objectStorage: adapter.objectStorage,
|
||||
});
|
||||
|
||||
@ -27,6 +27,7 @@ data:
|
||||
DURABLE_DELETION_ENABLED: "off"
|
||||
DIFY_INNER_API_URL: http://api:5001
|
||||
DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS: "60000"
|
||||
DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS: "30000"
|
||||
DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES: "8388608"
|
||||
DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS: "60000"
|
||||
DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: "8388608"
|
||||
|
||||
@ -22,6 +22,7 @@ UNSTRUCTURED_RETRY_DELAY_MS=
|
||||
DIFY_INNER_API_URL=http://host.docker.internal:5001
|
||||
DIFY_INNER_API_KEY=
|
||||
DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS=60000
|
||||
DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS=30000
|
||||
DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES=8388608
|
||||
DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS=60000
|
||||
DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES=8388608
|
||||
|
||||
@ -39,6 +39,7 @@ services:
|
||||
DIFY_INNER_API_KEY: ${DIFY_INNER_API_KEY:-}
|
||||
DIFY_INNER_API_URL: ${DIFY_INNER_API_URL:-http://host.docker.internal:5001}
|
||||
DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS: ${DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS:-60000}
|
||||
DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS: ${DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS:-30000}
|
||||
DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES:-8388608}
|
||||
DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS: ${DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS:-60000}
|
||||
DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES:-8388608}
|
||||
|
||||
112
knowledge-fs/packages/adapters/src/dify-remote-image.test.ts
Normal file
112
knowledge-fs/packages/adapters/src/dify-remote-image.test.ts
Normal file
@ -0,0 +1,112 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { DifyRemoteImageRequestError, createDifyRemoteImageFetcher } from "./dify-remote-image";
|
||||
|
||||
const pngBytes = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]);
|
||||
|
||||
describe("Dify remote image fetcher", () => {
|
||||
it("uses the authenticated SSRF-safe inner endpoint and returns bounded bytes", async () => {
|
||||
const fetch = vi.fn<typeof globalThis.fetch>().mockResolvedValue(
|
||||
new Response(pngBytes, {
|
||||
headers: {
|
||||
"Content-Length": String(pngBytes.byteLength),
|
||||
"Content-Type": "image/png",
|
||||
},
|
||||
}),
|
||||
);
|
||||
const resolver = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch,
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolver.fetch({ maxBytes: 1024, url: "https://cdn.example.test/a b.png?x=1&y=2" }),
|
||||
).resolves.toEqual({ body: pngBytes, contentType: "image/png" });
|
||||
|
||||
const requestUrl = fetch.mock.calls[0]?.[0].toString() ?? "";
|
||||
expect(requestUrl).toContain("/inner/api/knowledge-fs/remote-image?");
|
||||
expect(new URL(requestUrl).searchParams.get("url")).toBe(
|
||||
"https://cdn.example.test/a%20b.png?x=1&y=2",
|
||||
);
|
||||
expect(new Headers(fetch.mock.calls[0]?.[1]?.headers).get("X-Inner-Api-Key")).toBe("inner-key");
|
||||
});
|
||||
|
||||
it.each([400, 403, 404, 413, 415, 422])(
|
||||
"leaves terminally unavailable remote images inline for status %s",
|
||||
async (status) => {
|
||||
const resolver = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch: vi.fn<typeof globalThis.fetch>().mockResolvedValue(new Response(null, { status })),
|
||||
});
|
||||
|
||||
await expect(
|
||||
resolver.fetch({ maxBytes: 1024, url: "https://cdn.example.test/missing.png" }),
|
||||
).resolves.toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it("classifies transient responses, transport errors, and timeouts as retryable", async () => {
|
||||
const unavailable = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch: vi
|
||||
.fn<typeof globalThis.fetch>()
|
||||
.mockResolvedValue(new Response(null, { status: 503 })),
|
||||
});
|
||||
await expect(
|
||||
unavailable.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }),
|
||||
).rejects.toMatchObject({ retryable: true, status: 503 });
|
||||
|
||||
const offline = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch: vi.fn<typeof globalThis.fetch>().mockRejectedValue(new Error("offline")),
|
||||
});
|
||||
await expect(
|
||||
offline.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }),
|
||||
).rejects.toBeInstanceOf(DifyRemoteImageRequestError);
|
||||
|
||||
const stalled = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch: async (input, init) => {
|
||||
const signal = init?.signal ?? (input instanceof Request ? input.signal : undefined);
|
||||
return await new Promise<Response>((_resolve, reject) => {
|
||||
signal?.addEventListener("abort", () => reject(signal.reason), { once: true });
|
||||
});
|
||||
},
|
||||
requestTimeoutMs: 10,
|
||||
});
|
||||
await expect(
|
||||
stalled.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }),
|
||||
).rejects.toMatchObject({ retryable: true });
|
||||
});
|
||||
|
||||
it("rejects oversized or invalid successful responses", async () => {
|
||||
const oversized = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch: vi.fn<typeof globalThis.fetch>().mockResolvedValue(
|
||||
new Response(new Uint8Array([1, 2, 3]), {
|
||||
headers: { "Content-Length": "3", "Content-Type": "image/png" },
|
||||
}),
|
||||
),
|
||||
});
|
||||
await expect(
|
||||
oversized.fetch({ maxBytes: 2, url: "https://cdn.example.test/image.png" }),
|
||||
).rejects.toThrow("exceeds maxBytes=2");
|
||||
|
||||
const invalidType = createDifyRemoteImageFetcher({
|
||||
apiKey: "inner-key",
|
||||
baseUrl: "http://api:5001",
|
||||
fetch: vi
|
||||
.fn<typeof globalThis.fetch>()
|
||||
.mockResolvedValue(new Response(pngBytes, { headers: { "Content-Type": "text/plain" } })),
|
||||
});
|
||||
await expect(
|
||||
invalidType.fetch({ maxBytes: 1024, url: "https://cdn.example.test/image.png" }),
|
||||
).rejects.toThrow("content type is invalid");
|
||||
});
|
||||
});
|
||||
208
knowledge-fs/packages/adapters/src/dify-remote-image.ts
Normal file
208
knowledge-fs/packages/adapters/src/dify-remote-image.ts
Normal file
@ -0,0 +1,208 @@
|
||||
export interface RemoteDocumentImageFetchInput {
|
||||
readonly maxBytes: number;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly url: string;
|
||||
}
|
||||
|
||||
export interface ResolvedRemoteDocumentImage {
|
||||
readonly body: Uint8Array;
|
||||
readonly contentType: string;
|
||||
}
|
||||
|
||||
export interface RemoteDocumentImageFetcher {
|
||||
fetch(input: RemoteDocumentImageFetchInput): Promise<ResolvedRemoteDocumentImage | null>;
|
||||
}
|
||||
|
||||
export interface DifyRemoteImageFetcherOptions {
|
||||
readonly apiKey: string;
|
||||
readonly baseUrl: string;
|
||||
readonly fetch?: typeof globalThis.fetch;
|
||||
readonly requestTimeoutMs?: number;
|
||||
}
|
||||
|
||||
const defaultRequestTimeoutMs = 30_000;
|
||||
const terminalUnavailableStatuses = new Set([400, 403, 404, 413, 415, 422]);
|
||||
const supportedContentTypes = new Set(["image/gif", "image/jpeg", "image/png", "image/webp"]);
|
||||
|
||||
export class DifyRemoteImageRequestError extends Error {
|
||||
readonly code = "dify_remote_image_request_failed";
|
||||
readonly retryable: boolean;
|
||||
readonly status?: number;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
options: {
|
||||
readonly cause?: unknown;
|
||||
readonly retryable: boolean;
|
||||
readonly status?: number;
|
||||
},
|
||||
) {
|
||||
super(message, options.cause === undefined ? undefined : { cause: options.cause });
|
||||
this.name = "DifyRemoteImageRequestError";
|
||||
this.retryable = options.retryable;
|
||||
if (options.status !== undefined) this.status = options.status;
|
||||
}
|
||||
}
|
||||
|
||||
export function createDifyRemoteImageFetcher({
|
||||
apiKey,
|
||||
baseUrl,
|
||||
fetch = globalThis.fetch,
|
||||
requestTimeoutMs = defaultRequestTimeoutMs,
|
||||
}: DifyRemoteImageFetcherOptions): RemoteDocumentImageFetcher {
|
||||
const normalizedBaseUrl = requiredBaseUrl(baseUrl);
|
||||
const normalizedApiKey = requiredString(apiKey, "Dify inner API key");
|
||||
positiveSafeInteger(requestTimeoutMs, "requestTimeoutMs");
|
||||
|
||||
return {
|
||||
async fetch({ maxBytes, signal, url }) {
|
||||
positiveSafeInteger(maxBytes, "maxBytes");
|
||||
const remoteUrl = validRemoteUrl(url);
|
||||
const timeout = AbortSignal.timeout(requestTimeoutMs);
|
||||
const requestSignal = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(
|
||||
new URL(
|
||||
`/inner/api/knowledge-fs/remote-image?${new URLSearchParams({ url: remoteUrl }).toString()}`,
|
||||
normalizedBaseUrl,
|
||||
),
|
||||
{
|
||||
headers: { "X-Inner-Api-Key": normalizedApiKey },
|
||||
signal: requestSignal,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw signal.reason;
|
||||
throw new DifyRemoteImageRequestError("Dify remote image request failed", {
|
||||
cause: error,
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
if (terminalUnavailableStatuses.has(response.status)) {
|
||||
await response.body?.cancel();
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new DifyRemoteImageRequestError(
|
||||
`Dify remote image request failed with status ${response.status}`,
|
||||
{ retryable: isRetryableStatus(response.status), status: response.status },
|
||||
);
|
||||
}
|
||||
|
||||
const contentType = normalizedImageContentType(response.headers.get("content-type"));
|
||||
if (!contentType) {
|
||||
await response.body?.cancel();
|
||||
throw new DifyRemoteImageRequestError("Dify remote image content type is invalid", {
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
const body = await readBoundedBody(response, maxBytes, signal);
|
||||
return { body, contentType };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function readBoundedBody(
|
||||
response: Response,
|
||||
maxBytes: number,
|
||||
callerSignal: AbortSignal | undefined,
|
||||
): Promise<Uint8Array> {
|
||||
const declaredLength = Number.parseInt(response.headers.get("content-length") ?? "", 10);
|
||||
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
||||
await response.body?.cancel();
|
||||
throw new DifyRemoteImageRequestError(`Dify remote image exceeds maxBytes=${maxBytes}`, {
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
if (!response.body) return new Uint8Array();
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
totalBytes += value.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
throw new DifyRemoteImageRequestError(`Dify remote image exceeds maxBytes=${maxBytes}`, {
|
||||
retryable: false,
|
||||
});
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} catch (error) {
|
||||
if (callerSignal?.aborted) throw callerSignal.reason;
|
||||
if (error instanceof DifyRemoteImageRequestError) throw error;
|
||||
throw new DifyRemoteImageRequestError("Dify remote image response failed", {
|
||||
cause: error,
|
||||
retryable: true,
|
||||
});
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const body = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
body.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function normalizedImageContentType(value: string | null): string | null {
|
||||
const normalized = value?.split(";", 1)[0]?.trim().toLowerCase() ?? "";
|
||||
return supportedContentTypes.has(normalized) ? normalized : null;
|
||||
}
|
||||
|
||||
function validRemoteUrl(value: string): string {
|
||||
let url: URL;
|
||||
try {
|
||||
url = new URL(value);
|
||||
} catch {
|
||||
throw new Error("Remote image URL is invalid");
|
||||
}
|
||||
if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) {
|
||||
throw new Error("Remote image URL is invalid");
|
||||
}
|
||||
return url.href;
|
||||
}
|
||||
|
||||
function isRetryableStatus(status: number): boolean {
|
||||
return status === 408 || status === 425 || status === 429 || status >= 500;
|
||||
}
|
||||
|
||||
function requiredBaseUrl(value: string): string {
|
||||
const normalized = requiredString(value, "Dify inner API URL");
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(normalized);
|
||||
} catch {
|
||||
throw new Error("Dify inner API URL is invalid");
|
||||
}
|
||||
if (
|
||||
(parsed.protocol !== "http:" && parsed.protocol !== "https:") ||
|
||||
parsed.username ||
|
||||
parsed.password
|
||||
) {
|
||||
throw new Error("Dify inner API URL is invalid");
|
||||
}
|
||||
return parsed.href.endsWith("/") ? parsed.href : `${parsed.href}/`;
|
||||
}
|
||||
|
||||
function requiredString(value: string, name: string): string {
|
||||
const normalized = value.trim();
|
||||
if (!normalized) throw new Error(`${name} is required`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function positiveSafeInteger(value: number, name: string): void {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
}
|
||||
@ -2,6 +2,7 @@ export * from "./cache";
|
||||
export * from "./cloudflare-job-queue";
|
||||
export * from "./database";
|
||||
export * from "./dify-object-storage";
|
||||
export * from "./dify-remote-image";
|
||||
export * from "./job-queue";
|
||||
export * from "./memory-object-storage";
|
||||
export * from "./migration-runner";
|
||||
|
||||
@ -3,6 +3,7 @@ import { type PlatformAdapter, collectPlatformHealth } from "@knowledge/core";
|
||||
import { createMemoryCacheAdapter } from "./cache";
|
||||
import { createSchemaDatabaseAdapter } from "./database";
|
||||
import { createDifyObjectStorageAdapter } from "./dify-object-storage";
|
||||
import { createDifyRemoteImageFetcher } from "./dify-remote-image";
|
||||
import { createInlineJobQueueAdapter } from "./job-queue";
|
||||
import { type PgBossClient, createPgBossJobQueueAdapter } from "./pg-boss-job-queue";
|
||||
import {
|
||||
@ -52,6 +53,16 @@ export function createNodePlatformAdapter(
|
||||
return adapter;
|
||||
}
|
||||
|
||||
export function createNodeRemoteDocumentImageFetcher(options: NodePlatformAdapterOptions = {}) {
|
||||
const env = options.env ?? process.env;
|
||||
return createDifyRemoteImageFetcher({
|
||||
apiKey: env.DIFY_INNER_API_KEY?.trim() || defaultDifyInnerApiKey,
|
||||
baseUrl: env.DIFY_INNER_API_URL?.trim() || defaultDifyInnerApiUrl,
|
||||
...(options.difyStorageFetch ? { fetch: options.difyStorageFetch } : {}),
|
||||
requestTimeoutMs: parsePositiveInteger(env.DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS, 30_000),
|
||||
});
|
||||
}
|
||||
|
||||
function createNodeDatabaseAdapter(env: RuntimeEnv, databasePool?: PostgresPoolLike) {
|
||||
const configuredUrl = env.DATABASE_URL?.trim();
|
||||
|
||||
|
||||
@ -26,6 +26,115 @@ const firstArtifactId = "30000000-0000-4000-8000-000000000001";
|
||||
const retryArtifactId = "30000000-0000-4000-8000-000000000002";
|
||||
|
||||
describe("compileDocumentArtifact canonical artifact", () => {
|
||||
it("materializes parser-provided remote image refs before persisting the manifest", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 });
|
||||
const artifactSegments = createInMemoryArtifactSegmentRepository({
|
||||
maxBatchSize: 10,
|
||||
maxListLimit: 10,
|
||||
maxSegments: 10,
|
||||
});
|
||||
const documentMultimodalManifests = createInMemoryDocumentMultimodalManifestRepository({
|
||||
maxManifests: 4,
|
||||
});
|
||||
const outlines = createInMemoryDocumentOutlineRepository({ maxOutlines: 4 });
|
||||
const knowledgePaths = createInMemoryKnowledgePathRepository({
|
||||
maxListLimit: 20,
|
||||
maxPaths: 20,
|
||||
});
|
||||
const remoteBody = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10, 1, 2, 3]);
|
||||
const remoteFetches: unknown[] = [];
|
||||
|
||||
const artifact = await compileDocumentArtifact(
|
||||
{
|
||||
asset: { ...documentAsset(), filename: "linked-images.docx", mimeType: "application/docx" },
|
||||
body: new Uint8Array([1, 2, 3]),
|
||||
knowledgeSpaceId,
|
||||
permissionScope: [],
|
||||
tenantId: "tenant-1",
|
||||
traceId: randomUUID(),
|
||||
},
|
||||
{
|
||||
artifacts,
|
||||
artifactSegments,
|
||||
documentMultimodalManifests,
|
||||
documentMultimodalRemoteAssetFetcher: {
|
||||
fetch: async (input) => {
|
||||
remoteFetches.push(input);
|
||||
return { body: remoteBody, contentType: "image/png" };
|
||||
},
|
||||
},
|
||||
documentParser: {
|
||||
kind: "unstructured",
|
||||
parse: async (input) =>
|
||||
ParseArtifactSchema.parse({
|
||||
artifactHash: "c".repeat(64),
|
||||
contentType: "mixed",
|
||||
createdAt: "2026-07-13T00:00:00.000Z",
|
||||
documentAssetId: input.documentAssetId,
|
||||
elements: [
|
||||
{
|
||||
id: "linked-image",
|
||||
metadata: {
|
||||
assetRef: { uri: "https://cdn.example.test/office-linked.png" },
|
||||
},
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
},
|
||||
],
|
||||
id: firstArtifactId,
|
||||
metadata: {},
|
||||
parser: "unstructured",
|
||||
version: input.version,
|
||||
}),
|
||||
},
|
||||
generateArtifactSegmentId: randomUUID,
|
||||
generateKnowledgePathId: randomUUID,
|
||||
knowledgePaths,
|
||||
now: () => "2026-07-13T00:00:00.000Z",
|
||||
objectStorage: adapter.objectStorage,
|
||||
outlineBuilder: createDocumentOutlineBuilder({
|
||||
generateId: randomUUID,
|
||||
maxElements: 10,
|
||||
maxNodes: 10,
|
||||
maxSummaryChars: 1_000,
|
||||
now: () => "2026-07-13T00:00:00.000Z",
|
||||
}),
|
||||
outlines,
|
||||
synchronousUploadReindexer: null,
|
||||
traces: createNoopTraceRecorder(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(remoteFetches).toEqual([
|
||||
{ maxBytes: 10 * 1024 * 1024, url: "https://cdn.example.test/office-linked.png" },
|
||||
]);
|
||||
expect(artifact.elements[0]?.metadata).toMatchObject({
|
||||
assetRef: {
|
||||
contentType: "image/png",
|
||||
objectKey: expect.any(String),
|
||||
source: "remote-url",
|
||||
},
|
||||
});
|
||||
expect(artifact.elements[0]?.metadata).not.toHaveProperty("assetRef.uri");
|
||||
expect(artifact.artifactHash).not.toBe("c".repeat(64));
|
||||
expect(artifact.metadata).toMatchObject({
|
||||
multimodalMaterialization: {
|
||||
assetCount: 1,
|
||||
sourceArtifactHash: "c".repeat(64),
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
documentMultimodalManifests.getByDocumentVersion({ documentAssetId, version: 1 }),
|
||||
).resolves.toMatchObject({
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
assetRef: expect.objectContaining({ objectKey: expect.any(String) }),
|
||||
}),
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the first persisted artifact id for every derived write on parser retry", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 });
|
||||
|
||||
@ -18,7 +18,9 @@ import {
|
||||
buildDocumentOutlineKnowledgePath,
|
||||
buildDocumentSectionKnowledgePaths,
|
||||
} from "./document-knowledge-paths";
|
||||
import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact";
|
||||
import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor";
|
||||
import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor";
|
||||
import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder";
|
||||
import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository";
|
||||
import {
|
||||
@ -70,6 +72,7 @@ export interface CompileDocumentArtifactDeps {
|
||||
readonly documentMultimodalMaxExtractedAssets?: number | undefined;
|
||||
readonly documentMultimodalMaxLocalAssetBytes?: number | undefined;
|
||||
readonly documentMultimodalMaxPdfRasterizedAssets?: number | undefined;
|
||||
readonly documentMultimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined;
|
||||
readonly documentMultimodalManifests: DocumentMultimodalManifestRepository;
|
||||
readonly documentParser: ParserAdapter;
|
||||
readonly documentPdfRasterizer?: DocumentPdfRasterizer | undefined;
|
||||
@ -111,6 +114,7 @@ export async function compileDocumentArtifact(
|
||||
documentMultimodalMaxExtractedAssets,
|
||||
documentMultimodalMaxLocalAssetBytes,
|
||||
documentMultimodalMaxPdfRasterizedAssets,
|
||||
documentMultimodalRemoteAssetFetcher,
|
||||
documentMultimodalManifests,
|
||||
documentParser,
|
||||
documentPdfRasterizer,
|
||||
@ -190,13 +194,17 @@ export async function compileDocumentArtifact(
|
||||
? { imageVariantGenerator: documentMultimodalImageVariantGenerator }
|
||||
: {}),
|
||||
objectStorage,
|
||||
...(documentMultimodalRemoteAssetFetcher
|
||||
? { remoteAssetFetcher: documentMultimodalRemoteAssetFetcher }
|
||||
: {}),
|
||||
tenantId,
|
||||
}),
|
||||
);
|
||||
const materializedArtifact = finalizeDocumentMultimodalArtifact(assetExtractionResult.artifact);
|
||||
const artifactToPersist = ParseArtifactSchema.parse({
|
||||
...assetExtractionResult.artifact,
|
||||
...materializedArtifact,
|
||||
metadata: {
|
||||
...assetExtractionResult.artifact.metadata,
|
||||
...materializedArtifact.metadata,
|
||||
...(assetExtractionResult.extractedCount > 0
|
||||
? { multimodalAssetExtractionCount: assetExtractionResult.extractedCount }
|
||||
: {}),
|
||||
|
||||
@ -49,7 +49,10 @@ import {
|
||||
} from "./document-knowledge-paths";
|
||||
import type { DocumentModelBudget } from "./document-model-budget";
|
||||
import { finalizeDocumentMultimodalArtifact } from "./document-multimodal-artifact";
|
||||
import { extractDocumentMultimodalAssets } from "./document-multimodal-asset-extractor";
|
||||
import {
|
||||
type DocumentRemoteAssetFetcher,
|
||||
extractDocumentMultimodalAssets,
|
||||
} from "./document-multimodal-asset-extractor";
|
||||
import { createDocumentMultimodalManifestBuilder } from "./document-multimodal-manifest-builder";
|
||||
import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository";
|
||||
import type { DocumentOutlineBuilder } from "./document-outline-builder";
|
||||
@ -111,6 +114,7 @@ export interface DocumentCompilationWorkerOptions {
|
||||
readonly multimodalMaxConcurrency?: number | undefined;
|
||||
readonly multimodalMaxLocalAssetBytes?: number | undefined;
|
||||
readonly multimodalMaxPdfRasterizedAssets?: number | undefined;
|
||||
readonly multimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined;
|
||||
readonly multimodalManifests: DocumentMultimodalManifestRepository;
|
||||
readonly modelBudget?: DocumentModelBudget | undefined;
|
||||
readonly objectStorage: PlatformAdapter["objectStorage"];
|
||||
@ -247,6 +251,7 @@ export function createDocumentCompilationWorker({
|
||||
multimodalMaxConcurrency = 2,
|
||||
multimodalMaxLocalAssetBytes,
|
||||
multimodalMaxPdfRasterizedAssets,
|
||||
multimodalRemoteAssetFetcher,
|
||||
multimodalManifests,
|
||||
modelBudget,
|
||||
objectStorage,
|
||||
@ -469,6 +474,10 @@ export function createDocumentCompilationWorker({
|
||||
? { imageVariantGenerator: multimodalImageVariantGenerator }
|
||||
: {}),
|
||||
objectStorage: multimodalObjectStorage,
|
||||
...(multimodalRemoteAssetFetcher
|
||||
? { remoteAssetFetcher: multimodalRemoteAssetFetcher }
|
||||
: {}),
|
||||
...(signal ? { signal } : {}),
|
||||
tenantId: input.tenantId,
|
||||
writeOwnerId: multimodalWriteOwnerId,
|
||||
});
|
||||
|
||||
@ -11,6 +11,229 @@ const documentAssetId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c43";
|
||||
const parseArtifactId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c44";
|
||||
|
||||
describe("extractDocumentMultimodalAssets", () => {
|
||||
it("downloads and stores remote images independently of the source document format", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const remoteBody = new Uint8Array([
|
||||
137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 2, 0, 0, 0, 3, 8, 6, 0,
|
||||
0, 0, 0, 0, 0, 0,
|
||||
]);
|
||||
const fetchCalls: unknown[] = [];
|
||||
|
||||
const result = await extractDocumentMultimodalAssets({
|
||||
artifact: {
|
||||
artifactHash: "a".repeat(64),
|
||||
contentType: "mixed",
|
||||
createdAt: "2026-06-23T00:00:00.000Z",
|
||||
documentAssetId,
|
||||
elements: [
|
||||
{
|
||||
id: "markdown-image",
|
||||
metadata: { assetRef: { uri: "https://cdn.example.test/markdown.png" } },
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
},
|
||||
{
|
||||
id: "office-linked-image",
|
||||
metadata: { assetRef: { uri: "https://cdn.example.test/office.png" } },
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
},
|
||||
],
|
||||
id: parseArtifactId,
|
||||
metadata: {},
|
||||
parser: "native-markdown",
|
||||
version: 1,
|
||||
},
|
||||
knowledgeSpaceId,
|
||||
maxRemoteAssetBytes: 1024,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: {
|
||||
fetch: async (input) => {
|
||||
fetchCalls.push(input);
|
||||
return { body: remoteBody, contentType: "image/png; charset=binary" };
|
||||
},
|
||||
},
|
||||
tenantId: "tenant-1",
|
||||
});
|
||||
|
||||
expect(result.extractedCount).toBe(2);
|
||||
expect(fetchCalls).toEqual([
|
||||
{ maxBytes: 1024, url: "https://cdn.example.test/markdown.png" },
|
||||
{ maxBytes: 1024, url: "https://cdn.example.test/office.png" },
|
||||
]);
|
||||
for (const element of result.artifact.elements) {
|
||||
expect(element.metadata).toMatchObject({
|
||||
assetRef: {
|
||||
contentType: "image/png",
|
||||
height: 3,
|
||||
objectKey: expect.any(String),
|
||||
source: "remote-url",
|
||||
width: 2,
|
||||
},
|
||||
});
|
||||
expect(element.metadata).not.toHaveProperty("assetRef.uri");
|
||||
}
|
||||
expect(result.artifact.metadata).toMatchObject({
|
||||
multimodalAssets: { extractedCount: 2, sources: ["remote-url"] },
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves unavailable remote images inline without blocking embedded Office images", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const controller = new AbortController();
|
||||
const remoteFetches: unknown[] = [];
|
||||
|
||||
const result = await extractDocumentMultimodalAssets({
|
||||
artifact: {
|
||||
artifactHash: "a".repeat(64),
|
||||
contentType: "mixed",
|
||||
createdAt: "2026-06-23T00:00:00.000Z",
|
||||
documentAssetId,
|
||||
elements: [
|
||||
{
|
||||
id: "remote-missing",
|
||||
metadata: { assetRef: { uri: "https://cdn.example.test/missing.png" } },
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
},
|
||||
{
|
||||
id: "docx-embedded",
|
||||
metadata: {
|
||||
archivePath: "word/media/image1.png",
|
||||
assetRef: { uri: "data:image/png;base64,AQIDBA==" },
|
||||
},
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
},
|
||||
],
|
||||
id: parseArtifactId,
|
||||
metadata: {},
|
||||
parser: "unstructured",
|
||||
version: 1,
|
||||
},
|
||||
knowledgeSpaceId,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: {
|
||||
fetch: async (input) => {
|
||||
remoteFetches.push(input);
|
||||
return null;
|
||||
},
|
||||
},
|
||||
signal: controller.signal,
|
||||
tenantId: "tenant-1",
|
||||
});
|
||||
|
||||
expect(result.extractedCount).toBe(1);
|
||||
expect(remoteFetches).toEqual([
|
||||
{
|
||||
maxBytes: 10 * 1024 * 1024,
|
||||
signal: controller.signal,
|
||||
url: "https://cdn.example.test/missing.png",
|
||||
},
|
||||
]);
|
||||
expect(result.artifact.elements[0]?.metadata).toEqual({
|
||||
assetRef: { uri: "https://cdn.example.test/missing.png" },
|
||||
});
|
||||
expect(result.artifact.elements[1]?.metadata).toMatchObject({
|
||||
archivePath: "word/media/image1.png",
|
||||
assetRef: { objectKey: expect.any(String), source: "data-uri" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects invalid limits and dishonest remote fetcher responses", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const artifact = remoteImageArtifact("https://cdn.example.test/image.png");
|
||||
|
||||
await expect(
|
||||
extractDocumentMultimodalAssets({
|
||||
artifact,
|
||||
knowledgeSpaceId,
|
||||
maxRemoteAssetBytes: 0,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: { fetch: async () => null },
|
||||
tenantId: "tenant-1",
|
||||
}),
|
||||
).rejects.toThrow("Document multimodal remote asset max bytes must be at least 1");
|
||||
|
||||
await expect(
|
||||
extractDocumentMultimodalAssets({
|
||||
artifact,
|
||||
knowledgeSpaceId,
|
||||
maxRemoteAssetBytes: 1,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: {
|
||||
fetch: async () => ({ body: new Uint8Array([1, 2]), contentType: "image/png" }),
|
||||
},
|
||||
tenantId: "tenant-1",
|
||||
}),
|
||||
).rejects.toThrow("Document multimodal remote asset exceeds maxRemoteAssetBytes=1");
|
||||
|
||||
await expect(
|
||||
extractDocumentMultimodalAssets({
|
||||
artifact,
|
||||
knowledgeSpaceId,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: {
|
||||
fetch: async () => ({ body: new Uint8Array([1]), contentType: "text/plain" }),
|
||||
},
|
||||
tenantId: "tenant-1",
|
||||
}),
|
||||
).rejects.toThrow("Document multimodal remote asset content type is unsupported");
|
||||
|
||||
await expect(
|
||||
extractDocumentMultimodalAssets({
|
||||
artifact,
|
||||
knowledgeSpaceId,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: {
|
||||
fetch: async () => ({ body: new Uint8Array(), contentType: "image/png" }),
|
||||
},
|
||||
tenantId: "tenant-1",
|
||||
}),
|
||||
).rejects.toThrow("Document multimodal remote asset is empty");
|
||||
});
|
||||
|
||||
it("does not fetch remote refs after the extraction cap or URLs containing credentials", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const fetchCalls: unknown[] = [];
|
||||
const artifact = remoteImageArtifact("https://cdn.example.test/second.png");
|
||||
artifact.elements.unshift({
|
||||
id: "embedded-first",
|
||||
metadata: { assetRef: { uri: "data:image/png;base64,AQIDBA==" } },
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
});
|
||||
artifact.elements.push({
|
||||
id: "credential-url",
|
||||
metadata: { assetRef: { uri: "https://user:secret@cdn.example.test/private.png" } },
|
||||
sectionPath: [],
|
||||
type: "image",
|
||||
});
|
||||
|
||||
const result = await extractDocumentMultimodalAssets({
|
||||
artifact,
|
||||
knowledgeSpaceId,
|
||||
maxExtractedAssets: 1,
|
||||
objectStorage: adapter.objectStorage,
|
||||
remoteAssetFetcher: {
|
||||
fetch: async (input) => {
|
||||
fetchCalls.push(input);
|
||||
return null;
|
||||
},
|
||||
},
|
||||
tenantId: "tenant-1",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ extractedCount: 1, skippedForCapCount: 1 });
|
||||
expect(fetchCalls).toEqual([]);
|
||||
expect(result.artifact.elements[1]?.metadata).toEqual({
|
||||
assetRef: { uri: "https://cdn.example.test/second.png" },
|
||||
});
|
||||
expect(result.artifact.elements[2]?.metadata).toEqual({
|
||||
assetRef: { uri: "https://user:secret@cdn.example.test/private.png" },
|
||||
});
|
||||
});
|
||||
|
||||
it("stores embedded image data URIs and rewrites image asset refs", async () => {
|
||||
const adapter = createNodePlatformAdapter({ env: {} });
|
||||
const dataUri = "data:image/png;base64,AQIDBA==";
|
||||
@ -413,3 +636,24 @@ describe("extractDocumentMultimodalAssets", () => {
|
||||
).resolves.toEqual(new Uint8Array([5, 6, 7, 8]));
|
||||
});
|
||||
});
|
||||
|
||||
function remoteImageArtifact(uri: string) {
|
||||
return {
|
||||
artifactHash: "a".repeat(64),
|
||||
contentType: "mixed" as const,
|
||||
createdAt: "2026-06-23T00:00:00.000Z",
|
||||
documentAssetId,
|
||||
elements: [
|
||||
{
|
||||
id: "remote-image",
|
||||
metadata: { assetRef: { uri } },
|
||||
sectionPath: [],
|
||||
type: "image" as const,
|
||||
},
|
||||
],
|
||||
id: parseArtifactId,
|
||||
metadata: {},
|
||||
parser: "unstructured" as const,
|
||||
version: 1,
|
||||
};
|
||||
}
|
||||
|
||||
@ -22,12 +22,23 @@ export interface ExtractDocumentMultimodalAssetsInput {
|
||||
readonly maxEmbeddedAssetBytes?: number | undefined;
|
||||
readonly maxExtractedAssets?: number | undefined;
|
||||
readonly maxLocalAssetBytes?: number | undefined;
|
||||
readonly maxRemoteAssetBytes?: number | undefined;
|
||||
readonly imageVariantGenerator?: DocumentImageVariantGenerator | undefined;
|
||||
readonly objectStorage: PlatformAdapter["objectStorage"];
|
||||
readonly remoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly tenantId: string;
|
||||
readonly writeOwnerId?: string | undefined;
|
||||
}
|
||||
|
||||
export interface DocumentRemoteAssetFetcher {
|
||||
fetch(input: {
|
||||
readonly maxBytes: number;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly url: string;
|
||||
}): Promise<{ readonly body: Uint8Array; readonly contentType: string } | null>;
|
||||
}
|
||||
|
||||
export interface ExtractDocumentMultimodalAssetsResult {
|
||||
readonly artifact: ParseArtifact;
|
||||
readonly extractedCount: number;
|
||||
@ -39,7 +50,7 @@ interface DataUriImage {
|
||||
readonly body: Uint8Array;
|
||||
readonly contentType: string;
|
||||
readonly dimensions?: ImageDimensions | undefined;
|
||||
readonly source: "data-uri" | "local-file";
|
||||
readonly source: "data-uri" | "local-file" | "remote-url";
|
||||
}
|
||||
|
||||
interface ImageDimensions {
|
||||
@ -51,6 +62,13 @@ const dataUriPattern = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)$/iu;
|
||||
const defaultMaxEmbeddedAssetBytes = 10 * 1024 * 1024;
|
||||
const defaultMaxExtractedAssets = 1_000;
|
||||
const defaultMaxLocalAssetBytes = 50 * 1024 * 1024;
|
||||
const defaultMaxRemoteAssetBytes = 10 * 1024 * 1024;
|
||||
const supportedRemoteImageContentTypes = new Set([
|
||||
"image/gif",
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
]);
|
||||
|
||||
export async function extractDocumentMultimodalAssets({
|
||||
allowLocalAssetPaths = [],
|
||||
@ -59,8 +77,11 @@ export async function extractDocumentMultimodalAssets({
|
||||
maxEmbeddedAssetBytes = defaultMaxEmbeddedAssetBytes,
|
||||
maxExtractedAssets = defaultMaxExtractedAssets,
|
||||
maxLocalAssetBytes = defaultMaxLocalAssetBytes,
|
||||
maxRemoteAssetBytes = defaultMaxRemoteAssetBytes,
|
||||
imageVariantGenerator,
|
||||
objectStorage,
|
||||
remoteAssetFetcher,
|
||||
signal,
|
||||
tenantId,
|
||||
writeOwnerId,
|
||||
}: ExtractDocumentMultimodalAssetsInput): Promise<ExtractDocumentMultimodalAssetsResult> {
|
||||
@ -76,6 +97,10 @@ export async function extractDocumentMultimodalAssets({
|
||||
throw new Error("Document multimodal max extracted assets must be at least 1");
|
||||
}
|
||||
|
||||
if (!Number.isSafeInteger(maxRemoteAssetBytes) || maxRemoteAssetBytes < 1) {
|
||||
throw new Error("Document multimodal remote asset max bytes must be at least 1");
|
||||
}
|
||||
|
||||
let extractedCount = 0;
|
||||
let skippedForCapCount = 0;
|
||||
const extractionSources = new Set<string>();
|
||||
@ -90,7 +115,7 @@ export async function extractDocumentMultimodalAssets({
|
||||
|
||||
const assetRef = isPlainObject(element.metadata.assetRef) ? element.metadata.assetRef : null;
|
||||
const uri = typeof assetRef?.uri === "string" ? assetRef.uri.trim() : "";
|
||||
const image =
|
||||
let image =
|
||||
parseDataUriImage(uri, maxEmbeddedAssetBytes) ??
|
||||
(await readLocalImageAsset({
|
||||
allowedRoots: allowedLocalRoots,
|
||||
@ -99,6 +124,40 @@ export async function extractDocumentMultimodalAssets({
|
||||
uri,
|
||||
}));
|
||||
|
||||
if (!image && remoteAssetFetcher && isRemoteHttpUri(uri)) {
|
||||
if (extractedCount >= maxExtractedAssets) {
|
||||
skippedForCapCount += 1;
|
||||
elements.push(element);
|
||||
continue;
|
||||
}
|
||||
const fetched = await remoteAssetFetcher.fetch({
|
||||
maxBytes: maxRemoteAssetBytes,
|
||||
...(signal ? { signal } : {}),
|
||||
url: uri,
|
||||
});
|
||||
if (fetched) {
|
||||
if (fetched.body.byteLength > maxRemoteAssetBytes) {
|
||||
throw new Error(
|
||||
`Document multimodal remote asset exceeds maxRemoteAssetBytes=${maxRemoteAssetBytes}`,
|
||||
);
|
||||
}
|
||||
const contentType = normalizeRemoteImageContentType(fetched.contentType);
|
||||
if (!contentType) {
|
||||
throw new Error("Document multimodal remote asset content type is unsupported");
|
||||
}
|
||||
if (fetched.body.byteLength === 0) {
|
||||
throw new Error("Document multimodal remote asset is empty");
|
||||
}
|
||||
const dimensions = readImageDimensions(fetched.body, contentType);
|
||||
image = {
|
||||
body: fetched.body,
|
||||
contentType,
|
||||
...(dimensions ? { dimensions } : {}),
|
||||
source: "remote-url",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!assetRef || !image) {
|
||||
elements.push(element);
|
||||
continue;
|
||||
@ -204,6 +263,24 @@ export async function extractDocumentMultimodalAssets({
|
||||
};
|
||||
}
|
||||
|
||||
function isRemoteHttpUri(uri: string): boolean {
|
||||
try {
|
||||
const parsed = new URL(uri);
|
||||
return (
|
||||
(parsed.protocol === "http:" || parsed.protocol === "https:") &&
|
||||
!parsed.username &&
|
||||
!parsed.password
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRemoteImageContentType(value: string): string | null {
|
||||
const normalized = value.split(";", 1)[0]?.trim().toLowerCase() ?? "";
|
||||
return supportedRemoteImageContentTypes.has(normalized) ? normalized : null;
|
||||
}
|
||||
|
||||
async function storeGeneratedImageVariants({
|
||||
assetId,
|
||||
elementId,
|
||||
|
||||
@ -32,6 +32,7 @@ import type { DocumentCompilationJobStateMachine } from "./document-compilation-
|
||||
import { compileDocumentArtifact } from "./document-compilation-pipeline";
|
||||
import type { DocumentImageVariantGenerator } from "./document-image-variant-generator";
|
||||
import { buildDocumentKnowledgePath } from "./document-knowledge-paths";
|
||||
import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor";
|
||||
import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository";
|
||||
import type { DocumentOutlineBuilder } from "./document-outline-builder";
|
||||
import type { DocumentOutlineRepository } from "./document-outline-repository";
|
||||
@ -149,6 +150,7 @@ export interface RegisterDocumentWriteHandlersOptions {
|
||||
readonly documentMultimodalMaxExtractedAssets?: number | undefined;
|
||||
readonly documentMultimodalMaxLocalAssetBytes?: number | undefined;
|
||||
readonly documentMultimodalMaxPdfRasterizedAssets?: number | undefined;
|
||||
readonly documentMultimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher | undefined;
|
||||
readonly documentMultimodalManifests: DocumentMultimodalManifestRepository;
|
||||
readonly documentParser: ParserAdapter;
|
||||
readonly documentPdfRasterizer?: DocumentPdfRasterizer | undefined;
|
||||
@ -220,6 +222,7 @@ export function registerDocumentWriteHandlers({
|
||||
documentMultimodalMaxExtractedAssets,
|
||||
documentMultimodalMaxLocalAssetBytes,
|
||||
documentMultimodalMaxPdfRasterizedAssets,
|
||||
documentMultimodalRemoteAssetFetcher,
|
||||
documentMultimodalManifests,
|
||||
documentParser,
|
||||
documentPdfRasterizer,
|
||||
@ -1437,6 +1440,7 @@ export function registerDocumentWriteHandlers({
|
||||
documentMultimodalMaxExtractedAssets,
|
||||
documentMultimodalMaxLocalAssetBytes,
|
||||
documentMultimodalMaxPdfRasterizedAssets,
|
||||
documentMultimodalRemoteAssetFetcher,
|
||||
documentMultimodalManifests,
|
||||
documentParser,
|
||||
documentPdfRasterizer,
|
||||
|
||||
@ -28,6 +28,7 @@ import type {
|
||||
} from "./document-chunk-repository";
|
||||
import type { DocumentCompilationJobStateMachine } from "./document-compilation-job";
|
||||
import type { DocumentImageVariantGenerator } from "./document-image-variant-generator";
|
||||
import type { DocumentRemoteAssetFetcher } from "./document-multimodal-asset-extractor";
|
||||
import type { DocumentMultimodalManifestEnhancer } from "./document-multimodal-manifest-enhancer";
|
||||
import type { DocumentMultimodalManifestRepository } from "./document-multimodal-manifest-repository";
|
||||
import type { DocumentOutlineRepository } from "./document-outline-repository";
|
||||
@ -211,6 +212,7 @@ export interface KnowledgeGatewayOptions {
|
||||
documentMultimodalLocalAssetAllowlist?: readonly string[];
|
||||
documentMultimodalMaxLocalAssetBytes?: number;
|
||||
documentMultimodalMaxPdfRasterizedAssets?: number;
|
||||
documentMultimodalRemoteAssetFetcher?: DocumentRemoteAssetFetcher;
|
||||
documentPdfRasterizer?: DocumentPdfRasterizer;
|
||||
documentOutlineSummaryEnhancer?: DocumentOutlineSummaryEnhancer;
|
||||
documentOutlines?: DocumentOutlineRepository;
|
||||
|
||||
@ -105,6 +105,7 @@ export * from "./document-asset-repository";
|
||||
export * from "./document-asset-embedding-profile-guard";
|
||||
export * from "./durable-deletion-repository";
|
||||
export * from "./document-image-variant-generator";
|
||||
export * from "./document-multimodal-asset-extractor";
|
||||
export * from "./document-multimodal-enrichment-providers";
|
||||
export * from "./document-multimodal-candidate-resolver";
|
||||
export * from "./document-multimodal-evaluation";
|
||||
@ -735,6 +736,7 @@ export function createKnowledgeGateway({
|
||||
documentMultimodalMaxExtractedAssets,
|
||||
documentMultimodalMaxLocalAssetBytes,
|
||||
documentMultimodalMaxPdfRasterizedAssets,
|
||||
documentMultimodalRemoteAssetFetcher,
|
||||
documentPdfRasterizer,
|
||||
documentOutlineSummaryEnhancer,
|
||||
documentOutlines,
|
||||
@ -2012,6 +2014,7 @@ export function createKnowledgeGateway({
|
||||
...(documentMultimodalMaxExtractedAssets ? { documentMultimodalMaxExtractedAssets } : {}),
|
||||
...(documentMultimodalImageVariantGenerator ? { documentMultimodalImageVariantGenerator } : {}),
|
||||
...(documentMultimodalMaxLocalAssetBytes ? { documentMultimodalMaxLocalAssetBytes } : {}),
|
||||
...(documentMultimodalRemoteAssetFetcher ? { documentMultimodalRemoteAssetFetcher } : {}),
|
||||
...(documentMultimodalMaxPdfRasterizedAssets
|
||||
? { documentMultimodalMaxPdfRasterizedAssets }
|
||||
: {}),
|
||||
@ -2071,6 +2074,7 @@ export function createKnowledgeGateway({
|
||||
documentMultimodalMaxExtractedAssets,
|
||||
documentMultimodalMaxLocalAssetBytes,
|
||||
documentMultimodalMaxPdfRasterizedAssets,
|
||||
documentMultimodalRemoteAssetFetcher,
|
||||
documentMultimodalManifests: multimodalManifestRepository,
|
||||
documentParser,
|
||||
documentPdfRasterizer,
|
||||
|
||||
@ -1405,6 +1405,56 @@ describe("parser adapters", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("requests and preserves provider image payloads for legacy DOC files", async () => {
|
||||
const parser = createUnstructuredParserClient({
|
||||
endpoint: "https://unstructured.example.test",
|
||||
fetch: async (request) => {
|
||||
const form = await (request instanceof Request ? request : new Request(request)).formData();
|
||||
|
||||
expect(form.get("strategy")).toBe("hi_res");
|
||||
expect(form.getAll("extract_image_block_types")).toEqual(["Image"]);
|
||||
expect(form.get("extract_image_block_to_payload")).toBe("true");
|
||||
|
||||
return new Response(
|
||||
JSON.stringify([
|
||||
{
|
||||
metadata: {
|
||||
image_base64: "AQIDBA==",
|
||||
image_mime_type: "image/png",
|
||||
page_number: 1,
|
||||
},
|
||||
type: "Image",
|
||||
},
|
||||
]),
|
||||
{ headers: { "content-type": "application/json" }, status: 200 },
|
||||
);
|
||||
},
|
||||
generateId: () => "018f0d60-7a49-7cc2-9c1b-5b36f18f2c54",
|
||||
now: () => createdAt,
|
||||
});
|
||||
|
||||
const artifact = await parser.parse({
|
||||
body: new Uint8Array([0xd0, 0xcf, 0x11, 0xe0]),
|
||||
documentAssetId,
|
||||
filename: "legacy.doc",
|
||||
mimeType: "application/msword",
|
||||
parserHints: { requiresImages: true },
|
||||
version: 1,
|
||||
});
|
||||
|
||||
expect(artifact.elements).toEqual([
|
||||
expect.objectContaining({
|
||||
metadata: expect.objectContaining({
|
||||
assetRef: {
|
||||
contentType: "image/png",
|
||||
uri: "data:image/png;base64,AQIDBA==",
|
||||
},
|
||||
}),
|
||||
type: "image",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves nested Unstructured title paths from parent ids and category depth", async () => {
|
||||
const parser = createUnstructuredParserClient({
|
||||
endpoint: "https://unstructured.example.test",
|
||||
|
||||
@ -328,6 +328,7 @@ test("KnowledgeFS deployment env contains only operator-owned runtime inputs", (
|
||||
"UNSTRUCTURED_REQUEST_TIMEOUT_MS",
|
||||
"UNSTRUCTURED_MAX_RESPONSE_BYTES",
|
||||
"DIFY_OBJECT_STORAGE_REQUEST_TIMEOUT_MS",
|
||||
"DIFY_REMOTE_IMAGE_REQUEST_TIMEOUT_MS",
|
||||
]);
|
||||
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME=on$/m);
|
||||
assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_PDF_RASTERIZER=poppler$/m);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user