From f482bf0aa1706f96b87b5e55059bfb12465b854e Mon Sep 17 00:00:00 2001 From: Jyong Date: Thu, 23 Jul 2026 01:53:40 -0400 Subject: [PATCH] Refactor codebase and remove obsolete implementations --- api/controllers/inner_api/__init__.py | 2 + .../inner_api/knowledge_fs/__init__.py | 1 + .../inner_api/knowledge_fs/storage.py | 303 ++++ api/controllers/inner_api/wraps.py | 6 + api/extensions/storage/aws_s3_storage.py | 30 + api/services/knowledge_fs/object_storage.py | 346 ++++ .../inner_api/knowledge_fs/__init__.py | 1 + .../inner_api/knowledge_fs/test_storage.py | 175 ++ .../controllers/inner_api/test_auth_wraps.py | 28 + .../extensions/storage/test_aws_s3_storage.py | 65 + .../test_knowledge_fs_object_storage.py | 142 ++ docker/README.md | 38 +- docker/docker-compose-template.yaml | 7 +- docker/docker-compose.yaml | 7 +- .../core-services/knowledge-fs.env.example | 69 +- knowledge-fs/.env.example | 88 - knowledge-fs/README.md | 570 ++----- knowledge-fs/apps/api/Dockerfile | 2 - knowledge-fs/apps/api/package.json | 3 +- .../api/src/answer-generation-options.test.ts | 8 +- .../apps/api/src/answer-generation-options.ts | 4 +- .../api/src/datasource-invocation-client.ts | 2 +- .../datasource-runtime-code-health.test.ts | 31 +- .../src/datasource-runtime-options.test.ts | 59 +- .../api/src/datasource-runtime-options.ts | 22 +- .../apps/api/src/embedding-options.ts | 6 +- knowledge-fs/apps/api/src/index.ts | 194 +-- knowledge-fs/apps/api/src/llm-options.test.ts | 4 +- knowledge-fs/apps/api/src/llm-options.ts | 6 +- .../api/src/model-runtime-code-health.test.ts | 16 +- .../api/src/multimodal-answer-options.test.ts | 4 +- .../apps/api/src/multimodal-answer-options.ts | 6 +- .../src/multimodal-enrichment-options.test.ts | 4 +- .../api/src/multimodal-enrichment-options.ts | 6 +- .../apps/api/src/plugin-daemon-options.ts | 71 - .../apps/api/src/readiness-options.test.ts | 40 +- .../apps/api/src/readiness-options.ts | 20 +- .../apps/api/src/repository-options.test.ts | 4 +- knowledge-fs/apps/api/src/reranker-options.ts | 6 +- .../apps/api/src/server-options.test.ts | 4 +- ...source-credential-backfill-options.test.ts | 95 -- .../src/source-credential-backfill-options.ts | 115 -- .../src/source-oauth-provider-options.test.ts | 105 -- .../api/src/source-oauth-provider-options.ts | 251 --- .../api/src/source-secret-options.test.ts | 87 - .../apps/api/src/source-secret-options.ts | 80 - ...alone-datasource-invocation-client.test.ts | 76 - ...standalone-datasource-invocation-client.ts | 127 -- .../api/src/visual-embedding-options.test.ts | 6 +- .../apps/api/src/visual-embedding-options.ts | 8 +- knowledge-fs/apps/api/vitest.config.ts | 8 +- knowledge-fs/docs/api-reference.md | 5 +- knowledge-fs/docs/operator-manual.md | 33 +- knowledge-fs/docs/production-deployment.md | 620 ++----- knowledge-fs/docs/project-overview.md | 46 +- knowledge-fs/infra/aws_terraform/README.md | 93 -- knowledge-fs/infra/kubernetes/README.md | 15 +- knowledge-fs/infra/local/.env.example | 54 +- knowledge-fs/infra/local/README.md | 183 +-- .../infra/local/compose.middleware.yaml | 31 - knowledge-fs/infra/local/compose.yaml | 68 - knowledge-fs/package.json | 3 +- knowledge-fs/packages/adapters/package.json | 4 - .../packages/adapters/src/adapters.test.ts | 257 +-- .../packages/adapters/src/cloudflare.ts | 86 - .../packages/adapters/src/database.test.ts | 12 - .../adapters/src/dify-object-storage.test.ts | 84 + .../adapters/src/dify-object-storage.ts | 277 ++++ knowledge-fs/packages/adapters/src/index.ts | 4 +- .../src/memory-object-storage.test.ts | 127 ++ .../adapters/src/memory-object-storage.ts | 151 ++ knowledge-fs/packages/adapters/src/node.ts | 58 +- .../src/object-storage.integration.test.ts | 74 - .../adapters/src/object-storage.test.ts | 1424 ----------------- .../packages/adapters/src/object-storage.ts | 1122 ------------- .../api/src/core-resource-response-schemas.ts | 6 +- .../api/src/document-compilation-worker.ts | 2 +- .../api/src/gateway-document-write.test.ts | 29 +- ...ge-space-control-plane-diagnostics.test.ts | 8 +- ...knowledge-space-manifest-bootstrap.test.ts | 2 +- .../packages/api/src/upload-session.test.ts | 2 +- knowledge-fs/packages/api/vitest.config.ts | 3 + knowledge-fs/packages/core/src/models.test.ts | 2 +- knowledge-fs/packages/core/src/models.ts | 11 +- .../core/src/platform-adapter.test.ts | 4 +- .../packages/core/src/platform-adapter.ts | 4 +- .../plugin-daemon-client/package.json | 22 - .../plugin-daemon-client/src/index.test.ts | 468 ------ .../plugin-daemon-client/src/index.ts | 724 --------- .../plugin-daemon-client/tsconfig.json | 4 - .../plugin-daemon-client/vitest.config.ts | 18 - knowledge-fs/pnpm-lock.yaml | 1300 --------------- .../scripts/api-image-bundle-smoke.mjs | 9 +- .../scripts/api-image-bundle-smoke.test.mjs | 3 + .../scripts/audit-backend-dependencies.mjs | 84 + .../audit-backend-dependencies.test.mjs | 75 + knowledge-fs/scripts/compose-apps.test.mjs | 68 +- .../scripts/compose-middleware.test.mjs | 7 +- knowledge-fs/scripts/dify-compose-config.mjs | 2 - .../scripts/github-actions-workflow.test.mjs | 2 +- .../scripts/local-happy-path-smoke.mjs | 8 +- .../scripts/local-happy-path-smoke.test.mjs | 5 +- .../test/setup-dify-object-storage.ts | 124 ++ 103 files changed, 2696 insertions(+), 8489 deletions(-) create mode 100644 api/controllers/inner_api/knowledge_fs/__init__.py create mode 100644 api/controllers/inner_api/knowledge_fs/storage.py create mode 100644 api/services/knowledge_fs/object_storage.py create mode 100644 api/tests/unit_tests/controllers/inner_api/knowledge_fs/__init__.py create mode 100644 api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py create mode 100644 api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py create mode 100644 api/tests/unit_tests/services/test_knowledge_fs_object_storage.py delete mode 100644 knowledge-fs/.env.example delete mode 100644 knowledge-fs/apps/api/src/plugin-daemon-options.ts delete mode 100644 knowledge-fs/apps/api/src/source-credential-backfill-options.test.ts delete mode 100644 knowledge-fs/apps/api/src/source-credential-backfill-options.ts delete mode 100644 knowledge-fs/apps/api/src/source-oauth-provider-options.test.ts delete mode 100644 knowledge-fs/apps/api/src/source-oauth-provider-options.ts delete mode 100644 knowledge-fs/apps/api/src/source-secret-options.test.ts delete mode 100644 knowledge-fs/apps/api/src/source-secret-options.ts delete mode 100644 knowledge-fs/apps/api/src/standalone-datasource-invocation-client.test.ts delete mode 100644 knowledge-fs/apps/api/src/standalone-datasource-invocation-client.ts delete mode 100644 knowledge-fs/infra/aws_terraform/README.md delete mode 100644 knowledge-fs/packages/adapters/src/cloudflare.ts create mode 100644 knowledge-fs/packages/adapters/src/dify-object-storage.test.ts create mode 100644 knowledge-fs/packages/adapters/src/dify-object-storage.ts create mode 100644 knowledge-fs/packages/adapters/src/memory-object-storage.test.ts create mode 100644 knowledge-fs/packages/adapters/src/memory-object-storage.ts delete mode 100644 knowledge-fs/packages/adapters/src/object-storage.integration.test.ts delete mode 100644 knowledge-fs/packages/adapters/src/object-storage.test.ts delete mode 100644 knowledge-fs/packages/adapters/src/object-storage.ts delete mode 100644 knowledge-fs/packages/plugin-daemon-client/package.json delete mode 100644 knowledge-fs/packages/plugin-daemon-client/src/index.test.ts delete mode 100644 knowledge-fs/packages/plugin-daemon-client/src/index.ts delete mode 100644 knowledge-fs/packages/plugin-daemon-client/tsconfig.json delete mode 100644 knowledge-fs/packages/plugin-daemon-client/vitest.config.ts create mode 100644 knowledge-fs/scripts/audit-backend-dependencies.mjs create mode 100644 knowledge-fs/scripts/audit-backend-dependencies.test.mjs create mode 100644 knowledge-fs/test/setup-dify-object-storage.ts diff --git a/api/controllers/inner_api/__init__.py b/api/controllers/inner_api/__init__.py index f47861cf274..f278fbbb7d6 100644 --- a/api/controllers/inner_api/__init__.py +++ b/api/controllers/inner_api/__init__.py @@ -20,6 +20,7 @@ from . import runtime_credentials as _runtime_credentials from .agent import tools as _agent_tools from .app import dsl as _app_dsl from .knowledge import retrieval as _knowledge_retrieval +from .knowledge_fs import storage as _knowledge_fs_storage from .plugin import agent_config as _agent_config from .plugin import agent_drive as _agent_drive from .plugin import plugin as _plugin @@ -32,6 +33,7 @@ __all__ = [ "_agent_drive", "_agent_tools", "_app_dsl", + "_knowledge_fs_storage", "_knowledge_retrieval", "_mail", "_plugin", diff --git a/api/controllers/inner_api/knowledge_fs/__init__.py b/api/controllers/inner_api/knowledge_fs/__init__.py new file mode 100644 index 00000000000..a405c3cae7f --- /dev/null +++ b/api/controllers/inner_api/knowledge_fs/__init__.py @@ -0,0 +1 @@ +"""Trusted KnowledgeFS inner API endpoints.""" diff --git a/api/controllers/inner_api/knowledge_fs/storage.py b/api/controllers/inner_api/knowledge_fs/storage.py new file mode 100644 index 00000000000..d2652b974dc --- /dev/null +++ b/api/controllers/inner_api/knowledge_fs/storage.py @@ -0,0 +1,303 @@ +"""Trusted KnowledgeFS gateway to Dify's configured object-storage backend.""" + +import json +from base64 import b64decode +from binascii import Error as BinasciiError +from http import HTTPStatus +from typing import NoReturn + +from flask import Response, request +from flask_restx import Resource +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from pydantic.alias_generators import to_camel + +from controllers.common.schema import query_params_from_model, register_response_schema_models +from controllers.inner_api import inner_api_ns +from controllers.inner_api.wraps import knowledge_fs_inner_api_only +from fields.base import ResponseModel +from libs.exception import BaseHTTPException +from libs.helper import dump_response +from services.knowledge_fs.object_storage import ( + KNOWLEDGE_FS_OBJECT_MAX_BYTES, + KnowledgeFSObjectList, + KnowledgeFSObjectMetadata, + KnowledgeFSObjectStorageChecksumError, + KnowledgeFSObjectStorageCorruptError, + KnowledgeFSObjectStorageError, + KnowledgeFSObjectStorageInvalidInputError, + KnowledgeFSObjectStorageService, + KnowledgeFSObjectStorageTooLargeError, + KnowledgeFSObjectStorageUnavailableError, +) + +_METADATA_HEADER = "X-Knowledge-FS-Metadata" +_CHECKSUM_HEADER = "X-Knowledge-FS-Checksum-Sha256" +_CONTENT_TYPE_HEADER = "X-Knowledge-FS-Content-Type" +_MAX_ENCODED_METADATA_BYTES = 128 * 1024 +_metadata_adapter = TypeAdapter(dict[str, str]) + + +class KnowledgeFSObjectStorageHttpError(BaseHTTPException): + """Safe HTTP representation of a KnowledgeFS storage boundary error.""" + + error_code = "knowledge_fs_object_storage_failed" + description = "KnowledgeFS object storage request failed." + code = HTTPStatus.INTERNAL_SERVER_ERROR + + def __init__(self, *, error_code: str, description: str, status_code: HTTPStatus) -> None: + self.error_code = error_code + self.description = description + self.code = status_code + super().__init__(description) + + +class _CamelCaseResponse(ResponseModel): + model_config = ConfigDict(alias_generator=to_camel) + + +class KnowledgeFSObjectQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + key: str = Field(description="Logical KnowledgeFS object key") + + +class KnowledgeFSObjectListQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + prefix: str = Field(default="", description="Logical object-key prefix") + cursor: str | None = Field(default=None, description="Exclusive lexical key cursor") + limit: int = Field(default=100, ge=1, le=100, description="Maximum objects to return") + + +class KnowledgeFSObjectMetadataResponse(_CamelCaseResponse): + checksum_sha256_base64: str + content_type: str | None = None + key: str + metadata: dict[str, str] + size_bytes: int + + +class KnowledgeFSObjectListResponse(_CamelCaseResponse): + objects: list[KnowledgeFSObjectMetadataResponse] + next_cursor: str | None = None + + +class KnowledgeFSObjectHealthResponse(ResponseModel): + ok: bool + + +register_response_schema_models( + inner_api_ns, + KnowledgeFSObjectMetadataResponse, + KnowledgeFSObjectListResponse, + KnowledgeFSObjectHealthResponse, +) + + +@inner_api_ns.route("/knowledge-fs/storage/object") +class KnowledgeFSObjectApi(Resource): + """Read, write, or delete one logical KnowledgeFS object.""" + + @knowledge_fs_inner_api_only + @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery)) + @inner_api_ns.response( + HTTPStatus.OK, + "Object stored", + inner_api_ns.models[KnowledgeFSObjectMetadataResponse.__name__], + ) + def put(self) -> dict[str, object]: + try: + query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True)) + metadata = _decode_metadata_header(request.headers.get(_METADATA_HEADER)) + body = request.stream.read(KNOWLEDGE_FS_OBJECT_MAX_BYTES + 1) + if len(body) > KNOWLEDGE_FS_OBJECT_MAX_BYTES: + raise KnowledgeFSObjectStorageTooLargeError(f"object exceeds max bytes {KNOWLEDGE_FS_OBJECT_MAX_BYTES}") + result = KnowledgeFSObjectStorageService().put_object( + body=body, + checksum_sha256_base64=request.headers.get(_CHECKSUM_HEADER), + content_type=request.headers.get(_CONTENT_TYPE_HEADER), + key=query.key, + metadata=metadata, + ) + except ValidationError as exc: + raise _invalid_request_error() from exc + except KnowledgeFSObjectStorageError as exc: + _raise_http_error(exc) + return _metadata_response(result) + + @knowledge_fs_inner_api_only + @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery)) + @inner_api_ns.produces(["application/octet-stream"]) + def get(self) -> Response: + try: + query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True)) + service = KnowledgeFSObjectStorageService() + metadata = service.head_object(key=query.key) + if metadata is None: + raise _not_found_error() + body = service.load_stream(key=query.key) + if body is None: + raise _not_found_error() + except ValidationError as exc: + raise _invalid_request_error() from exc + except KnowledgeFSObjectStorageError as exc: + _raise_http_error(exc) + + response = Response( + body, + content_type=metadata.content_type or "application/octet-stream", + ) + response.content_length = metadata.size_bytes + response.headers[_CHECKSUM_HEADER] = metadata.checksum_sha256_base64 + return response + + @knowledge_fs_inner_api_only + @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery)) + @inner_api_ns.response(HTTPStatus.NO_CONTENT, "Object deleted") + def delete(self) -> tuple[str, int]: + try: + query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True)) + KnowledgeFSObjectStorageService().delete_object(key=query.key) + except ValidationError as exc: + raise _invalid_request_error() from exc + except KnowledgeFSObjectStorageError as exc: + _raise_http_error(exc) + return "", HTTPStatus.NO_CONTENT + + +@inner_api_ns.route("/knowledge-fs/storage/object/metadata") +class KnowledgeFSObjectMetadataApi(Resource): + """Read portable metadata for one logical KnowledgeFS object.""" + + @knowledge_fs_inner_api_only + @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery)) + @inner_api_ns.response( + HTTPStatus.OK, + "Object metadata", + inner_api_ns.models[KnowledgeFSObjectMetadataResponse.__name__], + ) + def get(self) -> dict[str, object]: + try: + query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True)) + result = KnowledgeFSObjectStorageService().head_object(key=query.key) + if result is None: + raise _not_found_error() + except ValidationError as exc: + raise _invalid_request_error() from exc + except KnowledgeFSObjectStorageError as exc: + _raise_http_error(exc) + return _metadata_response(result) + + +@inner_api_ns.route("/knowledge-fs/storage/objects") +class KnowledgeFSObjectListApi(Resource): + """List logical KnowledgeFS objects with bounded keyset pagination.""" + + @knowledge_fs_inner_api_only + @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectListQuery)) + @inner_api_ns.response( + HTTPStatus.OK, + "Object page", + inner_api_ns.models[KnowledgeFSObjectListResponse.__name__], + ) + def get(self) -> dict[str, object]: + try: + query = KnowledgeFSObjectListQuery.model_validate(request.args.to_dict(flat=True)) + result = KnowledgeFSObjectStorageService().list_objects( + cursor=query.cursor, + limit=query.limit, + prefix=query.prefix, + ) + except ValidationError as exc: + raise _invalid_request_error() from exc + except KnowledgeFSObjectStorageError as exc: + _raise_http_error(exc) + return _list_response(result) + + +@inner_api_ns.route("/knowledge-fs/storage/health") +class KnowledgeFSObjectHealthApi(Resource): + """Report whether Dify storage satisfies KnowledgeFS portable requirements.""" + + @knowledge_fs_inner_api_only + @inner_api_ns.response( + HTTPStatus.OK, + "Storage available", + inner_api_ns.models[KnowledgeFSObjectHealthResponse.__name__], + ) + @inner_api_ns.response(HTTPStatus.SERVICE_UNAVAILABLE, "Storage unavailable") + def get(self) -> dict[str, bool] | tuple[dict[str, bool], int]: + if KnowledgeFSObjectStorageService().health(): + return {"ok": True} + return {"ok": False}, HTTPStatus.SERVICE_UNAVAILABLE + + +def _decode_metadata_header(value: str | None) -> dict[str, str]: + if value is None: + return {} + if len(value.encode()) > _MAX_ENCODED_METADATA_BYTES: + raise KnowledgeFSObjectStorageInvalidInputError("object metadata header is too large") + try: + padding = "=" * (-len(value) % 4) + decoded = b64decode(value + padding, altchars=b"-_", validate=True) + return _metadata_adapter.validate_json(decoded) + except (BinasciiError, UnicodeEncodeError, ValidationError, json.JSONDecodeError) as exc: + raise KnowledgeFSObjectStorageInvalidInputError("object metadata header is invalid") from exc + + +def _metadata_response(metadata: KnowledgeFSObjectMetadata) -> dict[str, object]: + return dump_response(KnowledgeFSObjectMetadataResponse, metadata) + + +def _list_response(result: KnowledgeFSObjectList) -> dict[str, object]: + return dump_response(KnowledgeFSObjectListResponse, result) + + +def _raise_http_error(error: KnowledgeFSObjectStorageError) -> NoReturn: + if isinstance(error, KnowledgeFSObjectStorageTooLargeError): + raise KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_too_large", + description="KnowledgeFS object exceeds the configured size limit.", + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + ) from error + if isinstance(error, KnowledgeFSObjectStorageChecksumError): + raise KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_checksum_mismatch", + description="KnowledgeFS object checksum does not match the request body.", + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) from error + if isinstance(error, KnowledgeFSObjectStorageInvalidInputError): + raise _invalid_request_error() from error + if isinstance(error, KnowledgeFSObjectStorageCorruptError): + raise KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_corrupt", + description="KnowledgeFS object metadata is inconsistent.", + status_code=HTTPStatus.BAD_GATEWAY, + ) from error + if isinstance(error, KnowledgeFSObjectStorageUnavailableError): + raise KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_storage_unavailable", + description="Dify object storage is unavailable for KnowledgeFS.", + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + ) from error + raise KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_storage_failed", + description="KnowledgeFS object storage request failed.", + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) from error + + +def _invalid_request_error() -> KnowledgeFSObjectStorageHttpError: + return KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_storage_invalid_request", + description="KnowledgeFS object storage request is invalid.", + status_code=HTTPStatus.BAD_REQUEST, + ) + + +def _not_found_error() -> KnowledgeFSObjectStorageHttpError: + return KnowledgeFSObjectStorageHttpError( + error_code="knowledge_fs_object_not_found", + description="KnowledgeFS object was not found.", + status_code=HTTPStatus.NOT_FOUND, + ) diff --git a/api/controllers/inner_api/wraps.py b/api/controllers/inner_api/wraps.py index 5ddb55aaa3f..5dd88392a9c 100644 --- a/api/controllers/inner_api/wraps.py +++ b/api/controllers/inner_api/wraps.py @@ -107,3 +107,9 @@ def agent_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]: """ return plugin_inner_api_only(view) + + +def knowledge_fs_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]: + """Authenticate the trusted KnowledgeFS process on the shared inner bridge.""" + + return plugin_inner_api_only(view) diff --git a/api/extensions/storage/aws_s3_storage.py b/api/extensions/storage/aws_s3_storage.py index 018aa17ac45..6fc79bb89e3 100644 --- a/api/extensions/storage/aws_s3_storage.py +++ b/api/extensions/storage/aws_s3_storage.py @@ -92,3 +92,33 @@ class AwsS3Storage(BaseStorage): @override def delete(self, filename: str): self.client.delete_object(Bucket=self.bucket_name, Key=filename) + + @override + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: + """Recursively list keys below a portable storage directory.""" + if not files and not directories: + raise ValueError("At least one of files or directories must be True") + + normalized_path = path.strip("/") + prefix = f"{normalized_path}/" if normalized_path else "" + results: set[str] = set() + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.bucket_name, Prefix=prefix): + for item in page.get("Contents", []): + key = item.get("Key") + if not isinstance(key, str) or not key.startswith(prefix): + continue + if key.endswith("/"): + if directories: + results.add(key) + continue + + if files: + results.add(key) + if directories: + current = prefix + for segment in key[len(prefix) :].split("/")[:-1]: + current = f"{current}{segment}/" + results.add(current) + + return sorted(results) diff --git a/api/services/knowledge_fs/object_storage.py b/api/services/knowledge_fs/object_storage.py new file mode 100644 index 00000000000..7085dc891f9 --- /dev/null +++ b/api/services/knowledge_fs/object_storage.py @@ -0,0 +1,346 @@ +"""KnowledgeFS object operations backed by Dify's configured storage provider. + +KnowledgeFS is a separate Node process, so it reaches this service through a +trusted inner API rather than importing Python storage implementations or +holding a second set of provider credentials. Logical KnowledgeFS keys are +confined below a dedicated physical namespace. Metadata is stored as a small +sidecar because Dify's common ``BaseStorage`` contract intentionally exposes +portable byte operations rather than provider-specific object metadata. + +Object keys are immutable/versioned in KnowledgeFS product flows. A backend +must also support ``scan`` for bounded logical pagination and cleanup; health +fails closed when that capability is unavailable. +""" + +from __future__ import annotations + +import json +from base64 import b64decode, b64encode +from collections.abc import Generator, Mapping +from dataclasses import dataclass +from hashlib import sha256 +from typing import Protocol, TypedDict + +from extensions.ext_storage import storage + +KNOWLEDGE_FS_OBJECT_MAX_BYTES = 64 * 1024 * 1024 +_MAX_KEY_BYTES = 1024 +_MAX_LIST_LIMIT = 100 +_MAX_METADATA_ENTRIES = 64 +_MAX_METADATA_KEY_BYTES = 64 +_MAX_METADATA_VALUE_BYTES = 1024 +_DATA_ROOT = "knowledge-fs/objects" +_METADATA_ROOT = "knowledge-fs/object-metadata" +_HEALTH_SCAN_ROOT = "knowledge-fs/health-capability" + + +class KnowledgeFSObjectStorageError(Exception): + """Base error for the trusted KnowledgeFS storage gateway.""" + + +class KnowledgeFSObjectStorageInvalidInputError(KnowledgeFSObjectStorageError): + """The caller supplied an invalid key, cursor, metadata value, or limit.""" + + +class KnowledgeFSObjectStorageTooLargeError(KnowledgeFSObjectStorageInvalidInputError): + """The caller supplied an object body above the portable gateway limit.""" + + +class KnowledgeFSObjectStorageChecksumError(KnowledgeFSObjectStorageError): + """The supplied checksum does not match the uploaded object body.""" + + +class KnowledgeFSObjectStorageCorruptError(KnowledgeFSObjectStorageError): + """The physical object and its portable metadata sidecar disagree.""" + + +class KnowledgeFSObjectStorageUnavailableError(KnowledgeFSObjectStorageError): + """The selected Dify storage backend lacks a required portable capability.""" + + +class _StorageBackend(Protocol): + def save(self, filename: str, data: bytes) -> None: ... + + def load_once(self, filename: str) -> bytes: ... + + def load_stream(self, filename: str) -> Generator[bytes, None, None]: ... + + def exists(self, filename: str) -> bool: ... + + def delete(self, filename: str) -> None: ... + + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: ... + + +class _StoredMetadataPayload(TypedDict): + checksum_sha256_base64: str + content_type: str | None + key: str + metadata: dict[str, str] + size_bytes: int + version: int + + +@dataclass(frozen=True, slots=True) +class KnowledgeFSObjectMetadata: + """Portable metadata returned to the KnowledgeFS adapter.""" + + checksum_sha256_base64: str + content_type: str | None + key: str + metadata: Mapping[str, str] + size_bytes: int + + +@dataclass(frozen=True, slots=True) +class KnowledgeFSObjectList: + """One keyset-paginated page of logical KnowledgeFS objects.""" + + objects: tuple[KnowledgeFSObjectMetadata, ...] + next_cursor: str | None = None + + +class KnowledgeFSObjectStorageService: + """Confine KnowledgeFS object I/O to Dify's unified storage namespace.""" + + _backend: _StorageBackend + + def __init__(self, *, backend: _StorageBackend = storage) -> None: + self._backend = backend + + def put_object( + self, + *, + key: str, + body: bytes, + metadata: Mapping[str, str], + checksum_sha256_base64: str | None = None, + content_type: str | None = None, + ) -> KnowledgeFSObjectMetadata: + """Validate and persist one immutable logical object and its metadata sidecar.""" + normalized_key = _normalize_key(key) + normalized_metadata = _normalize_metadata(metadata) + normalized_content_type = _normalize_content_type(content_type) + if len(body) > KNOWLEDGE_FS_OBJECT_MAX_BYTES: + raise KnowledgeFSObjectStorageTooLargeError(f"object exceeds max bytes {KNOWLEDGE_FS_OBJECT_MAX_BYTES}") + + actual_checksum = b64encode(sha256(body).digest()).decode() + if checksum_sha256_base64 is not None: + expected_checksum = _normalize_checksum(checksum_sha256_base64) + if expected_checksum != actual_checksum: + raise KnowledgeFSObjectStorageChecksumError("object checksum does not match body") + + object_metadata = KnowledgeFSObjectMetadata( + checksum_sha256_base64=actual_checksum, + content_type=normalized_content_type, + key=normalized_key, + metadata=normalized_metadata, + size_bytes=len(body), + ) + data_path = _data_path(normalized_key) + self._backend.save(data_path, body) + try: + self._backend.save(_metadata_path(normalized_key), _encode_metadata(object_metadata)) + except Exception: + self._backend.delete(data_path) + raise + return object_metadata + + def head_object(self, *, key: str) -> KnowledgeFSObjectMetadata | None: + """Return portable metadata without loading the object body.""" + normalized_key = _normalize_key(key) + if not self._backend.exists(_data_path(normalized_key)): + return None + try: + encoded = self._backend.load_once(_metadata_path(normalized_key)) + except FileNotFoundError as exc: + raise KnowledgeFSObjectStorageCorruptError("object metadata sidecar is missing") from exc + metadata = _decode_metadata(encoded) + if metadata.key != normalized_key: + raise KnowledgeFSObjectStorageCorruptError("object metadata key does not match") + return metadata + + def load_stream(self, *, key: str) -> Generator[bytes, None, None] | None: + """Return the configured backend's stream for one logical object.""" + normalized_key = _normalize_key(key) + data_path = _data_path(normalized_key) + if not self._backend.exists(data_path): + return None + return self._backend.load_stream(data_path) + + def delete_object(self, *, key: str) -> None: + """Idempotently delete an object and its portable metadata sidecar.""" + normalized_key = _normalize_key(key) + self._backend.delete(_data_path(normalized_key)) + self._backend.delete(_metadata_path(normalized_key)) + + def list_objects( + self, + *, + prefix: str, + limit: int, + cursor: str | None = None, + ) -> KnowledgeFSObjectList: + """List logical keys using a stable lexical cursor and portable metadata.""" + normalized_prefix = _normalize_prefix(prefix) + if not isinstance(limit, int) or isinstance(limit, bool) or not 1 <= limit <= _MAX_LIST_LIMIT: + raise KnowledgeFSObjectStorageInvalidInputError("list limit must be between 1 and 100") + normalized_cursor = _normalize_key(cursor) if cursor is not None else None + if normalized_cursor is not None and not normalized_cursor.startswith(normalized_prefix): + raise KnowledgeFSObjectStorageInvalidInputError("list cursor must be within prefix") + + try: + physical_paths = self._backend.scan( + _physical_scan_prefix(normalized_prefix), + files=True, + directories=False, + ) + except FileNotFoundError: + physical_paths = [] + except NotImplementedError as exc: + raise KnowledgeFSObjectStorageUnavailableError( + "configured Dify storage backend does not support object listing" + ) from exc + except Exception as exc: + raise KnowledgeFSObjectStorageUnavailableError("Dify storage listing failed") from exc + + data_prefix = f"{_DATA_ROOT}/" + logical_keys = sorted( + { + physical_path[len(data_prefix) :] + for physical_path in physical_paths + if physical_path.startswith(data_prefix) + and physical_path[len(data_prefix) :].startswith(normalized_prefix) + and (normalized_cursor is None or physical_path[len(data_prefix) :] > normalized_cursor) + } + ) + page_keys = logical_keys[: limit + 1] + has_more = len(page_keys) > limit + objects: list[KnowledgeFSObjectMetadata] = [] + for logical_key in page_keys[:limit]: + metadata = self.head_object(key=logical_key) + if metadata is not None: + objects.append(metadata) + next_cursor = objects[-1].key if has_more and objects else None + return KnowledgeFSObjectList(objects=tuple(objects), next_cursor=next_cursor) + + def health(self) -> bool: + """Check the portable list capability without scanning the object namespace.""" + try: + self._backend.scan(_HEALTH_SCAN_ROOT, files=True, directories=False) + except FileNotFoundError: + return True + except Exception: + return False + return True + + +def _normalize_key(value: str) -> str: + if not isinstance(value, str): + raise KnowledgeFSObjectStorageInvalidInputError("object key must be a string") + normalized = value.strip() + segments = normalized.split("/") + if ( + not normalized + or normalized.startswith("/") + or "\\" in normalized + or "\x00" in normalized + or any(segment in {"", ".", ".."} for segment in segments) + or len(normalized.encode()) > _MAX_KEY_BYTES + ): + raise KnowledgeFSObjectStorageInvalidInputError("object key is invalid") + return normalized + + +def _normalize_prefix(value: str) -> str: + if value == "": + return value + normalized = value.strip() + candidate = normalized.removesuffix("/") + _normalize_key(candidate) + return normalized + + +def _normalize_metadata(metadata: Mapping[str, str]) -> dict[str, str]: + if len(metadata) > _MAX_METADATA_ENTRIES: + raise KnowledgeFSObjectStorageInvalidInputError("object metadata has too many entries") + normalized: dict[str, str] = {} + for key, value in metadata.items(): + if ( + not isinstance(key, str) + or not isinstance(value, str) + or not key + or len(key.encode()) > _MAX_METADATA_KEY_BYTES + or len(value.encode()) > _MAX_METADATA_VALUE_BYTES + ): + raise KnowledgeFSObjectStorageInvalidInputError("object metadata is invalid") + normalized[key] = value + return normalized + + +def _normalize_content_type(value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + if not normalized or len(normalized) > 255 or "\r" in normalized or "\n" in normalized: + raise KnowledgeFSObjectStorageInvalidInputError("object content type is invalid") + return normalized + + +def _normalize_checksum(value: str) -> str: + try: + decoded = b64decode(value, validate=True) + except ValueError as exc: + raise KnowledgeFSObjectStorageInvalidInputError("object checksum is invalid") from exc + if len(decoded) != 32 or b64encode(decoded).decode() != value: + raise KnowledgeFSObjectStorageInvalidInputError("object checksum is invalid") + return value + + +def _data_path(key: str) -> str: + return f"{_DATA_ROOT}/{key}" + + +def _metadata_path(key: str) -> str: + digest = sha256(key.encode()).hexdigest() + return f"{_METADATA_ROOT}/{digest[:2]}/{digest[2:4]}/{digest}.json" + + +def _physical_scan_prefix(prefix: str) -> str: + parent = prefix.rstrip("/").rpartition("/")[0] + return f"{_DATA_ROOT}/{parent}" if parent else _DATA_ROOT + + +def _encode_metadata(metadata: KnowledgeFSObjectMetadata) -> bytes: + payload: _StoredMetadataPayload = { + "checksum_sha256_base64": metadata.checksum_sha256_base64, + "content_type": metadata.content_type, + "key": metadata.key, + "metadata": dict(metadata.metadata), + "size_bytes": metadata.size_bytes, + "version": 1, + } + return json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() + + +def _decode_metadata(encoded: bytes) -> KnowledgeFSObjectMetadata: + try: + payload = json.loads(encoded) + if not isinstance(payload, dict) or payload.get("version") != 1: + raise ValueError + key = _normalize_key(payload["key"]) + checksum = _normalize_checksum(payload["checksum_sha256_base64"]) + content_type = _normalize_content_type(payload["content_type"]) + metadata = _normalize_metadata(payload["metadata"]) + size_bytes = payload["size_bytes"] + if not isinstance(size_bytes, int) or isinstance(size_bytes, bool) or size_bytes < 0: + raise ValueError + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise KnowledgeFSObjectStorageCorruptError("object metadata sidecar is invalid") from exc + return KnowledgeFSObjectMetadata( + checksum_sha256_base64=checksum, + content_type=content_type, + key=key, + metadata=metadata, + size_bytes=size_bytes, + ) diff --git a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/__init__.py b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/__init__.py new file mode 100644 index 00000000000..b98aaad6d16 --- /dev/null +++ b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/__init__.py @@ -0,0 +1 @@ +"""Unit tests for KnowledgeFS inner API controllers.""" diff --git a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py new file mode 100644 index 00000000000..403a60fdf59 --- /dev/null +++ b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_storage.py @@ -0,0 +1,175 @@ +"""Unit tests for the KnowledgeFS unified object-storage inner API.""" + +import inspect +import json +from base64 import urlsafe_b64encode +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask, Response + +from controllers.inner_api.knowledge_fs.storage import ( + KnowledgeFSObjectApi, + KnowledgeFSObjectHealthApi, + KnowledgeFSObjectListApi, + KnowledgeFSObjectMetadataApi, + KnowledgeFSObjectStorageHttpError, +) +from services.knowledge_fs.object_storage import ( + KnowledgeFSObjectList, + KnowledgeFSObjectMetadata, + KnowledgeFSObjectStorageChecksumError, + KnowledgeFSObjectStorageUnavailableError, +) + + +@pytest.fixture +def object_metadata() -> KnowledgeFSObjectMetadata: + return KnowledgeFSObjectMetadata( + checksum_sha256_base64="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + content_type="text/plain", + key="tenant-1/spaces/space-1/file.txt", + metadata={"tenantId": "tenant-1"}, + size_bytes=4, + ) + + +def _metadata_header(metadata: dict[str, str]) -> str: + return urlsafe_b64encode(json.dumps(metadata).encode()).decode().rstrip("=") + + +@patch("controllers.inner_api.knowledge_fs.storage.KnowledgeFSObjectStorageService") +def test_put_decodes_portable_metadata_and_returns_camel_case( + service_cls: MagicMock, + app: Flask, + object_metadata: KnowledgeFSObjectMetadata, +) -> None: + service_cls.return_value.put_object.return_value = object_metadata + handler = KnowledgeFSObjectApi() + + with app.test_request_context( + f"/?key={object_metadata.key}", + method="PUT", + data=b"data", + headers={ + "Content-Type": "application/octet-stream", + "X-Knowledge-FS-Checksum-Sha256": object_metadata.checksum_sha256_base64, + "X-Knowledge-FS-Content-Type": "text/plain", + "X-Knowledge-FS-Metadata": _metadata_header({"tenantId": "tenant-1"}), + }, + ): + result = inspect.unwrap(handler.put)(handler) + + assert result == { + "checksumSha256Base64": object_metadata.checksum_sha256_base64, + "contentType": "text/plain", + "key": object_metadata.key, + "metadata": {"tenantId": "tenant-1"}, + "sizeBytes": 4, + } + service_cls.return_value.put_object.assert_called_once_with( + body=b"data", + checksum_sha256_base64=object_metadata.checksum_sha256_base64, + content_type="text/plain", + key=object_metadata.key, + metadata={"tenantId": "tenant-1"}, + ) + + +@patch("controllers.inner_api.knowledge_fs.storage.KnowledgeFSObjectStorageService") +def test_get_streams_object_with_portable_headers( + service_cls: MagicMock, + app: Flask, + object_metadata: KnowledgeFSObjectMetadata, +) -> None: + service_cls.return_value.head_object.return_value = object_metadata + service_cls.return_value.load_stream.return_value = iter((b"da", b"ta")) + handler = KnowledgeFSObjectApi() + + with app.test_request_context(f"/?key={object_metadata.key}"): + result = inspect.unwrap(handler.get)(handler) + assert isinstance(result, Response) + assert result.get_data() == b"data" + + assert result.content_length == 4 + assert result.content_type == "text/plain" + assert result.headers["X-Knowledge-FS-Checksum-Sha256"] == object_metadata.checksum_sha256_base64 + + +@patch("controllers.inner_api.knowledge_fs.storage.KnowledgeFSObjectStorageService") +def test_head_list_delete_and_health_handlers( + service_cls: MagicMock, + app: Flask, + object_metadata: KnowledgeFSObjectMetadata, +) -> None: + service = service_cls.return_value + service.head_object.return_value = object_metadata + service.list_objects.return_value = KnowledgeFSObjectList( + objects=(object_metadata,), + next_cursor=object_metadata.key, + ) + service.health.return_value = True + + with app.test_request_context(f"/?key={object_metadata.key}"): + metadata_result = inspect.unwrap(KnowledgeFSObjectMetadataApi().get)(KnowledgeFSObjectMetadataApi()) + with app.test_request_context(f"/?prefix=tenant-1/spaces/&cursor={object_metadata.key}&limit=1"): + list_result = inspect.unwrap(KnowledgeFSObjectListApi().get)(KnowledgeFSObjectListApi()) + with app.test_request_context(f"/?key={object_metadata.key}", method="DELETE"): + delete_result = inspect.unwrap(KnowledgeFSObjectApi().delete)(KnowledgeFSObjectApi()) + with app.test_request_context(): + health_result = inspect.unwrap(KnowledgeFSObjectHealthApi().get)(KnowledgeFSObjectHealthApi()) + + assert metadata_result["key"] == object_metadata.key + assert list_result == { + "nextCursor": object_metadata.key, + "objects": [metadata_result], + } + assert delete_result == ("", 204) + assert health_result == {"ok": True} + service.delete_object.assert_called_once_with(key=object_metadata.key) + service.list_objects.assert_called_once_with( + cursor=object_metadata.key, + limit=1, + prefix="tenant-1/spaces/", + ) + + +@patch("controllers.inner_api.knowledge_fs.storage.KnowledgeFSObjectStorageService") +def test_missing_object_returns_404(service_cls: MagicMock, app: Flask) -> None: + service_cls.return_value.head_object.return_value = None + handler = KnowledgeFSObjectMetadataApi() + + with app.test_request_context("/?key=tenant-1/missing"): + with pytest.raises(KnowledgeFSObjectStorageHttpError) as exc_info: + inspect.unwrap(handler.get)(handler) + + assert exc_info.value.code == 404 + + +@pytest.mark.parametrize( + ("error", "expected_status"), + [ + (KnowledgeFSObjectStorageChecksumError("mismatch"), 422), + (KnowledgeFSObjectStorageUnavailableError("unavailable"), 503), + ], +) +@patch("controllers.inner_api.knowledge_fs.storage.KnowledgeFSObjectStorageService") +def test_storage_errors_are_mapped_to_safe_http_statuses( + service_cls: MagicMock, + error: Exception, + expected_status: int, + app: Flask, +) -> None: + service_cls.return_value.put_object.side_effect = error + handler = KnowledgeFSObjectApi() + + with app.test_request_context( + "/?key=tenant-1/file.txt", + method="PUT", + data=b"data", + headers={"X-Knowledge-FS-Metadata": _metadata_header({})}, + ): + with pytest.raises(KnowledgeFSObjectStorageHttpError) as exc_info: + inspect.unwrap(handler.put)(handler) + + assert exc_info.value.code == expected_status diff --git a/api/tests/unit_tests/controllers/inner_api/test_auth_wraps.py b/api/tests/unit_tests/controllers/inner_api/test_auth_wraps.py index 324d66c0b64..92121ddda78 100644 --- a/api/tests/unit_tests/controllers/inner_api/test_auth_wraps.py +++ b/api/tests/unit_tests/controllers/inner_api/test_auth_wraps.py @@ -16,6 +16,7 @@ from controllers.inner_api.wraps import ( enterprise_inner_api_only, enterprise_inner_api_user_auth, inner_api_only, + knowledge_fs_inner_api_only, plugin_inner_api_only, ) from models.enums import EndUserType @@ -385,3 +386,30 @@ class TestPluginInnerApiOnly: with pytest.raises(HTTPException) as exc_info: protected_view() assert exc_info.value.code == 404 + + +class TestKnowledgeFSInnerApiOnly: + """KnowledgeFS uses the same trusted transport key without exposing plugin semantics.""" + + def test_should_allow_valid_shared_inner_key(self, app: Flask): + @knowledge_fs_inner_api_only + def protected_view(): + return "success" + + with app.test_request_context(headers={"X-Inner-Api-Key": "valid_plugin_key"}): + with patch.object(dify_config, "PLUGIN_DAEMON_KEY", "plugin_key"): + with patch.object(dify_config, "INNER_API_KEY_FOR_PLUGIN", "valid_plugin_key"): + assert protected_view() == "success" + + def test_should_hide_endpoint_for_invalid_key(self, app: Flask): + @knowledge_fs_inner_api_only + def protected_view(): + return "success" + + with app.test_request_context(headers={"X-Inner-Api-Key": "invalid"}): + with patch.object(dify_config, "PLUGIN_DAEMON_KEY", "plugin_key"): + with patch.object(dify_config, "INNER_API_KEY_FOR_PLUGIN", "valid_plugin_key"): + with pytest.raises(HTTPException) as exc_info: + protected_view() + + assert exc_info.value.code == 404 diff --git a/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py b/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py new file mode 100644 index 00000000000..997f39e8dd2 --- /dev/null +++ b/api/tests/unit_tests/extensions/storage/test_aws_s3_storage.py @@ -0,0 +1,65 @@ +"""Unit tests for the common AWS S3 storage adapter.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from extensions.storage.aws_s3_storage import AwsS3Storage + + +@pytest.fixture +def s3_storage() -> tuple[AwsS3Storage, MagicMock]: + client = MagicMock() + client.head_bucket.return_value = {} + with ( + patch("extensions.storage.aws_s3_storage.boto3.client", return_value=client), + patch("extensions.storage.aws_s3_storage.dify_config.S3_USE_AWS_MANAGED_IAM", False), + patch("extensions.storage.aws_s3_storage.dify_config.S3_BUCKET_NAME", "dify-files"), + ): + storage = AwsS3Storage() + return storage, client + + +def test_scan_lists_recursive_files_and_derived_directories( + s3_storage: tuple[AwsS3Storage, MagicMock], +) -> None: + storage, client = s3_storage + paginator = client.get_paginator.return_value + paginator.paginate.return_value = [ + { + "Contents": [ + {"Key": "knowledge-fs/objects/tenant-1/a.txt"}, + {"Key": "knowledge-fs/objects/tenant-1/nested/b.txt"}, + {"Key": "knowledge-fs/objects/tenant-1/empty/"}, + ] + }, + {"Contents": [{"Key": "knowledge-fs/objects/tenant-1/nested/c.txt"}]}, + ] + + result = storage.scan( + "knowledge-fs/objects/tenant-1", + files=True, + directories=True, + ) + + assert result == [ + "knowledge-fs/objects/tenant-1/a.txt", + "knowledge-fs/objects/tenant-1/empty/", + "knowledge-fs/objects/tenant-1/nested/", + "knowledge-fs/objects/tenant-1/nested/b.txt", + "knowledge-fs/objects/tenant-1/nested/c.txt", + ] + client.get_paginator.assert_called_once_with("list_objects_v2") + paginator.paginate.assert_called_once_with( + Bucket="dify-files", + Prefix="knowledge-fs/objects/tenant-1/", + ) + + +def test_scan_rejects_request_without_files_or_directories( + s3_storage: tuple[AwsS3Storage, MagicMock], +) -> None: + storage, _ = s3_storage + + with pytest.raises(ValueError, match="At least one"): + storage.scan("knowledge-fs", files=False, directories=False) diff --git a/api/tests/unit_tests/services/test_knowledge_fs_object_storage.py b/api/tests/unit_tests/services/test_knowledge_fs_object_storage.py new file mode 100644 index 00000000000..ed93a98cbc4 --- /dev/null +++ b/api/tests/unit_tests/services/test_knowledge_fs_object_storage.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from base64 import b64encode +from collections.abc import Generator +from hashlib import sha256 +from pathlib import Path + +import pytest + +from extensions.storage.opendal_storage import OpenDALStorage +from services.knowledge_fs.object_storage import ( + KnowledgeFSObjectStorageChecksumError, + KnowledgeFSObjectStorageInvalidInputError, + KnowledgeFSObjectStorageService, + KnowledgeFSObjectStorageUnavailableError, +) + + +class FakeStorage: + objects: dict[str, bytes] + scan_supported: bool + + def __init__(self, *, scan_supported: bool = True) -> None: + self.objects = {} + self.scan_supported = scan_supported + + def save(self, filename: str, data: bytes) -> None: + self.objects[filename] = bytes(data) + + def load_once(self, filename: str) -> bytes: + try: + return self.objects[filename] + except KeyError as exc: + raise FileNotFoundError("missing") from exc + + def load_stream(self, filename: str) -> Generator[bytes, None, None]: + yield self.load_once(filename) + + def exists(self, filename: str) -> bool: + return filename in self.objects + + def delete(self, filename: str) -> None: + self.objects.pop(filename, None) + + def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]: + if not self.scan_supported: + raise NotImplementedError("scan unsupported") + return sorted(key for key in self.objects if key.startswith(path)) + + +def test_round_trips_metadata_streams_lists_and_deletes_objects() -> None: + backend = FakeStorage() + service = KnowledgeFSObjectStorageService(backend=backend) + body = b"knowledge-fs" + checksum = b64encode(sha256(body).digest()).decode() + + stored = service.put_object( + key="tenant-1/spaces/space-1/documents/doc-1/file.txt", + body=body, + checksum_sha256_base64=checksum, + content_type="text/plain", + metadata={"assetId": "doc-1", "tenantId": "tenant-1"}, + ) + + assert stored.key == "tenant-1/spaces/space-1/documents/doc-1/file.txt" + assert stored.size_bytes == len(body) + assert stored.checksum_sha256_base64 == checksum + assert stored.content_type == "text/plain" + assert stored.metadata == {"assetId": "doc-1", "tenantId": "tenant-1"} + assert service.head_object(key=stored.key) == stored + assert b"".join(service.load_stream(key=stored.key) or ()) == body + + listed = service.list_objects(prefix="tenant-1/spaces/space-1/", limit=10) + assert listed.objects == (stored,) + assert listed.next_cursor is None + + service.delete_object(key=stored.key) + assert service.head_object(key=stored.key) is None + assert service.load_stream(key=stored.key) is None + + +def test_list_uses_lexical_cursor_without_exposing_physical_storage_paths() -> None: + service = KnowledgeFSObjectStorageService(backend=FakeStorage()) + for name in ("a.txt", "b.txt", "c.txt"): + service.put_object( + key=f"tenant-1/spaces/space-1/{name}", + body=name.encode(), + content_type="text/plain", + metadata={}, + ) + + first = service.list_objects(prefix="tenant-1/spaces/space-1/", limit=2) + assert [item.key for item in first.objects] == [ + "tenant-1/spaces/space-1/a.txt", + "tenant-1/spaces/space-1/b.txt", + ] + assert first.next_cursor == "tenant-1/spaces/space-1/b.txt" + + second = service.list_objects( + prefix="tenant-1/spaces/space-1/", + cursor=first.next_cursor, + limit=2, + ) + assert [item.key for item in second.objects] == ["tenant-1/spaces/space-1/c.txt"] + assert second.next_cursor is None + + +def test_rejects_checksum_mismatch_and_path_traversal() -> None: + service = KnowledgeFSObjectStorageService(backend=FakeStorage()) + + with pytest.raises(KnowledgeFSObjectStorageChecksumError): + service.put_object( + key="tenant-1/file.txt", + body=b"body", + checksum_sha256_base64=b64encode(bytes(32)).decode(), + metadata={}, + ) + + with pytest.raises(KnowledgeFSObjectStorageInvalidInputError): + service.put_object(key="../dify-secret", body=b"body", metadata={}) + + +def test_fails_health_and_listing_when_unified_backend_cannot_scan() -> None: + service = KnowledgeFSObjectStorageService(backend=FakeStorage(scan_supported=False)) + + assert service.health() is False + with pytest.raises(KnowledgeFSObjectStorageUnavailableError): + service.list_objects(prefix="tenant-1/", limit=10) + + +def test_round_trips_through_dify_default_opendal_filesystem(tmp_path: Path) -> None: + service = KnowledgeFSObjectStorageService(backend=OpenDALStorage(scheme="fs", root=str(tmp_path))) + + stored = service.put_object( + key="tenant-1/spaces/space-1/file.txt", + body=b"data", + metadata={"tenantId": "tenant-1"}, + ) + + assert service.head_object(key=stored.key) == stored + assert service.list_objects(prefix="tenant-1/spaces/space-1/", limit=10).objects == (stored,) + assert service.health() is True diff --git a/docker/README.md b/docker/README.md index 48beb6bfa86..6b921e31bb8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -38,27 +38,41 @@ Welcome to the new `docker` directory for deploying Dify using Docker Compose. T - Copy `envs/core-services/shared.env.example` to `envs/core-services/shared.env`. - Set `ENABLE_OTEL=true` and configure `OTLP_BASE_ENDPOINT`. Tune the other `OTEL_*` knobs in the same file if needed. -### KnowledgeFS Integration Baseline +### KnowledgeFS Integration Service -The optional `knowledge-fs` profile adds an internal KnowledgeFS API service for deployment -validation. It is disabled during the normal Dify startup, publishes no host port, has no nginx -route, and reuses the existing `plugin_daemon` service and key on the default Compose network. -`KNOWLEDGE_FS_ENABLED` remains `false`, so this profile does not enable product traffic. +The default deployment starts an internal KnowledgeFS API service. It publishes no host port and +has no nginx route. Model, datasource, and object-storage calls go through Dify's inner API. +Dify resolves active Workspace/plugin configuration and remains the only owner of physical +storage credentials. `KNOWLEDGE_FS_ENABLED` remains `false`, so starting the container does not +enable product traffic. -Copy and fill the dedicated service configuration only when validating this baseline: +Copy and fill the dedicated service configuration before enabling KnowledgeFS traffic. The default +image is the CI-published deployment-branch image; set `KNOWLEDGE_FS_API_IMAGE` to pin another tag +or immutable SHA tag. If the image repository is private, authenticate the deployment host with +`docker login` before pulling: ```bash cp envs/core-services/knowledge-fs.env.example envs/core-services/knowledge-fs.env -docker compose --profile knowledge-fs config -docker compose --profile knowledge-fs build knowledge_fs +docker compose config +docker compose pull knowledge_fs +docker compose up -d ``` -Use a dedicated KnowledgeFS database and object-storage bucket. Do not point `DATABASE_URL` at -Dify's application database, reuse Dataset/Document tables, or run a data migration as part of -this profile. KnowledgeFS migrations remain a separate controlled operator step. +The service env file intentionally contains only operator-owned inputs: the dedicated database, +durable compilation switch, Capability v2 public verification material, and optional Unstructured +endpoint. Compose injects the Dify inner API connection and integrated mode. KnowledgeFS stores +objects below an internal namespace in Dify's configured `STORAGE_TYPE`; do not duplicate Dify's +bucket, endpoint, or provider credentials in `knowledge-fs.env`. Feature-specific rollout flags +and capacity tunables should be added only when deliberately overriding their safe runtime +defaults. + +Use a dedicated KnowledgeFS database. Do not point `DATABASE_URL` at Dify's application database, +reuse Dataset/Document tables, or run a data migration as part of this service. KnowledgeFS +migrations remain a separate controlled operator step. The selected Dify storage backend must +support recursive `scan`; the currently verified paths are S3 and OpenDAL/local. The production entrypoint supports the explicitly selected Capability v2 verifier and accepts only -public JWKS material. A manually started `knowledge_fs` container can still return `200` from +public JWKS material. The `knowledge_fs` container can still return `200` from `/health` while `/ready` returns `503` when the verifier or another durable dependency is missing; this is intentional fail-closed behavior. Do not add a proxy route, set `KNOWLEDGE_FS_ENABLED=true`, or send product traffic until readiness returns `200` and the migration diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 26ba21df326..fbefceedbd0 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -634,14 +634,13 @@ services: condition: service_healthy required: false - # KnowledgeFS integration baseline. This service is profile-gated, has no public port or nginx - # route, and remains unready until a production auth verifier is assembled in a later phase. + # KnowledgeFS integration service. It starts with the default deployment, has no public port or + # nginx route, and remains fail-closed until its production dependencies are configured. knowledge_fs: build: context: ../knowledge-fs dockerfile: apps/api/Dockerfile - image: ${KNOWLEDGE_FS_API_IMAGE:-knowledge-fs-api:local} - profiles: ["knowledge-fs"] + image: ${KNOWLEDGE_FS_API_IMAGE:-langgenius/dify-knowledge-fs-api:deploy-konwledge} restart: always env_file: - path: ./envs/core-services/knowledge-fs.env diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 85d70922783..29829ea2379 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -640,14 +640,13 @@ services: condition: service_healthy required: false - # KnowledgeFS integration baseline. This service is profile-gated, has no public port or nginx - # route, and remains unready until a production auth verifier is assembled in a later phase. + # KnowledgeFS integration service. It starts with the default deployment, has no public port or + # nginx route, and remains fail-closed until its production dependencies are configured. knowledge_fs: build: context: ../knowledge-fs dockerfile: apps/api/Dockerfile - image: ${KNOWLEDGE_FS_API_IMAGE:-knowledge-fs-api:local} - profiles: ["knowledge-fs"] + image: ${KNOWLEDGE_FS_API_IMAGE:-langgenius/dify-knowledge-fs-api:deploy-konwledge} restart: always env_file: - path: ./envs/core-services/knowledge-fs.env diff --git a/docker/envs/core-services/knowledge-fs.env.example b/docker/envs/core-services/knowledge-fs.env.example index 0cd0cede156..d4f39d8f9da 100644 --- a/docker/envs/core-services/knowledge-fs.env.example +++ b/docker/envs/core-services/knowledge-fs.env.example @@ -1,68 +1,25 @@ # ------------------------------------------------------------------ -# KnowledgeFS integration baseline (optional `knowledge-fs` profile). -# Copy to knowledge-fs.env only when validating the internal service. -# The Dify product flag remains disabled and no public proxy route is added. +# KnowledgeFS integration service (started by the default Dify Compose deployment). +# Copy to knowledge-fs.env and configure durable dependencies before enabling product traffic. +# The Dify product flag remains disabled by default and no public proxy route is added. +# Dify inner API connection values and integrated mode are injected by docker-compose.yaml. +# Physical object storage is owned by Dify's configured STORAGE_TYPE and credentials; do not copy +# provider endpoints, bucket names, access keys, or secret keys into this service. +# Optional feature flags and capacity tunables use safe runtime defaults and belong here only when +# an operator intentionally enables or overrides that feature. # ------------------------------------------------------------------ -# Dedicated KnowledgeFS persistence. Do not point this at Dify's application database or reuse -# Dataset/Document tables. Apply KnowledgeFS migrations through its controlled migration runner. +# Required durable database. Do not point this at Dify's application database or reuse +# Dataset/Document tables. Apply KnowledgeFS migrations before starting product traffic. DATABASE_URL= -KNOWLEDGE_DATABASE_REPOSITORIES=on KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME=on -# KnowledgeFS model calls go to Dify's inner API. Dify then resolves the workspace's active -# provider/model credentials through ModelManager and invokes plugin-daemon internally. -DIFY_INNER_API_URL=http://api:5001 -DIFY_INNER_API_KEY= -DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES=8388608 -DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS=60000 -DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES=8388608 -DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS=60000 - -# Capability v2 stays unselected until a public-only current/previous JWKS is provided and -# deployment readiness has been verified. KnowledgeFS must never receive Dify's private key. +# Required before enabling integrated product traffic. Set ENABLED=true only after providing the +# public-only current/previous JWKS that matches Dify's signer. Never copy Dify's private key here. KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false KNOWLEDGE_FS_CAPABILITY_V2_PUBLIC_JWKS= -KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane -KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs -# These process-level flags only declare that the integrated runtime is deployed. Product traffic -# remains fail-closed until Dify's cutover handshake creates the KFS durable per-Workspace -# activation. The legacy flag is an emergency all-Workspace freeze, not the rollout allowlist. -KNOWLEDGE_INTEGRATED_MODE_ENABLED=false -KNOWLEDGE_LEGACY_ACL_READ_ONLY=false -# P9 final deployment profile. Keep false throughout P8 rollout and the P9 zero-traffic window. -# Set true only after every affected Workspace is ready for cleanup and the reviewed removal -# deployment is authorized; this permanently removes legacy API-key/ACL route registration. -KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED=false - -# Direct upload and Research streaming remain disabled until Capability v2, object-storage -# lifecycle, exact browser origins, and tenant smoke evidence are all ready. -KNOWLEDGE_DIRECT_UPLOAD_ENABLED=off -KNOWLEDGE_DIRECT_UPLOAD_ALLOWED_ORIGINS= -KNOWLEDGE_DIRECT_UPLOAD_MAX_FILE_BYTES=107374182400 -KNOWLEDGE_DIRECT_UPLOAD_MULTIPART_PART_BYTES=16777216 -KNOWLEDGE_DIRECT_UPLOAD_MULTIPART_THRESHOLD_BYTES=67108864 -KNOWLEDGE_DIRECT_UPLOAD_SMALL_FALLBACK_MAX_BYTES=8388608 -KNOWLEDGE_DIRECT_UPLOAD_PRESIGN_TTL_SECONDS=600 -KNOWLEDGE_DIRECT_UPLOAD_SESSION_TTL_MS=3600000 -KNOWLEDGE_DIRECT_UPLOAD_CLEANUP_INTERVAL_MS=60000 -KNOWLEDGE_DIRECT_UPLOAD_CLEANUP_BATCH_SIZE=100 -KNOWLEDGE_DIRECT_UPLOAD_CLEANUP_STALE_MS=300000 -KNOWLEDGE_DIRECT_UPLOAD_INCOMPLETE_MULTIPART_DAYS=2 -KNOWLEDGE_DIRECT_STREAM_ENABLED=off -KNOWLEDGE_DIRECT_STREAM_ALLOWED_ORIGINS= -KNOWLEDGE_DIRECT_STREAM_MAX_CONNECTION_MS=300000 - -# Dedicated S3-compatible object bucket. -MINIO_ENDPOINT= -MINIO_BUCKET=knowledge-fs -MINIO_ACCESS_KEY= -MINIO_SECRET_KEY= -MINIO_REGION=us-east-1 # Complex-document parser reachable from the default Compose network or an external endpoint. +# Leave both values blank only when PDF/Office parsing is intentionally unavailable. UNSTRUCTURED_API_URL= UNSTRUCTURED_API_KEY= - -# Destructive processing stays disabled in this deployment-only baseline. -DURABLE_DELETION_ENABLED=off diff --git a/knowledge-fs/.env.example b/knowledge-fs/.env.example deleted file mode 100644 index 722b4bcd79e..00000000000 --- a/knowledge-fs/.env.example +++ /dev/null @@ -1,88 +0,0 @@ -POSTGRES_DB=knowledge_fs -POSTGRES_PASSWORD=knowledge_fs -POSTGRES_PORT=5432 -POSTGRES_USER=knowledge_fs -DATABASE_URL=postgresql://knowledge_fs:knowledge_fs@127.0.0.1:5432/knowledge_fs -KNOWLEDGE_DATABASE_REPOSITORIES= -DURABLE_DELETION_ENABLED=off -DURABLE_DELETION_WRITER_FENCE_VERSION= -# Keep this stable for the lifetime of deletion idempotency ledgers; do not rotate silently. -DURABLE_DELETION_HMAC_KEY_BASE64= - -MINIO_ACCESS_KEY=knowledge -MINIO_API_PORT=9000 -MINIO_BUCKET=knowledge-fs -MINIO_CONSOLE_PORT=9001 -MINIO_ENDPOINT=http://127.0.0.1:9000 -MINIO_REGION=us-east-1 -MINIO_ROOT_PASSWORD=knowledge-secret -MINIO_ROOT_USER=knowledge -MINIO_SECRET_KEY=knowledge-secret - -R2_ACCESS_KEY_ID= -R2_ACCOUNT_ID= -R2_BUCKET= -R2_REGION=auto -R2_SECRET_ACCESS_KEY= - -UNSTRUCTURED_PORT=8000 -UNSTRUCTURED_API_URL=http://127.0.0.1:8000 -UNSTRUCTURED_API_KEY= -UNSTRUCTURED_MAX_RESPONSE_BYTES= -UNSTRUCTURED_MAX_RETRIES= -UNSTRUCTURED_RETRY_DELAY_MS= - -OPENAI_API_KEY= -ANTHROPIC_API_KEY= -GEMINI_API_KEY= -KNOWLEDGE_EMBEDDING_PROVIDER= -KNOWLEDGE_EMBEDDING_MODEL= -KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER= -KNOWLEDGE_ENTITY_EXTRACTION_MODEL= -KNOWLEDGE_ENTITY_EXTRACTION_MAX_ENTITIES_PER_NODE= -KNOWLEDGE_ENTITY_EXTRACTION_MAX_NODES_PER_RUN= -KNOWLEDGE_ENTITY_EXTRACTION_MAX_OUTPUT_TOKENS= -KNOWLEDGE_RELATION_EXTRACTION_MODEL= -KNOWLEDGE_RELATION_EXTRACTION_MAX_RELATIONS_PER_NODE= -KNOWLEDGE_RELATION_EXTRACTION_MAX_OUTPUT_TOKENS= -KNOWLEDGE_COMMUNITY_SUMMARY_MODEL= -KNOWLEDGE_COMMUNITY_SUMMARY_MAX_OUTPUT_TOKENS= -# Graph-expanded retrieval (deep/research modes). On by default when the graph -# repository is wired; set KNOWLEDGE_GRAPH_EXPANSION=off to disable. Tuning -# knobs fall back to built-in defaults (in parentheses) when unset. -KNOWLEDGE_GRAPH_EXPANSION= -KNOWLEDGE_GRAPH_EXPANSION_MAX_DEPTH= # traversal hops, 1-2 (2) -KNOWLEDGE_GRAPH_EXPANSION_FANOUT= # neighbors expanded per node (20) -KNOWLEDGE_GRAPH_EXPANSION_MAX_SEED_ENTITIES= # seeds taken from base hits (5) -KNOWLEDGE_GRAPH_EXPANSION_MAX_TRAVERSAL_NODES= # traversal node budget (50) -KNOWLEDGE_GRAPH_EXPANSION_GRAPH_TOP_K= # entity names used to re-retrieve (10) -KNOWLEDGE_GRAPH_EXPANSION_GRAPH_BOOST= # fusion weight of graph hits (0.2) -KNOWLEDGE_GRAPH_EXPANSION_TIMEOUT_MS= # traversal time budget (250) -# Scheduled source sync. Sources opt in via metadata.syncPolicy — -# {"everyHours": 6} or {"dailyAt": ["03:00"], "utcOffset": "+08:00"}. -# The scheduler is on by default (set KNOWLEDGE_SOURCE_SYNC=off to disable) and -# is multi-replica safe: per-source runs are serialized by an atomic DB claim. -KNOWLEDGE_SOURCE_SYNC= -KNOWLEDGE_SOURCE_SYNC_TICK_MS= # scheduler tick interval (60000) -KNOWLEDGE_SOURCE_SYNC_MAX_SOURCES_PER_TICK= # sources scanned per tick (200) -# Answer synthesis LLM (opt-in). Unset/off => extractive evidence answers. -# Set to openai|anthropic|gemini to let that provider write grounded answers. -KNOWLEDGE_ANSWER_PROVIDER= -KNOWLEDGE_ANSWER_MODEL= -KNOWLEDGE_ANSWER_MAX_OUTPUT_TOKENS= -# Gateway span tracing. off (default) | console (one JSON line per span) | -# otlp (OTLP/HTTP JSON to an OpenTelemetry collector). -KNOWLEDGE_TRACING= -KNOWLEDGE_TRACING_OTLP_ENDPOINT= # e.g. http://localhost:4318/v1/traces -KNOWLEDGE_TRACING_OTLP_HEADERS= # optional JSON object, e.g. {"authorization":"Bearer …"} -KNOWLEDGE_TRACING_SERVICE_NAME= # resource service.name (knowledge-fs-api) -KNOWLEDGE_TRACING_FLUSH_MS= # export batch interval (5000) - -API_PORT=8788 -ADMIN_PORT=3000 -KNOWLEDGE_API_BASE_URL=http://localhost:8788 -NEXT_PUBLIC_API_BASE_URL=http://localhost:8788 - -KNOWLEDGE_DEV_AUTH_TOKEN=dev-token -KNOWLEDGE_DEV_SUBJECT_ID=dev-user -KNOWLEDGE_DEV_TENANT_ID=tenant-dev diff --git a/knowledge-fs/README.md b/knowledge-fs/README.md index 06a58c1804f..c2c5edbecc5 100644 --- a/knowledge-fs/README.md +++ b/knowledge-fs/README.md @@ -1,531 +1,159 @@ # KnowledgeFS -KnowledgeFS is a TypeScript knowledge platform for retrieval-augmented systems. It combines a Hono Knowledge Gateway, a Next.js Admin Console, portable platform adapters, and bounded in-process compute primitives. +KnowledgeFS is Dify's backend knowledge runtime. It provides tenant-scoped ingestion, parsing, +indexing, retrieval, KnowledgeFS commands, MCP tools, durable jobs, traces, and evaluation APIs. -The project is intentionally built around two deployment targets: +KnowledgeFS is not an independently deployable product. It must run with the Dify API: -- SaaS target: Cloudflare Workers, R2, KV, and TiDB Cloud. -- Standalone target: Docker, Node.js, MinIO, Redis or in-memory cache, PostgreSQL with pgvector and full-text search. +- Dify owns model and datasource plugin credentials. +- Dify creates model and datasource plugin instances and performs plugin invocation. +- Dify owns physical object storage through its configured `STORAGE_TYPE`. +- KnowledgeFS reaches those capabilities only through the authenticated Dify inner API. +- `KNOWLEDGE_INTEGRATED_MODE_ENABLED` is a Workspace rollout/cutover gate; it never selects a + different runtime or credential owner. -The `.harness` directory is the project information base. It contains the architecture notes, iteration plan, agent development rules, temporary task/progress documents, and change records. - -## Current Capabilities - -- Hono API boundary with OpenAPI generation. -- Auth subject middleware for tenant-scoped business routes. -- KnowledgeSpace CRUD with in-memory and database-backed repository contracts. -- Document upload, object storage persistence, synchronous MVP parsing, parse artifact persistence, and read APIs. -- Native Markdown and HTML parsers plus Unstructured API client skeleton. -- Deterministic KnowledgeNode chunking through the shared TypeScript compute runtime. -- Dense vector and full-text projection contracts. -- Hybrid retrieval, reranking integration, metadata and permission filtering, EvidenceBundle assembly, and AnswerTrace recording. -- LLM provider abstraction, evidence-driven prompt packing, SSE streaming generation, citation normalization, generation cache, and skip path. -- KnowledgeFS path/resource model with `ls`, `tree`, `cat`, `stat`, `grep`, `find`, `diff`, and `open_node` command surfaces. -- MCP server skeleton and KnowledgeFS/retrieval/safe-shell tools. -- Rate limiting, degradation flags, component health reporting, and CI retrieval regression gate with recall, citation, faithfulness, and no-answer thresholds. -- Next.js Admin Console with upload health, retrieval preview, trace viewer, evaluation dashboard, Retrieval Studio, trace comparison, human annotation, and failed query diagnostics surfaces. - -## Architecture +## Runtime architecture ```text -apps/ - api/ Node standalone entrypoint for the Hono Knowledge Gateway - admin/ Next.js Admin Console and thin UI BFF - -packages/ - adapters/ Platform adapters for database, object storage, cache, and jobs - api/ Hono gateway, repositories, retrieval, KnowledgeFS, MCP, auth, traces - compute/ Bounded TypeScript compute: chunking, token counting, RRF, packing, diff - core/ Shared schemas, platform contracts, command registry models - database/ Schema catalog and checked-in SQL migration artifacts - dify-model-runtime-client/ - Bounded Dify inner-API client for tenant model catalog and invocation - embeddings/ Embedding and reranker providers plus version-aware cache wrappers - generation/ LLM providers, prompt packing, generation cache, streaming helpers - parsers/ Parser contracts, native parsers, Unstructured client, router - -infra/ - local/ Local Docker Compose stack: compose files, .env.example, pgvector init, services guide - aws_terraform/ Target AWS Standalone architecture (EC2 + Aurora Serverless v2 + S3); diagram only, no Terraform yet +Dify API + ├─ model manager / plugin daemon + ├─ datasource plugins + ├─ unified object storage + └─ authenticated inner API + │ + ▼ +KnowledgeFS API + ├─ document compilation and retrieval + ├─ KnowledgeFS / MCP command surfaces + ├─ PostgreSQL repositories and durable jobs + └─ optional Unstructured parser dependency ``` -TypeScript owns orchestration, IO, HTTP, MCP, database access, storage, cache, jobs, provider adapters, Admin UI, and all bounded compute primitives. +Main directories: -## Infrastructure - -The `infra/` directory holds everything needed to *run* KnowledgeFS, organized by deployment target: - -- **`infra/local/`** — the local Docker Compose stack used for development and CI. It contains `compose.yaml` (full stack: PostgreSQL + pgvector, MinIO, Unstructured, plus the API/Admin app containers behind the `apps` profile), `compose.middleware.yaml` (middleware only: Postgres, MinIO, bucket bootstrap, Unstructured), `.env.example` (copy to `infra/local/.env`), `postgres-init/01-enable-pgvector.sql` (auto-enables the `vector` extension on a fresh data volume), and a [services guide](infra/local/README.md). The `pnpm dev:*`, `compose:*`, and `local:db:migrate` scripts all target these files. -- **`infra/aws_terraform/`** — the target **AWS Standalone** deployment: a single EC2 host running the `api` + `unstructured` containers, Aurora Serverless v2 PostgreSQL + pgvector, and AWS S3 for object storage (the API authenticates to S3 via an EC2 IAM instance role). Currently the [target architecture diagram and component/env mapping](infra/aws_terraform/README.md) only — the Terraform modules and deployment runbook are not written yet. - -## Prerequisites - -- Node.js 22 or newer. -- pnpm 10.33.0 through Corepack. -- Docker Desktop or compatible Docker Compose runtime. - -Recommended setup: - -```bash -corepack enable -pnpm install +```text +apps/api/ KnowledgeFS backend entrypoint +packages/api/ Hono gateway, repositories, retrieval, jobs, auth +packages/adapters/ Database, Dify storage, cache, and queue adapters +packages/dify-model-runtime-client/ Bounded Dify model inner-API client +packages/dify-datasource-runtime-client/ + Bounded Dify datasource inner-API client +packages/core/ Shared contracts and schemas +packages/database/ Schema catalog and SQL migrations +packages/compute/ Bounded pure TypeScript compute +packages/parsers/ Native and Unstructured parser adapters +infra/local/ Developer harness; requires a running Dify API +infra/kubernetes/ Inert Dify integration baseline ``` -## Environment +The repository still contains reusable lower-level adapters and an optional local Admin test +harness. They are development assets, not alternative production deployment modes. -Copy local Compose defaults before running infrastructure: +## Required production configuration -```bash -cp infra/local/.env.example infra/local/.env -``` +The canonical Dify Compose service loads +`docker/envs/core-services/knowledge-fs.env.example`. Operator-owned inputs are limited to: -Important local variables: +- `DATABASE_URL` +- `DIFY_INNER_API_URL` and `DIFY_INNER_API_KEY`, injected by Dify Compose +- `UNSTRUCTURED_API_URL` and optional `UNSTRUCTURED_API_KEY` +- KnowledgeFS capability/JWKS and document-compilation rollout settings -- `API_PORT`: Hono API port, default `8788` for local source and host access. The API container still listens on `8787` internally. -- `ADMIN_PORT`: Admin Console port, default `3000`. -- `POSTGRES_*`: local PostgreSQL credentials and port. -- `DATABASE_URL`: optional source-run PostgreSQL connection string. When set, the Node adapter uses a pool-backed PostgreSQL executor and the API app uses database-backed core repositories. -- `KNOWLEDGE_DATABASE_REPOSITORIES`: set to `off`, `false`, or `0` to keep bounded memory repositories even when `DATABASE_URL` is present. -- Durable deletion is deliberately deployment-gated. Set `DURABLE_DELETION_ENABLED=true` only - after every running writer understands migration `0017` tombstones, and declare - `DURABLE_DELETION_WRITER_FENCE_VERSION=0017`. A stable, canonical base64 key of at least 32 bytes - is then required in `DURABLE_DELETION_HMAC_KEY_BASE64`. Keep that key unchanged while deletion - jobs or idempotency ledgers are retained; silent rotation makes old request fingerprints - unverifiable. With the gate off, destructive routes remain unavailable. -- `MINIO_*`: local MinIO credentials, bucket, API port, and console port. -- `UNSTRUCTURED_PORT`: local Unstructured API port. -- `UNSTRUCTURED_API_URL`: Unstructured base URL or full partition endpoint used by the source-run API for PDF, Word, PowerPoint, and other complex document parsing. -- `DIFY_INNER_API_URL`, `DIFY_INNER_API_KEY`: Dify API inner boundary used for embedding, - rerank, LLM, multimodal embedding, model catalog, and datasource calls in integrated mode. The - key must match Dify's `INNER_API_KEY_FOR_PLUGIN`. KnowledgeFS sends model routing identity or a - datasource `credentialId`; Dify resolves credential bytes and invokes plugin-daemon. -- `PLUGIN_DAEMON_URL`, `PLUGIN_DAEMON_KEY`: legacy direct datasource transport used only by the - standalone profile (`KNOWLEDGE_INTEGRATED_MODE_ENABLED` is not `true`). -- `R2_*`: optional Cloudflare R2-compatible storage configuration. +Do not configure storage-provider credentials, model-provider keys, datasource credentials, or a +direct Plugin Daemon endpoint in KnowledgeFS. The Dify inner key must match +`INNER_API_KEY_FOR_PLUGIN`. -Without database or object-storage runtime configuration, local gateway paths use bounded in-memory fallbacks. With `DATABASE_URL` set, the Node adapter executes parameterized PostgreSQL queries and the API app persists core workspace, document, artifact, node, and projection records through database-backed repositories. With MinIO variables present, it can use S3-compatible object storage. +See [production deployment](docs/production-deployment.md) and the +[operator manual](docs/operator-manual.md). -## Local Development +## Development + +Prerequisites: + +- Node.js 22+ +- pnpm 10.33.0 through Corepack +- Docker +- A reachable Dify API Install dependencies: ```bash +corepack enable pnpm install +cp infra/local/.env.example infra/local/.env ``` -Start infrastructure only: +Set `DIFY_INNER_API_URL` and `DIFY_INNER_API_KEY` in the ignored local env, then start the local +database and parser: ```bash pnpm dev:infra -``` - -This uses `infra/local/compose.middleware.yaml` and starts only PostgreSQL, MinIO, a one-shot MinIO bucket bootstrap service, and Unstructured. It does not start the API or Admin containers, so you can run `apps/api` and `apps/admin` from your local source tree. - -Apply PostgreSQL migrations when using `DATABASE_URL` with the source-run API: - -```bash pnpm local:db:migrate ``` -Start the full local Compose stack: - -```bash -pnpm dev:stack -``` - -This starts infrastructure plus the API and production Admin app containers. The local API explicitly runs with -`NODE_ENV=development` so its static token verifier cannot be confused with a production verifier. The API listens -on `http://localhost:8788` by default; the Admin Console listens on `http://localhost:3000`. - -Run the API directly from the workspace: +Run the backend from source: ```bash pnpm dev:api ``` -The source API dev script loads `infra/local/.env` automatically, so `DATABASE_URL`, MinIO, and local auth settings match the migration command. -If you already have a local `infra/local/.env`, make sure `API_PORT`, -`KNOWLEDGE_API_BASE_URL`, and `NEXT_PUBLIC_API_BASE_URL` point at the same -host-visible API port. - -If the Admin Console reports every health card as unavailable, or upload fails -with `Knowledge API upload route was not found`, first confirm that the API base -points at the KnowledgeFS API and not another local service: - -```bash -curl http://localhost:${API_PORT:-8788}/health -``` - -The response should be a KnowledgeFS health payload with `runtime` and -`components`. If another service answers on that port, run the API on a free port -and point the Admin BFF at the same base: - -```bash -API_PORT=8790 pnpm dev:api -KNOWLEDGE_API_BASE_URL=http://localhost:8790 NEXT_PUBLIC_API_BASE_URL=http://localhost:8790 PORT=3000 pnpm --filter @knowledge/admin dev -``` - -Run the Admin Console directly: +For the optional local Admin test harness: ```bash pnpm --filter @knowledge/admin dev ``` -The Admin dev server should use values from `infra/local/.env`. Restart it after changing `KNOWLEDGE_API_BASE_URL` or `NEXT_PUBLIC_API_BASE_URL`; -Next keeps those values from process startup. - -Browse the gateway's OpenAPI document in Swagger UI: - -```bash -pnpm dev:api # gateway must be running (local source default :8788) -pnpm swagger # serves Swagger UI on http://localhost:8088 -``` - -`pnpm swagger` runs a small dependency-free Node reverse proxy (`tools/swagger/`) -that serves a Swagger UI shell and forwards `/openapi.json` and "Try it out" -requests to the gateway from the same origin, which avoids the gateway's lack of -CORS headers. It targets `http://localhost:8788` by default; override the target -or port with `KFS_API` and `KFS_SWAGGER_PORT`. See -[`tools/swagger/README.md`](tools/swagger/README.md) for details. - -Run the local source happy-path smoke from another terminal after `pnpm dev:infra`, -`pnpm dev:api`, and the Admin dev server are available: +Run the bounded local smoke after Dify and the local processes are available: ```bash pnpm local:happy-path ``` -The smoke validates the middleware Compose config, builds the Admin app, checks API -health, checks Admin BFF health, bootstraps the `workspace` KnowledgeSpace if -needed, uploads a Markdown document through the Admin BFF proxy, reads the -`DocumentAsset`, reads parse artifact version 1, and runs a bounded query evidence -check against the uploaded content without manual database edits. To -point the smoke at a non-default Admin server, set `LOCAL_SMOKE_ADMIN_BASE`. -To skip the Admin build during rapid local iteration, run -`LOCAL_SMOKE_SKIP_ADMIN_BUILD=1 pnpm local:happy-path`. -To include the checked-in PostgreSQL migrations in the same smoke run, run -`LOCAL_SMOKE_RUN_MIGRATIONS=1 pnpm local:happy-path`. -To require the durable local setup explicitly, run `pnpm local:happy-path:durable`. -That command applies migrations, requires `DATABASE_URL` plus MinIO env, and fails if -database or object storage health is not green. -To validate only the API source process without requiring the Admin dev server, run -`pnpm local:happy-path:api`. - -## Admin Console Guide - -The Admin Console at `http://localhost:3000` is both an operator console and a -retrieval quality workbench. Most sidebar entries jump to panels on the main -page; `Documents` opens a dedicated document list for the selected -KnowledgeSpace. - -Use `System health` first when the console looks empty or unavailable. It shows -whether the Admin app can reach the Knowledge Gateway and whether API components -are healthy. If all cards are unavailable, verify `KNOWLEDGE_API_BASE_URL`, -`NEXT_PUBLIC_API_BASE_URL`, the Admin token, and the API `/health` response. - -Use `Control plane` to inspect the selected KnowledgeSpace's bounded operational -state: manifest version, storage provider, object key prefix, parser policy, -projection set, document count, raw document bytes, and active sessions. This is -the first place to check when a space appears to use the wrong storage, parser, -or projection version. - -Use `FSCK` for read-only consistency diagnostics. The panel runs a bounded dry -run and summarizes scanned items, errors, warnings, repairable issues, and the -first page of findings. It is useful when uploaded documents, raw objects, -parse artifacts, paths, or references appear out of sync. - -Use `GC` to review staged-object cleanup candidates. The panel starts with a -dry run and shows candidate type, reason, estimated bytes, and dry-run id. Only -execute `Delete candidate` for candidates returned by the dry run and after -confirming the failed staged commit or abandoned object no longer needs recovery. - -Use `Upload intake` to upload a single document into a KnowledgeSpace. Choose the -space, optionally provide a `sourceId`, choose a Markdown, HTML, PDF, Word, -PowerPoint, or text file, then submit `Upload document`. After upload, use the -result links or `Documents` to inspect the document status and parse artifact. - -Use `Retrieval workspace` to run a live query. Enter a question, select `fast`, -`deep`, or `research`, then submit `Run query`. The response shows the generated -answer, inline citations, confidence, freshness, and trace id. That trace id -feeds the retrieval, trace review, failed diagnostics, and comparison panels. - -Use `KnowledgeFS` to browse the virtual filesystem. Enter a path such as -`/knowledge/by-topic`, `/knowledge/by-entity`, `/knowledge/by-community`, or -`/knowledge/by-type`, then submit `Browse path`. This view shows how published -knowledge is organized for human, API, and agent reads. - -Use `Documents` to open the document asset list for the active KnowledgeSpace. -The list reads the document asset API directly, so it does not depend on a -KnowledgeFS path view being published. Open a document to view parser status, -object key, size, version, and the parse artifact for version 1. - -Use `Entity browser` to inspect graph relationships around one entity. Enter an -entity id and submit `Traverse graph`. The panel shows related entities, -relation edges, traversal depth, fanout, and confidence. It is useful for -debugging entity extraction and graph-expanded retrieval. - -Use `Semantic views` to inspect live topic, entity, and community views exposed -through KnowledgeFS. The topic view maps to `/knowledge/by-topic`; the entity -view maps to `/knowledge/by-entity`; and the community view maps to -`/knowledge/by-community`. The Admin panel renders these as topic groups, -readable entities, and knowledge communities with short summaries instead of raw -graph ids. When LLM semantic extraction is configured, upload/compilation runs -entity extraction, relation extraction, graph indexing, community -materialization, and community summary generation as one post-processing flow. -Community materialization uses explicit graph relations when available, falls -back to entity co-occurrence only for disconnected extracted entities, links the -source documents under each community, and stores an LLM summary on the -community path. - -Topic entries appear after topic-view materialization. Entity and community -entries can be populated automatically during synchronous compute-backed -ingestion or durable compilation workers when semantic entity extraction is -configured; otherwise click `Extract entities` to backfill extracted node -entities and refresh `/knowledge/by-entity`, then click -`Materialize communities` to publish `/knowledge/by-community`. If the panel -says `Not materialized`, click `Materialize topic view` to publish uploaded -documents under `/knowledge/by-topic/uploaded-documents`. In the Node API app -entity and relation extraction use LLM-backed extraction when -`OPENAI_API_KEY` or `ANTHROPIC_API_KEY` is configured; set -`KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER=openai|anthropic` and -`KNOWLEDGE_ENTITY_EXTRACTION_MODEL` to pin the provider/model, and -`KNOWLEDGE_ENTITY_EXTRACTION_MAX_NODES_PER_RUN` to bound each extraction pass. -Set `KNOWLEDGE_RELATION_EXTRACTION_MODEL` or `KNOWLEDGE_COMMUNITY_SUMMARY_MODEL` -when relation extraction or community summaries should use a different model. -Without an LLM provider, semantic extraction is disabled instead of falling back -to noisy regex extraction. Use `Documents` and `KnowledgeFS` to inspect uploaded -content before or after these operator actions. - -Use `Document diff` to compare two KnowledgeFS paths. Provide an old path and a -new path, then submit `Run diff`. The panel shows text-level differences and, -when the semantic diff provider is configured, citation-ready semantic change -summaries. - -Use `Golden questions` to manage the evaluation set. Create questions with -expected evidence ids and tags, update or delete existing questions by id, and -record human annotations for answer correctness and evidence relevance. This -data drives regression evaluation and bad-case review. - -Use `Evaluation dashboard` to monitor quality review progress. It summarizes -the golden question count, annotated items, production bad cases, and pending -queue. Use `Production bad-case capture` with a trace id, reason, and tags to -turn a real failed answer into an evaluation item. - -Use `Retrieval Studio` after running a query. It shows the latest trace id, -query mode, query text, and evidence entries used by the retrieval pipeline. It -is the quickest way to inspect which evidence supported an answer. - -Use `Trace comparison` to compare two query traces side by side. Run one query -or provide a current `traceId`, enter another trace id in `Compare trace ID`, -then submit `Compare traces`. This is useful when comparing retrieval strategy, -parser, projection, or prompt changes. - -Use `Failed diagnostics` after a query produces a poor result. The panel shows -candidate ranking plus missing or conflicting evidence entries when available. -Use it to determine whether the issue came from recall, filtering, ranking, -conflicting evidence, or missing evidence. - -Use `Trace review` to inspect the step-by-step execution path for a query trace: -recall, reranking, evidence selection, generation, and any skipped or failed -steps. This is the most detailed view for debugging one answer. - -The usual local workflow is: check `System health` and `Control plane`, upload a -document through `Upload intake`, confirm it in `Documents`, browse organization -through `KnowledgeFS` or `Semantic views`, run a query in `Retrieval workspace`, -inspect evidence in `Retrieval Studio` and `Trace review`, then capture bad -answers into `Golden questions` and the `Evaluation dashboard`. - -Validate the middleware-only Compose file: +It validates health, workspace bootstrap, Markdown upload, parse artifacts, and query evidence. +Use `LOCAL_SMOKE_ADMIN_BASE` when the Admin harness is not on its default port. Other useful forms: ```bash -pnpm compose:middleware:config +LOCAL_SMOKE_RUN_MIGRATIONS=1 pnpm local:happy-path +pnpm local:happy-path:durable +pnpm local:happy-path:api +``` + +The durable smoke requires database health and Dify-backed object-storage health. The API-only +smoke skips the Admin BFF. + +See [the local developer guide](infra/local/README.md) for details. + +## Validation + +```bash +pnpm typecheck +pnpm test +pnpm lint:backend +pnpm openapi:export:test +pnpm db:migrations:check pnpm compose:middleware:test -``` - -Validate the full app profile contract without starting services: - -```bash pnpm compose:apps:test -``` - -Validate Compose configuration without starting services: - -```bash -pnpm compose:config -docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml --profile apps config -``` - -## Verification - -The main local gate is: - -```bash -pnpm check -``` - -It runs: - -- TypeScript typechecking. -- Unit tests. -- Coverage gates. -- Retrieval regression evaluation. -- Database migration drift check. - -Additional full verification used before implementation commits: - -```bash -pnpm build -pnpm lint -pnpm compose:apps:test -pnpm compose:config -docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml --profile apps config -pnpm docker:api:build -pnpm docker:api:bundle-smoke +pnpm dify:compose:config git diff --check ``` -Run the live MinIO smoke separately when local containers are available: - -```bash -docker compose --env-file infra/local/.env -f infra/local/compose.yaml up -d minio minio-bootstrap -pnpm test:minio -``` - -The live MinIO smoke is intentionally not part of `pnpm check` so CI and daily development do not require long-running containers. - -## Database Migrations - -The schema catalog in `packages/database` is the source for generated SQL artifacts. - -Generate checked-in migration artifacts: - -```bash -pnpm db:migrations:write -``` - -Check for drift: - -```bash -pnpm db:migrations:check -``` - -Generated migrations live in `packages/database/migrations` and currently include PostgreSQL and TiDB initial schema artifacts. - -## TypeScript Compute - -Bounded pure compute lives in `packages/compute`. It provides deterministic document chunking, -approximate token counting, reciprocal-rank fusion, evidence packing, and line/word text diff. -The API imports this package directly; there is no generated runtime artifact or separate build -step. - -Build the production app images directly: +Build the backend production bundle or image: ```bash +pnpm --filter @knowledge/api-app build:prod pnpm docker:api:build -pnpm docker:admin:build +pnpm docker:api:bundle-smoke ``` -Run `pnpm docker:api:bundle-smoke` to start the built API image in an isolated -`NODE_ENV=test` process and verify that the standalone Hono bundle responds on `/health` with -`components.compute === true`. This is a bundle/startup check only: it does not validate -production fail-closed configuration, database repositories, durable compilation, object storage, -or external providers. `pnpm docker:api:http-smoke` remains a compatibility alias for this isolated -check. -Run `pnpm docker:admin:http-smoke` to start the production Admin image and verify -the Next.js standalone homepage renders. -Run `pnpm docker:apps:smoke` when you want one command to build both app images, run the isolated -API bundle check, and run the Admin image homepage check. Use a deployed or Compose-backed health, -upload, and query flow to validate production runtime configuration. +The isolated image smoke proves the bundle can boot and remains unhealthy while Dify is absent; a +Dify Compose/Kubernetes smoke is required to validate the real inner API, storage, database, +models, and datasources. -## API And Auth Notes +## API and design references -- `/health` and `/openapi.json` are public. -- Business routes are protected by Bearer-token auth. -- Auth subject data is server-derived and includes `subjectId`, `tenantId`, and `scopes`. -- KnowledgeSpace, document, parse artifact, retrieval, and KnowledgeFS operations are tenant scoped. -- Cross-tenant resource access returns not found semantics where appropriate. - -Read scopes: - -- `knowledge-spaces:read` -- `knowledge-spaces:*` - -Write scopes: - -- `knowledge-spaces:write` -- `knowledge-spaces:*` - -## Performance And Safety Rules - -Project development treats performance bugs as correctness bugs. In particular: - -- No unbounded list, dequeue, file read, stream read, or cache entry. -- No N+1 database access on hot paths. -- All database reads require explicit row limits. -- Tenant and permission dimensions must be part of data access and cache boundaries. -- Cache keys must include model, strategy, permission, and index versions where relevant. -- User input must be parameterized in SQL and never interpolated into query strings. -- Object and cache values must use clone/copy semantics to avoid internal state leaks. -- Streaming and provider responses must be bounded by byte limits. - -## CI - -GitHub Actions runs on pushes to `main` and pull requests. The workflow performs: - -- Dependency install with frozen lockfile. -- `pnpm check`. -- Explicit retrieval regression evaluation. -- Build. -- Lint. -- Compose config validation. -- TypeScript compute tests and coverage gates. - -The retrieval regression gate uses `.harness/evaluation/retrieval-regression-report.json` and fails on severe recall, citation-hit, citation-accuracy, faithfulness, no-answer, baseline-delta, or sample-size regressions. - -## Workflow Runtime Boundary - -KnowledgeFS currently runs durable work through `JobQueueAdapter` implementations and explicit TypeScript state machines. The future Temporal-compatible boundary is documented in [docs/temporal-compatible-interface.md](docs/temporal-compatible-interface.md); it defines how document compilation, retention cleanup, bulk operations, and model upgrade workflows can later move to Temporal without leaking Temporal SDK concepts into API routes or repositories. - -## Documentation - -- [API Reference](docs/api-reference.md): route map, auth/scopes, error semantics, ingestion, query, evaluation, KnowledgeFS, job, retention, and snapshot endpoints. -- [Production deployment guide](docs/production-deployment.md): SaaS and Standalone deployment shape, environment variables, release gates, smoke checks, rollback, and current production wiring gaps. -- [Operator Manual](docs/operator-manual.md): daily health checks, release checklist, ingestion/retrieval/evaluation operations, incident response, rollback, observability, and performance guardrails. -- [Local infrastructure guide](infra/local/README.md): Docker Compose services, pgvector init, and MinIO bucket bootstrap. -- [AWS Terraform plan](infra/aws_terraform/README.md): target AWS Standalone architecture diagram and component/env mapping (Terraform code pending). - -## Development Workflow - -The active project workflow is documented in `.harness/agents/development-requirements.md` and `.harness/docs/TEMP-task-document.md`. - -Core rules: - -- Follow test-driven development for behavior changes. -- Keep coverage at or above 90%. -- Record every implementation slice under `.harness/changes`. -- Update `.harness/docs/TEMP-progress-document.md` as work completes. -- Commit and push after each verified implementation slice. -- After every 10 implementation commits from the latest review checkpoint, pause feature work and review project health. - -## Useful Commands - -```bash -pnpm install -pnpm check -pnpm build -pnpm lint -pnpm dev:infra -pnpm dev:stack -pnpm compose:config -pnpm db:migrations:write -pnpm db:migrations:check -``` - -## Project Status - -This repository is under active iterative development. The latest authoritative status is in: - -- `.harness/docs/iteration-plan.md` -- `.harness/docs/TEMP-task-document.md` -- `.harness/docs/TEMP-progress-document.md` -- `.harness/changes/` +- [API reference](docs/api-reference.md) +- [Production deployment](docs/production-deployment.md) +- [Operator manual](docs/operator-manual.md) +- [Project overview](docs/project-overview.md) +- [Kubernetes Dify integration baseline](infra/kubernetes/README.md) +- [OpenAPI snapshot](openapi/knowledge-fs.openapi.json) diff --git a/knowledge-fs/apps/api/Dockerfile b/knowledge-fs/apps/api/Dockerfile index 1584b725045..80ad61e6434 100644 --- a/knowledge-fs/apps/api/Dockerfile +++ b/knowledge-fs/apps/api/Dockerfile @@ -19,7 +19,6 @@ COPY packages/dify-model-runtime-client/package.json packages/dify-model-runtime COPY packages/embeddings/package.json packages/embeddings/package.json COPY packages/generation/package.json packages/generation/package.json COPY packages/parsers/package.json packages/parsers/package.json -COPY packages/plugin-daemon-client/package.json packages/plugin-daemon-client/package.json RUN pnpm install --frozen-lockfile --filter @knowledge/api-app... @@ -34,7 +33,6 @@ COPY packages/dify-model-runtime-client packages/dify-model-runtime-client COPY packages/embeddings packages/embeddings COPY packages/generation packages/generation COPY packages/parsers packages/parsers -COPY packages/plugin-daemon-client packages/plugin-daemon-client RUN pnpm --filter @knowledge/api-app build:prod diff --git a/knowledge-fs/apps/api/package.json b/knowledge-fs/apps/api/package.json index 63d628131c7..6aadc39636e 100644 --- a/knowledge-fs/apps/api/package.json +++ b/knowledge-fs/apps/api/package.json @@ -4,7 +4,7 @@ "type": "module", "scripts": { "build": "tsc --noEmit", - "build:prod": "esbuild src/server.ts --bundle --platform=node --format=esm --target=node22 --banner:js=\"import { createRequire } from 'module';const require = createRequire(import.meta.url);\" --outfile=dist/server.mjs", + "build:prod": "esbuild src/server.ts --bundle --platform=node --format=esm --target=node22 --banner:js=\"import { createRequire as __knowledgeCreateRequire } from 'node:module';const require = __knowledgeCreateRequire(import.meta.url);\" --outfile=dist/server.mjs", "dev": "NODE_ENV=development node --env-file-if-exists=../../infra/local/.env --import tsx --watch src/server.ts", "start": "node dist/server.mjs", "test": "vitest run --passWithNoTests", @@ -21,7 +21,6 @@ "@knowledge/embeddings": "workspace:*", "@knowledge/generation": "workspace:*", "@knowledge/parsers": "workspace:*", - "@knowledge/plugin-daemon-client": "workspace:*", "hono": "^4.12.25" }, "devDependencies": { diff --git a/knowledge-fs/apps/api/src/answer-generation-options.test.ts b/knowledge-fs/apps/api/src/answer-generation-options.test.ts index 7ea3b64651b..0732e2a8414 100644 --- a/knowledge-fs/apps/api/src/answer-generation-options.test.ts +++ b/knowledge-fs/apps/api/src/answer-generation-options.test.ts @@ -20,7 +20,7 @@ describe("createApiAnswerGenerationOptions", () => { KNOWLEDGE_ANSWER_MODEL: "gpt-4.1-mini", KNOWLEDGE_ANSWER_PLUGIN_ID: "langgenius/openai", KNOWLEDGE_ANSWER_PLUGIN_PROVIDER: "openai", - KNOWLEDGE_ANSWER_PROVIDER: "plugin-daemon", + KNOWLEDGE_ANSWER_PROVIDER: "dify-model-runtime", }); expect(options?.model).toBe("gpt-4.1-mini"); @@ -51,7 +51,7 @@ describe("createApiAnswerGenerationOptions", () => { KNOWLEDGE_ANSWER_MODEL: "legacy-model", KNOWLEDGE_ANSWER_PLUGIN_ID: "vendor/reasoning", KNOWLEDGE_ANSWER_PLUGIN_PROVIDER: "vendor", - KNOWLEDGE_ANSWER_PROVIDER: "plugin-daemon", + KNOWLEDGE_ANSWER_PROVIDER: "dify-model-runtime", }); if (!options) throw new Error("Expected answer generation options"); const generate = (provider: typeof options.provider, model: string, tenantId: string) => @@ -119,7 +119,7 @@ describe("createApiAnswerGenerationOptions", () => { KNOWLEDGE_ANSWER_MODEL: "gpt-4.1", KNOWLEDGE_ANSWER_PLUGIN_ID: "langgenius/openai", KNOWLEDGE_ANSWER_PLUGIN_PROVIDER: "openai", - KNOWLEDGE_ANSWER_PROVIDER: "plugin-daemon", + KNOWLEDGE_ANSWER_PROVIDER: "dify-model-runtime", }); expect(options?.model).toBe("gpt-4.1"); @@ -128,7 +128,7 @@ describe("createApiAnswerGenerationOptions", () => { it("requires Dify model routing config and rejects unknown providers", () => { expect(() => - createApiAnswerGenerationOptions({ KNOWLEDGE_ANSWER_PROVIDER: "plugin-daemon" }), + createApiAnswerGenerationOptions({ KNOWLEDGE_ANSWER_PROVIDER: "dify-model-runtime" }), ).toThrow("KNOWLEDGE_ANSWER_MODEL is required for answer generation"); expect(() => createApiAnswerGenerationOptions({ KNOWLEDGE_ANSWER_PROVIDER: "mistral" }), diff --git a/knowledge-fs/apps/api/src/answer-generation-options.ts b/knowledge-fs/apps/api/src/answer-generation-options.ts index 2b254054c7e..1e41d1c9f35 100644 --- a/knowledge-fs/apps/api/src/answer-generation-options.ts +++ b/knowledge-fs/apps/api/src/answer-generation-options.ts @@ -121,9 +121,9 @@ function answerEnabled(value: string | undefined): boolean { return false; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return true; } - throw new Error("KNOWLEDGE_ANSWER_PROVIDER must be dify-model-runtime, plugin-daemon, or off"); + throw new Error("KNOWLEDGE_ANSWER_PROVIDER must be dify-model-runtime or off"); } diff --git a/knowledge-fs/apps/api/src/datasource-invocation-client.ts b/knowledge-fs/apps/api/src/datasource-invocation-client.ts index 882b9e0388a..9b60f89d6db 100644 --- a/knowledge-fs/apps/api/src/datasource-invocation-client.ts +++ b/knowledge-fs/apps/api/src/datasource-invocation-client.ts @@ -42,7 +42,7 @@ export type ApiDatasourceInvocationInput = readonly operation: "validate_credentials"; }); -/** Deployment adapter shared by datasource connectors; only standalone implementations see secrets. */ +/** Dify-backed deployment adapter shared by datasource connectors. */ export interface ApiDatasourceInvocationClient { dispatch(input: ApiDatasourceInvocationInput): AsyncGenerator; } diff --git a/knowledge-fs/apps/api/src/datasource-runtime-code-health.test.ts b/knowledge-fs/apps/api/src/datasource-runtime-code-health.test.ts index bae06b530b9..d22535d8113 100644 --- a/knowledge-fs/apps/api/src/datasource-runtime-code-health.test.ts +++ b/knowledge-fs/apps/api/src/datasource-runtime-code-health.test.ts @@ -23,17 +23,6 @@ describe("KnowledgeFS datasource-runtime architecture", () => { } }); - it("isolates direct plugin-daemon credentials to the standalone adapter", () => { - const source = readFileSync( - resolve(import.meta.dirname, "standalone-datasource-invocation-client.ts"), - "utf8", - ); - - expect(source).toContain("@knowledge/plugin-daemon-client"); - expect(source).toMatch(/\bcredentials\s*:/u); - expect(source).toContain("dispatchDatasourceStream"); - }); - it("keeps the Dify wire contract credential-reference-only", () => { const source = readFileSync( resolve(import.meta.dirname, "../../../packages/dify-datasource-runtime-client/src/index.ts"), @@ -45,21 +34,15 @@ describe("KnowledgeFS datasource-runtime architecture", () => { expect(source).not.toContain("@knowledge/plugin-daemon-client"); }); - it("assembles Dify credential ownership whenever integrated mode is enabled", () => { + it("assembles Dify credential ownership independently of rollout mode", () => { const source = readFileSync(resolve(import.meta.dirname, "index.ts"), "utf8"); expect(source).toContain("createApiDatasourceInvocationClient"); - expect(source).toMatch(/credentialMode:\s*integratedModeEnabled\s*\?\s*"dify-managed"/u); - expect(source).toMatch(/inlineSourceCredentialsAllowed:\s*!integratedModeEnabled/u); - expect(source).toMatch( - /integratedModeEnabled\s*\?\s*undefined\s*:\s*createApiSourceSecretStore/u, - ); - expect(source).toMatch( - /sourceOAuthOptions\s*=\s*integratedModeEnabled\s*\?[^:]+providerIds:\s*new Set\(\)/su, - ); - expect(source).toMatch( - /sourceCredentialBackfill\s*=\s*integratedModeEnabled\s*\?\s*undefined/u, - ); - expect(source).toMatch(/!integratedModeEnabled\s*&&\s*sourceSecretStore/u); + expect(source).toContain('credentialMode: "dify-managed"'); + expect(source).toContain("inlineSourceCredentialsAllowed: false"); + expect(source).not.toContain("createApiSourceSecretStore"); + expect(source).not.toContain("createApiSourceOAuthProviderOptions"); + expect(source).not.toContain("createApiSourceCredentialBackfillAssembly"); + expect(source).not.toContain("sourceConnectionSecretCleanup"); }); }); diff --git a/knowledge-fs/apps/api/src/datasource-runtime-options.test.ts b/knowledge-fs/apps/api/src/datasource-runtime-options.test.ts index 2de1e306229..eaddf69f3d4 100644 --- a/knowledge-fs/apps/api/src/datasource-runtime-options.test.ts +++ b/knowledge-fs/apps/api/src/datasource-runtime-options.test.ts @@ -26,20 +26,21 @@ const DIFY_SOURCE: Source = { afterEach(() => vi.unstubAllGlobals()); describe("createApiDatasourceInvocationClient", () => { - it("selects Dify exclusively in integrated mode", async () => { + it("always selects Dify even before Workspace rollout activation", async () => { const requests: { readonly init?: RequestInit; readonly input: RequestInfo | URL }[] = []; const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { requests.push({ input, ...(init ? { init } : {}) }); return responseFromBytes(frame({ data: { result: [] }, error: "" })); }); vi.stubGlobal("fetch", fetchMock); - const client = createApiDatasourceInvocationClient({ + const env = { DIFY_INNER_API_KEY: "inner-key", DIFY_INNER_API_URL: "http://api:5001", - KNOWLEDGE_INTEGRATED_MODE_ENABLED: "true", - // If the standalone branch is accidentally constructed, this must fail validation. + KNOWLEDGE_INTEGRATED_MODE_ENABLED: "false", + // A direct plugin-daemon branch must never be constructed. PLUGIN_DAEMON_MAX_RESPONSE_BYTES: "invalid", - }); + }; + const client = createApiDatasourceInvocationClient(env); await collect( client.dispatch({ @@ -60,43 +61,41 @@ describe("createApiDatasourceInvocationClient", () => { expect(body).not.toHaveProperty("credentials"); }); - it("keeps the direct daemon path available only for standalone mode", async () => { - const requests: { readonly input: RequestInfo | URL }[] = []; - const fetchMock = vi.fn(async (input: RequestInfo | URL) => { - requests.push({ input }); - return new Response(`${JSON.stringify({ code: 0, data: { result: [] } })}\n`, { - status: 200, - }); - }); + it("rejects inline source credentials before invoking Dify", async () => { + const fetchMock = vi.fn(async () => + responseFromBytes(frame({ data: { result: [] }, error: "" })), + ); vi.stubGlobal("fetch", fetchMock); const source: Source = { ...DIFY_SOURCE, metadata: { - credentials: { token: "standalone-only" }, + credentialId: "dify-credential-1", + credentials: { token: "must-not-cross" }, datasource: "notion_datasource", pluginId: "langgenius/notion_datasource", provider: "notion_datasource", + providerKind: "online-document", }, }; - const client = createApiDatasourceInvocationClient({ - // If the Dify branch is accidentally constructed, this must fail validation. - DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES: "invalid", - PLUGIN_DAEMON_KEY: "daemon-key", + const env = { + DIFY_INNER_API_KEY: "inner-key", + DIFY_INNER_API_URL: "http://api:5001", + PLUGIN_DAEMON_KEY: "must-not-be-used", PLUGIN_DAEMON_URL: "http://plugin-daemon:5002", - }); + }; + const client = createApiDatasourceInvocationClient(env); - await collect( - client.dispatch({ - operation: "get_online_document_pages", - source, - tenantId: "tenant-1", - }), - ); + await expect( + collect( + client.dispatch({ + operation: "get_online_document_pages", + source, + tenantId: "tenant-1", + }), + ), + ).rejects.toThrow("Inline datasource credentials are forbidden"); - expect(fetchMock).toHaveBeenCalledOnce(); - expect(String(requests[0]?.input)).toContain( - "/plugin/tenant-1/dispatch/datasource/get_online_document_pages", - ); + expect(fetchMock).not.toHaveBeenCalled(); }); }); diff --git a/knowledge-fs/apps/api/src/datasource-runtime-options.ts b/knowledge-fs/apps/api/src/datasource-runtime-options.ts index 89a73bf5ec1..2b6890aa9a0 100644 --- a/knowledge-fs/apps/api/src/datasource-runtime-options.ts +++ b/knowledge-fs/apps/api/src/datasource-runtime-options.ts @@ -4,28 +4,14 @@ import { type DifyDatasourceRuntimeClientEnv, createApiDifyDatasourceRuntimeClient, } from "./dify-datasource-runtime-options"; -import { - type PluginDaemonClientEnv, - createApiPluginDaemonDatasourceClient, -} from "./plugin-daemon-options"; -import { createStandaloneDatasourceInvocationClient } from "./standalone-datasource-invocation-client"; -export interface ApiDatasourceRuntimeEnv - extends DifyDatasourceRuntimeClientEnv, - PluginDaemonClientEnv { - readonly KNOWLEDGE_INTEGRATED_MODE_ENABLED?: string | undefined; -} +export interface ApiDatasourceRuntimeEnv extends DifyDatasourceRuntimeClientEnv {} -/** Selects exactly one credential owner for datasource calls at deployment assembly time. */ +/** Routes datasource calls through Dify, which owns plugin credentials and invocation. */ export function createApiDatasourceInvocationClient( env: ApiDatasourceRuntimeEnv = process.env, ): ApiDatasourceInvocationClient { - if (env.KNOWLEDGE_INTEGRATED_MODE_ENABLED?.trim().toLowerCase() === "true") { - return createDifyDatasourceInvocationClient({ - client: createApiDifyDatasourceRuntimeClient(env), - }); - } - return createStandaloneDatasourceInvocationClient({ - client: createApiPluginDaemonDatasourceClient(env), + return createDifyDatasourceInvocationClient({ + client: createApiDifyDatasourceRuntimeClient(env), }); } diff --git a/knowledge-fs/apps/api/src/embedding-options.ts b/knowledge-fs/apps/api/src/embedding-options.ts index 52ffa703de2..76d64b3a06b 100644 --- a/knowledge-fs/apps/api/src/embedding-options.ts +++ b/knowledge-fs/apps/api/src/embedding-options.ts @@ -153,16 +153,14 @@ function normalizedProvider( return "off"; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return "dify-model-runtime"; } if (normalized === "static") { return "static"; } - throw new Error( - "KNOWLEDGE_EMBEDDING_PROVIDER must be dify-model-runtime, plugin-daemon, static, or off", - ); + throw new Error("KNOWLEDGE_EMBEDDING_PROVIDER must be dify-model-runtime, static, or off"); } function optionalPositiveIntegerEnv(value: string | undefined, name: string): number | undefined { diff --git a/knowledge-fs/apps/api/src/index.ts b/knowledge-fs/apps/api/src/index.ts index 28afe6509ed..d75dc6f750e 100644 --- a/knowledge-fs/apps/api/src/index.ts +++ b/knowledge-fs/apps/api/src/index.ts @@ -13,8 +13,6 @@ import { createDocumentMultimodalCandidateResolver, createHybridQueryGenerator, createInMemoryKnowledgeSpaceManifestRepository, - createInMemorySourceRepository, - createInMemorySourceRetiredSecretCleanupRepository, createJointCasSourceLogicalRevisionPublisher, createKnowledgeGateway, createKnowledgeSpaceAuthorizationGuard, @@ -29,10 +27,7 @@ import { createRetrievalExecutionLeaseCoordinator, createRetrievalPlanner, createRetrievalTestExecutor, - createSourceConnectionSecretCleanupRuntime, createSourceConnectionService, - createSourceCredentialService, - createSourceRetiredSecretCleanupRuntime, createStaticSourceProviderCatalog, } from "@knowledge/api"; @@ -80,14 +75,8 @@ import { createApiResearchTaskRuntime, } from "./research-task-runtime-options"; import { createApiRetriever } from "./retriever-options"; -import { createApiSourceCredentialBackfillAssembly } from "./source-credential-backfill-options"; import { createApiSourceCredentialTesterOptions } from "./source-credential-options"; -import { createApiSourceOAuthProviderOptions } from "./source-oauth-provider-options"; import { createApiSourceBulkRemovalRequester } from "./source-product-options"; -import { - assertApiSourceSecretDurability, - createApiSourceSecretStore, -} from "./source-secret-options"; import { createApiTidbFtsPostingBackfillAssembly } from "./tidb-fts-posting-backfill-options"; import { createApiTracingOptions } from "./tracing-options"; import { @@ -186,134 +175,41 @@ const difyManagedDatasourceFields = [ { name: "datasource", required: true, secret: false, type: "string" as const }, { name: "providerKind", required: true, secret: false, type: "string" as const }, ] as const; -const standaloneDatasourceFields = [ - { name: "pluginId", required: true, secret: false, type: "string" as const }, - { name: "provider", required: true, secret: false, type: "string" as const }, - { name: "datasource", required: true, secret: false, type: "string" as const }, - { - format: "password" as const, - name: "apiKey", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "token", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "accessToken", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "clientId", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "clientSecret", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "accessKeyId", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "secretAccessKey", - required: false, - secret: true, - type: "string" as const, - }, - { - format: "password" as const, - name: "sessionToken", - required: false, - secret: true, - type: "string" as const, - }, -] as const; -const sourceOAuthOptions = integratedModeEnabled - ? { - providerIds: new Set(), - registry: { get: (_providerId: string) => undefined }, - } - : createApiSourceOAuthProviderOptions(process.env); -// Persisted provider IDs remain stable across standalone and integrated deployments; only the -// runtime binding and credential owner change. -const supportedSourceProviderIds = new Set([ - "plugin-daemon-website", - "plugin-daemon-online-document", - "plugin-daemon-online-drive", -]); -for (const providerId of sourceOAuthOptions.providerIds) { - if (!supportedSourceProviderIds.has(providerId)) { - throw new Error(`OAuth source provider ${providerId} is not present in the source catalog`); - } -} -const sourceAuthKinds = (providerId: string) => - integratedModeEnabled - ? (["endpoint"] as const) - : ([ - "api-key" as const, - "endpoint" as const, - ...(sourceOAuthOptions.providerIds.has(providerId) ? ["oauth2" as const] : []), - ] as const); -const sourceConfigurationFields = integratedModeEnabled - ? difyManagedDatasourceFields - : standaloneDatasourceFields; +const sourceOAuthProviders = { get: (_providerId: string) => undefined }; +// These IDs are persisted contract values. Their runtime and credential owner is always Dify. const sourceProviderCatalog = createStaticSourceProviderCatalog([ { - authKinds: sourceAuthKinds("plugin-daemon-website"), + authKinds: ["endpoint"], available: true, capabilities: ["website-crawl"], - configuration: sourceConfigurationFields, - displayName: integratedModeEnabled ? "Dify website crawl" : "Plugin daemon website crawl", + configuration: difyManagedDatasourceFields, + displayName: "Dify website crawl", id: "plugin-daemon-website", }, { - authKinds: sourceAuthKinds("plugin-daemon-online-document"), + authKinds: ["endpoint"], available: true, capabilities: ["online-document"], - configuration: sourceConfigurationFields, - displayName: integratedModeEnabled ? "Dify online document" : "Plugin daemon online document", + configuration: difyManagedDatasourceFields, + displayName: "Dify online document", id: "plugin-daemon-online-document", }, { - authKinds: sourceAuthKinds("plugin-daemon-online-drive"), + authKinds: ["endpoint"], available: true, capabilities: ["online-drive"], - configuration: sourceConfigurationFields, - displayName: integratedModeEnabled ? "Dify online drive" : "Plugin daemon online drive", + configuration: difyManagedDatasourceFields, + displayName: "Dify online drive", id: "plugin-daemon-online-drive", }, ]); -const sourceOAuthProviders = sourceOAuthOptions.registry; const tracingOptions = createApiTracingOptions(); const autoRetrievalModeResolver = createLlmAutoRetrievalModeResolver({ providerFactory: profileReasoningCapability.providerFactory, ...(tracingOptions ? { traces: tracingOptions.traces } : {}), }); -const sourceSecretStore = integratedModeEnabled - ? undefined - : createApiSourceSecretStore(adapter.objectStorage); const databaseRepositories = createApiDatabaseRepositories({ database: adapter.database, - sourceCredentialFingerprinter: sourceSecretStore?.fingerprint, }); const retrievalExecutionLeases = databaseRepositories.durableDeletionEnabled && databaseRepositories.usesDatabaseRepositories @@ -335,7 +231,6 @@ const durableDeletion = createApiDurableDeletionAssembly({ enabled: databaseRepositories.durableDeletionEnabled, production: process.env.NODE_ENV === "production", repository: databaseRepositories.durableDeletionRepository, - secretStore: sourceSecretStore, usesDatabaseRepositories: databaseRepositories.usesDatabaseRepositories, }); const deletionLifecycleFence = databaseRepositories.deletionLifecycleFenceReader @@ -383,51 +278,7 @@ assertApiKnowledgeFsDurability({ production: process.env.NODE_ENV === "production", sessions: databaseRepositories.knowledgeFsSessions, }); -assertApiSourceSecretDurability({ - objectStorageKind: adapter.objectStorage.kind, - production: process.env.NODE_ENV === "production", - secretStoreConfigured: sourceSecretStore !== undefined, - usesDatabaseLifecycleLedger: databaseRepositories.sourceRetiredSecretCleanups !== undefined, -}); -const sourceRepository = - repositoryOptions.sources ?? - (sourceSecretStore ? createInMemorySourceRepository({ maxSources: 1_000 }) : undefined); -const sourceRetiredSecretCleanups = - databaseRepositories.sourceRetiredSecretCleanups ?? - (sourceSecretStore && sourceRepository - ? createInMemorySourceRetiredSecretCleanupRepository({ - maxClaimBatchSize: 25, - maxJobs: 10_000, - sources: sourceRepository, - }) - : undefined); -const sourceCredentials = - sourceSecretStore && sourceRepository && sourceRetiredSecretCleanups - ? createSourceCredentialService({ - retiredSecrets: sourceRetiredSecretCleanups, - secretStore: sourceSecretStore, - sources: sourceRepository, - }) - : undefined; -const sourceRetiredSecretCleanup = - sourceSecretStore && sourceRetiredSecretCleanups - ? createSourceRetiredSecretCleanupRuntime({ - intervalMs: 10_000, - leaseMs: 30_000, - maxClaimBatchSize: 25, - maxRetryCount: 20, - repository: sourceRetiredSecretCleanups, - secretStore: sourceSecretStore, - workerId: `source-retired-secret-cleanup:${randomUUID()}`, - }) - : undefined; -const sourceCredentialBackfill = integratedModeEnabled - ? undefined - : createApiSourceCredentialBackfillAssembly({ - repository: databaseRepositories.sourceCredentialBackfills, - secretStore: sourceSecretStore, - sources: databaseRepositories.sourceCredentialBackfills ? sourceRepository : undefined, - }); +const sourceRepository = repositoryOptions.sources; const knowledgeSpaceProfileBackfill = createApiKnowledgeSpaceProfileBackfillAssembly({ preflight: modelCapabilityPreflight, publicationBindings: databaseRepositories.knowledgeSpaceProfilePublications, @@ -578,8 +429,7 @@ const sourceProductAuthorization = repositoryOptions.knowledgeSpaceAccess const sourceConnectionService = sourceProductAuthorization && repositoryOptions.knowledgeSpaceAccess && - databaseRepositories.sourceConnections && - (integratedModeEnabled || sourceSecretStore) + databaseRepositories.sourceConnections ? createSourceConnectionService({ access: repositoryOptions.knowledgeSpaceAccess, allowDevelopmentLoopbackOAuthRedirects: @@ -591,19 +441,9 @@ const sourceConnectionService = .filter(Boolean), authorization: sourceProductAuthorization, catalog: sourceProviderCatalog, - credentialMode: integratedModeEnabled ? "dify-managed" : "local", + credentialMode: "dify-managed", oauth: sourceOAuthProviders, repository: databaseRepositories.sourceConnections, - ...(sourceSecretStore ? { secrets: sourceSecretStore } : {}), - }) - : undefined; -const sourceConnectionSecretCleanup = - !integratedModeEnabled && sourceSecretStore && databaseRepositories.sourceConnections - ? createSourceConnectionSecretCleanupRuntime({ - oauth: sourceOAuthProviders, - repository: databaseRepositories.sourceConnections, - secrets: sourceSecretStore, - workerId: `source-connection-secret-cleanup:${randomUUID()}`, }) : undefined; const sourceLogicalRevisions = @@ -938,9 +778,8 @@ const app = createKnowledgeGateway({ repository: databaseRepositories.knowledgeSpaceProfileMigrations, }), ...(sourceRepository ? { sources: sourceRepository } : {}), - ...(sourceCredentials ? { sourceCredentials } : {}), ...(sourceProduct ? { sourceProduct } : {}), - inlineSourceCredentialsAllowed: !integratedModeEnabled, + inlineSourceCredentialsAllowed: false, knowledgeSpaceManifests, legacyAccessMutationsReadOnly, legacyAuthorizationRemoved, @@ -962,9 +801,6 @@ documentCompilationRuntime?.start(); tidbFtsPostingBackfill?.start(); researchTaskRuntime?.start(); durableDeletion?.start(); -sourceCredentialBackfill?.start(); -sourceRetiredSecretCleanup?.start(); -sourceConnectionSecretCleanup?.start(); knowledgeSpaceProfileBackfill?.start(); uploadSessions?.start(); diff --git a/knowledge-fs/apps/api/src/llm-options.test.ts b/knowledge-fs/apps/api/src/llm-options.test.ts index 0d68fdb8b5c..dba54aa45bf 100644 --- a/knowledge-fs/apps/api/src/llm-options.test.ts +++ b/knowledge-fs/apps/api/src/llm-options.test.ts @@ -6,7 +6,7 @@ const PLUGIN_ENV = { KNOWLEDGE_ENTITY_EXTRACTION_MODEL: "entity-model", KNOWLEDGE_ENTITY_EXTRACTION_PLUGIN_ID: "langgenius/openai", KNOWLEDGE_ENTITY_EXTRACTION_PLUGIN_PROVIDER: "openai", - KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER: "plugin-daemon", + KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER: "dify-model-runtime", } as const; describe("createApiSemanticEntityExtractionOptions", () => { @@ -42,7 +42,7 @@ describe("createApiSemanticEntityExtractionOptions", () => { it("requires Dify model routing config and validates numeric bounds", () => { expect(() => createApiSemanticEntityExtractionOptions({ - KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER: "plugin-daemon", + KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER: "dify-model-runtime", }), ).toThrow("KNOWLEDGE_ENTITY_EXTRACTION_MODEL is required for semantic entity extraction"); expect(() => diff --git a/knowledge-fs/apps/api/src/llm-options.ts b/knowledge-fs/apps/api/src/llm-options.ts index 01ac5f58a7a..49a43b08213 100644 --- a/knowledge-fs/apps/api/src/llm-options.ts +++ b/knowledge-fs/apps/api/src/llm-options.ts @@ -115,11 +115,9 @@ function semanticExtractionEnabled(value: string | undefined): boolean { return false; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return true; } - throw new Error( - "KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER must be dify-model-runtime, plugin-daemon, or off", - ); + throw new Error("KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER must be dify-model-runtime or off"); } diff --git a/knowledge-fs/apps/api/src/model-runtime-code-health.test.ts b/knowledge-fs/apps/api/src/model-runtime-code-health.test.ts index d62c2e21f99..c3f6ef4533f 100644 --- a/knowledge-fs/apps/api/src/model-runtime-code-health.test.ts +++ b/knowledge-fs/apps/api/src/model-runtime-code-health.test.ts @@ -26,17 +26,11 @@ describe("KnowledgeFS model-runtime architecture", () => { } }); - it("keeps the plugin-daemon client datasource-only", () => { - const source = readFileSync( - resolve(import.meta.dirname, "../../../packages/plugin-daemon-client/src/index.ts"), - "utf8", - ); + it("keeps the API app independent from direct plugin-daemon transports", () => { + const packageJson = readFileSync(resolve(import.meta.dirname, "../package.json"), "utf8"); + const dockerfile = readFileSync(resolve(import.meta.dirname, "../Dockerfile"), "utf8"); - expect(source).not.toContain("dispatchUnary"); - expect(source).not.toContain("dispatchStream"); - expect(source).not.toContain("dispatch/model"); - expect(source).not.toContain("listModelProviders"); - expect(source).not.toContain("validateModelCredentials"); - expect(source).not.toContain("validateProviderCredentials"); + expect(packageJson).not.toContain("@knowledge/plugin-daemon-client"); + expect(dockerfile).not.toContain("packages/plugin-daemon-client"); }); }); diff --git a/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts b/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts index ed891e93164..3c44674befd 100644 --- a/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts +++ b/knowledge-fs/apps/api/src/multimodal-answer-options.test.ts @@ -110,7 +110,7 @@ describe("createApiMultimodalAnswerOptions", () => { const adapter = createNodePlatformAdapter({ env: {} }); expect(() => createApiMultimodalAnswerOptions({ - env: { KNOWLEDGE_MULTIMODAL_ANSWER_PROVIDER: "plugin-daemon" }, + env: { KNOWLEDGE_MULTIMODAL_ANSWER_PROVIDER: "dify-model-runtime" }, objectStorage: adapter.objectStorage, }), ).toThrow("KNOWLEDGE_MULTIMODAL_ANSWER_MODEL is required for multimodal answer generation"); @@ -127,7 +127,7 @@ describe("createApiMultimodalAnswerOptions", () => { KNOWLEDGE_MULTIMODAL_ANSWER_MODEL: "gpt-vision", KNOWLEDGE_MULTIMODAL_ANSWER_PLUGIN_ID: "langgenius/openai", KNOWLEDGE_MULTIMODAL_ANSWER_PLUGIN_PROVIDER: "openai", - KNOWLEDGE_MULTIMODAL_ANSWER_PROVIDER: "plugin-daemon", + KNOWLEDGE_MULTIMODAL_ANSWER_PROVIDER: "dify-model-runtime", }, objectStorage: adapter.objectStorage, }), diff --git a/knowledge-fs/apps/api/src/multimodal-answer-options.ts b/knowledge-fs/apps/api/src/multimodal-answer-options.ts index ea0c2717f1c..09a42bcd081 100644 --- a/knowledge-fs/apps/api/src/multimodal-answer-options.ts +++ b/knowledge-fs/apps/api/src/multimodal-answer-options.ts @@ -221,13 +221,11 @@ function multimodalAnswerEnabled(value: string | undefined): boolean { return false; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return true; } - throw new Error( - "KNOWLEDGE_MULTIMODAL_ANSWER_PROVIDER must be dify-model-runtime, plugin-daemon, or off", - ); + throw new Error("KNOWLEDGE_MULTIMODAL_ANSWER_PROVIDER must be dify-model-runtime or off"); } function imageDetailEnv(value: string | undefined): "auto" | "high" | "low" { diff --git a/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts b/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts index bee523066cb..4e9af7fc09b 100644 --- a/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts +++ b/knowledge-fs/apps/api/src/multimodal-enrichment-options.test.ts @@ -165,7 +165,7 @@ describe("createApiMultimodalEnrichmentOptions", () => { const adapter = createNodePlatformAdapter({ env: {} }); expect(() => createApiMultimodalEnrichmentOptions({ - env: { KNOWLEDGE_MULTIMODAL_ENRICHMENT_PROVIDER: "plugin-daemon" }, + env: { KNOWLEDGE_MULTIMODAL_ENRICHMENT_PROVIDER: "dify-model-runtime" }, objectStorage: adapter.objectStorage, }), ).toThrow("KNOWLEDGE_MULTIMODAL_ENRICHMENT_MODEL is required for multimodal enrichment"); @@ -182,7 +182,7 @@ describe("createApiMultimodalEnrichmentOptions", () => { KNOWLEDGE_MULTIMODAL_ENRICHMENT_MODEL: "gpt-vision", KNOWLEDGE_MULTIMODAL_ENRICHMENT_PLUGIN_ID: "langgenius/openai", KNOWLEDGE_MULTIMODAL_ENRICHMENT_PLUGIN_PROVIDER: "openai", - KNOWLEDGE_MULTIMODAL_ENRICHMENT_PROVIDER: "plugin-daemon", + KNOWLEDGE_MULTIMODAL_ENRICHMENT_PROVIDER: "dify-model-runtime", }, objectStorage: adapter.objectStorage, }), diff --git a/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts b/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts index bdd4ad7277e..d8e32e60832 100644 --- a/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts +++ b/knowledge-fs/apps/api/src/multimodal-enrichment-options.ts @@ -308,13 +308,11 @@ function enrichmentEnabled(value: string | undefined): boolean { return false; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return true; } - throw new Error( - "KNOWLEDGE_MULTIMODAL_ENRICHMENT_PROVIDER must be dify-model-runtime, plugin-daemon, or off", - ); + throw new Error("KNOWLEDGE_MULTIMODAL_ENRICHMENT_PROVIDER must be dify-model-runtime or off"); } function imageDetailEnv(value: string | undefined): "auto" | "high" | "low" { diff --git a/knowledge-fs/apps/api/src/plugin-daemon-options.ts b/knowledge-fs/apps/api/src/plugin-daemon-options.ts deleted file mode 100644 index a9d89acd668..00000000000 --- a/knowledge-fs/apps/api/src/plugin-daemon-options.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - type PluginDaemonDatasourceClient, - createPluginDaemonClient, -} from "@knowledge/plugin-daemon-client"; - -export interface PluginDaemonClientEnv { - readonly PLUGIN_DAEMON_KEY?: string | undefined; - readonly PLUGIN_DAEMON_MAX_RESPONSE_BYTES?: string | undefined; - readonly PLUGIN_DAEMON_MAX_RETRIES?: string | undefined; - readonly PLUGIN_DAEMON_RETRY_DELAY_MS?: string | undefined; - readonly PLUGIN_DAEMON_URL?: string | undefined; -} - -const DEFAULT_PLUGIN_DAEMON_URL = "http://localhost:5002"; -const DEFAULT_PLUGIN_DAEMON_KEY = "plugin-api-key"; - -/** Builds the shared plugin-daemon transport client from environment configuration. */ -export function createApiPluginDaemonDatasourceClient( - env: PluginDaemonClientEnv, -): PluginDaemonDatasourceClient { - const maxResponseBytes = optionalNonNegativeInt( - env.PLUGIN_DAEMON_MAX_RESPONSE_BYTES, - "PLUGIN_DAEMON_MAX_RESPONSE_BYTES", - 1, - ); - const maxRetries = optionalNonNegativeInt( - env.PLUGIN_DAEMON_MAX_RETRIES, - "PLUGIN_DAEMON_MAX_RETRIES", - ); - const retryDelayMs = optionalNonNegativeInt( - env.PLUGIN_DAEMON_RETRY_DELAY_MS, - "PLUGIN_DAEMON_RETRY_DELAY_MS", - ); - - const client = createPluginDaemonClient({ - apiKey: pluginDaemonTrimmed(env.PLUGIN_DAEMON_KEY) ?? DEFAULT_PLUGIN_DAEMON_KEY, - baseUrl: pluginDaemonTrimmed(env.PLUGIN_DAEMON_URL) ?? DEFAULT_PLUGIN_DAEMON_URL, - ...(maxResponseBytes === undefined ? {} : { maxResponseBytes }), - ...(maxRetries === undefined ? {} : { maxRetries }), - ...(retryDelayMs === undefined ? {} : { retryDelayMs }), - }); - return { - dispatchDatasourceStream: (input) => client.dispatchDatasourceStream(input), - }; -} - -export function pluginDaemonTrimmed(value: string | undefined): string | undefined { - const text = value?.trim(); - - return text ? text : undefined; -} - -function optionalNonNegativeInt( - value: string | undefined, - name: string, - min = 0, -): number | undefined { - const raw = pluginDaemonTrimmed(value); - - if (!raw) { - return undefined; - } - - const parsed = Number(raw); - - if (!Number.isInteger(parsed) || parsed < min) { - throw new Error(`${name} must be an integer >= ${min}`); - } - - return parsed; -} diff --git a/knowledge-fs/apps/api/src/readiness-options.test.ts b/knowledge-fs/apps/api/src/readiness-options.test.ts index 4406c9d656a..00000c77661 100644 --- a/knowledge-fs/apps/api/src/readiness-options.test.ts +++ b/knowledge-fs/apps/api/src/readiness-options.test.ts @@ -11,59 +11,67 @@ describe("createApiDeploymentReadinessChecks", () => { expect(await checks["auth.verifier"]?.()).toBe(false); expect(await checks["dify-model-runtime.configuration"]?.()).toBe(false); - expect(await checks["plugin-daemon.configuration"]?.()).toBe(false); + expect(await checks["dify-datasource-runtime.configuration"]?.()).toBe(false); + expect(await checks["dify-object-storage.configuration"]?.()).toBe(false); + expect(checks["plugin-daemon.configuration"]).toBeUndefined(); }); - it("accepts the production deployment inputs only when both are explicitly assembled", async () => { + it("accepts production only when the Dify dependency is explicitly assembled", async () => { const checks = createApiDeploymentReadinessChecks({ authVerifierConfigured: true, env: { NODE_ENV: "production", DIFY_INNER_API_KEY: "inner-key", DIFY_INNER_API_URL: "http://api:5001", - PLUGIN_DAEMON_KEY: "server-key", - PLUGIN_DAEMON_URL: "http://plugin_daemon:5002", }, }); expect(await checks["auth.verifier"]?.()).toBe(true); expect(await checks["dify-model-runtime.configuration"]?.()).toBe(true); - expect(await checks["plugin-daemon.configuration"]?.()).toBe(true); + expect(await checks["dify-datasource-runtime.configuration"]?.()).toBe(true); + expect(await checks["dify-object-storage.configuration"]?.()).toBe(true); + expect(checks["plugin-daemon.configuration"]).toBeUndefined(); }); - it("uses Dify as the sole datasource runtime in integrated production mode", async () => { + it("uses Dify as the sole runtime even when rollout mode is disabled", async () => { + const env = { + NODE_ENV: "production", + DIFY_INNER_API_KEY: "inner-key", + DIFY_INNER_API_URL: "http://api:5001", + KNOWLEDGE_INTEGRATED_MODE_ENABLED: "false", + }; const checks = createApiDeploymentReadinessChecks({ authVerifierConfigured: true, - env: { - NODE_ENV: "production", - DIFY_INNER_API_KEY: "inner-key", - DIFY_INNER_API_URL: "http://api:5001", - KNOWLEDGE_INTEGRATED_MODE_ENABLED: "true", - }, + env, }); expect(await checks["dify-model-runtime.configuration"]?.()).toBe(true); expect(await checks["dify-datasource-runtime.configuration"]?.()).toBe(true); + expect(await checks["dify-object-storage.configuration"]?.()).toBe(true); expect(checks["plugin-daemon.configuration"]).toBeUndefined(); }); - it("fails the integrated datasource check when Dify inner API wiring is absent", async () => { + it("fails every Dify dependency check when inner API wiring is absent", async () => { + const env = { KNOWLEDGE_INTEGRATED_MODE_ENABLED: "false", NODE_ENV: "production" }; const checks = createApiDeploymentReadinessChecks({ authVerifierConfigured: true, - env: { KNOWLEDGE_INTEGRATED_MODE_ENABLED: "true", NODE_ENV: "production" }, + env, }); + expect(await checks["dify-model-runtime.configuration"]?.()).toBe(false); expect(await checks["dify-datasource-runtime.configuration"]?.()).toBe(false); + expect(await checks["dify-object-storage.configuration"]?.()).toBe(false); expect(checks["plugin-daemon.configuration"]).toBeUndefined(); }); - it("does not require internal transport wiring for the standalone development profile", async () => { + it("allows default loopback Dify wiring in development", async () => { const checks = createApiDeploymentReadinessChecks({ authVerifierConfigured: true, env: { NODE_ENV: "development" }, }); expect(await checks["dify-model-runtime.configuration"]?.()).toBe(true); - expect(await checks["plugin-daemon.configuration"]?.()).toBe(true); + expect(await checks["dify-datasource-runtime.configuration"]?.()).toBe(true); + expect(await checks["dify-object-storage.configuration"]?.()).toBe(true); }); }); diff --git a/knowledge-fs/apps/api/src/readiness-options.ts b/knowledge-fs/apps/api/src/readiness-options.ts index e864b01eeea..632da6652d5 100644 --- a/knowledge-fs/apps/api/src/readiness-options.ts +++ b/knowledge-fs/apps/api/src/readiness-options.ts @@ -3,10 +3,7 @@ import type { GatewayReadinessChecks } from "@knowledge/api"; export interface ApiDeploymentReadinessEnv { readonly DIFY_INNER_API_KEY?: string | undefined; readonly DIFY_INNER_API_URL?: string | undefined; - readonly KNOWLEDGE_INTEGRATED_MODE_ENABLED?: string | undefined; readonly NODE_ENV?: string | undefined; - readonly PLUGIN_DAEMON_KEY?: string | undefined; - readonly PLUGIN_DAEMON_URL?: string | undefined; } export interface CreateApiDeploymentReadinessChecksOptions { @@ -16,26 +13,21 @@ export interface CreateApiDeploymentReadinessChecksOptions { /** * Converts deployment assembly state into explicit package-level readiness checks. Production - * requires a real auth verifier plus explicit Dify model-runtime transport. Datasources use the - * same Dify inner API in integrated mode and retain plugin-daemon only for standalone mode. + * requires a real auth verifier plus an explicit Dify inner-API transport. Models, datasources, + * and object storage all use that same dependency. */ export function createApiDeploymentReadinessChecks({ authVerifierConfigured, env = process.env, }: CreateApiDeploymentReadinessChecksOptions): GatewayReadinessChecks { const production = env.NODE_ENV?.trim() === "production"; - const integratedModeEnabled = - env.KNOWLEDGE_INTEGRATED_MODE_ENABLED?.trim().toLowerCase() === "true"; - const pluginDaemonConfigured = - !production || Boolean(env.PLUGIN_DAEMON_URL?.trim() && env.PLUGIN_DAEMON_KEY?.trim()); - const difyModelRuntimeConfigured = + const difyInnerApiConfigured = !production || Boolean(env.DIFY_INNER_API_URL?.trim() && env.DIFY_INNER_API_KEY?.trim()); return { "auth.verifier": () => authVerifierConfigured, - "dify-model-runtime.configuration": () => difyModelRuntimeConfigured, - ...(integratedModeEnabled - ? { "dify-datasource-runtime.configuration": () => difyModelRuntimeConfigured } - : { "plugin-daemon.configuration": () => pluginDaemonConfigured }), + "dify-model-runtime.configuration": () => difyInnerApiConfigured, + "dify-datasource-runtime.configuration": () => difyInnerApiConfigured, + "dify-object-storage.configuration": () => difyInnerApiConfigured, }; } diff --git a/knowledge-fs/apps/api/src/repository-options.test.ts b/knowledge-fs/apps/api/src/repository-options.test.ts index b78d0d8ad42..555818b7e13 100644 --- a/knowledge-fs/apps/api/src/repository-options.test.ts +++ b/knowledge-fs/apps/api/src/repository-options.test.ts @@ -396,10 +396,10 @@ describe("API app repository wiring", () => { expect(source).toContain("assertApiDocumentWriteSafety"); expect(source).toContain("assertApiAgentWorkspaceSnapshotDurability"); expect(source).toContain("assertApiKnowledgeFsDurability"); - expect(source).toContain("createApiSourceCredentialBackfillAssembly"); + expect(source).not.toContain("createApiSourceCredentialBackfillAssembly"); expect(source).toContain("createApiSourceBulkRemovalRequester"); expect(source).toContain("createApiUploadSessionAssembly"); - expect(source).toContain("sourceCredentialBackfill?.start()"); + expect(source).not.toContain("sourceCredentialBackfill?.start()"); expect(source).toContain("uploadSessions?.start()"); expect(source).toContain("uploadSessions: uploadSessions.sessions"); expect(source).toContain('"direct-upload.configuration"'); diff --git a/knowledge-fs/apps/api/src/reranker-options.ts b/knowledge-fs/apps/api/src/reranker-options.ts index 66c70a6b956..1e314a80683 100644 --- a/knowledge-fs/apps/api/src/reranker-options.ts +++ b/knowledge-fs/apps/api/src/reranker-options.ts @@ -137,16 +137,14 @@ function normalizedProvider( return "off"; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return "dify-model-runtime"; } if (normalized === "static") { return "static"; } - throw new Error( - "KNOWLEDGE_RERANK_PROVIDER must be dify-model-runtime, plugin-daemon, static, or off", - ); + throw new Error("KNOWLEDGE_RERANK_PROVIDER must be dify-model-runtime, static, or off"); } function trimmed(value: string | undefined): string | undefined { diff --git a/knowledge-fs/apps/api/src/server-options.test.ts b/knowledge-fs/apps/api/src/server-options.test.ts index 9d493c0ce20..4d259650ca6 100644 --- a/knowledge-fs/apps/api/src/server-options.test.ts +++ b/knowledge-fs/apps/api/src/server-options.test.ts @@ -38,9 +38,7 @@ describe("API Dockerfile production runtime", () => { expect(dockerfile).toContain( "COPY packages/dify-model-runtime-client packages/dify-model-runtime-client", ); - expect(dockerfile).toContain( - "COPY packages/plugin-daemon-client packages/plugin-daemon-client", - ); + expect(dockerfile).not.toContain("packages/plugin-daemon-client"); expect(dockerfile).not.toMatch(/\b(?:rustup|cargo|wasm-bindgen|knowledge_compute)\b/); }); }); diff --git a/knowledge-fs/apps/api/src/source-credential-backfill-options.test.ts b/knowledge-fs/apps/api/src/source-credential-backfill-options.test.ts deleted file mode 100644 index f25e4ee1d2c..00000000000 --- a/knowledge-fs/apps/api/src/source-credential-backfill-options.test.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { - SourceCredentialBackfillRepository, - SourceRepository, - SourceSecretStore, -} from "@knowledge/api"; -import { describe, expect, it, vi } from "vitest"; - -import { createApiSourceCredentialBackfillAssembly } from "./source-credential-backfill-options"; - -describe("API source credential backfill assembly", () => { - it("does not install a replica-local fallback when any durable boundary is absent", () => { - const dependencies = stubs(); - expect(createApiSourceCredentialBackfillAssembly({})).toBeUndefined(); - expect( - createApiSourceCredentialBackfillAssembly({ - repository: dependencies.repository, - secretStore: dependencies.secretStore, - }), - ).toBeUndefined(); - expect( - createApiSourceCredentialBackfillAssembly({ - repository: dependencies.repository, - sources: dependencies.sources, - }), - ).toBeUndefined(); - expect( - createApiSourceCredentialBackfillAssembly({ - secretStore: dependencies.secretStore, - sources: dependencies.sources, - }), - ).toBeUndefined(); - }); - - it("validates bounded worker settings and starts idempotently", () => { - const dependencies = stubs(); - expect(() => - createApiSourceCredentialBackfillAssembly({ - ...dependencies, - env: { KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_CLAIM_BATCH: "0" }, - }), - ).toThrow("KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_CLAIM_BATCH"); - expect(() => - createApiSourceCredentialBackfillAssembly({ - ...dependencies, - env: { KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_MAX_RETRIES: "-1" }, - }), - ).toThrow("KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_MAX_RETRIES"); - - const assembly = createApiSourceCredentialBackfillAssembly({ - ...dependencies, - env: { - KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_CLAIM_BATCH: "2", - KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_DISCOVERY_BATCH: "3", - KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_INTERVAL_MS: "1000", - KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_LEASE_MS: "30000", - KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_MAX_RETRIES: "5", - }, - }); - expect(assembly).toBeDefined(); - assembly?.start(); - assembly?.start(); - assembly?.stop(); - assembly?.stop(); - }); -}); - -function stubs(): { - repository: SourceCredentialBackfillRepository; - secretStore: SourceSecretStore; - sources: Pick; -} { - return { - repository: { - claim: vi.fn(async () => []), - complete: vi.fn(), - discover: vi.fn(async () => ({ created: 0, scanned: 0 })), - fail: vi.fn(), - get: vi.fn(async () => null), - heartbeat: vi.fn(), - refresh: vi.fn(), - release: vi.fn(), - retryableFailure: vi.fn(), - retry: vi.fn(async () => null), - } as unknown as SourceCredentialBackfillRepository, - secretStore: { - delete: vi.fn(), - get: vi.fn(async () => null), - put: vi.fn(), - } as unknown as SourceSecretStore, - sources: { - get: vi.fn(async () => null), - update: vi.fn(async () => null), - }, - }; -} diff --git a/knowledge-fs/apps/api/src/source-credential-backfill-options.ts b/knowledge-fs/apps/api/src/source-credential-backfill-options.ts deleted file mode 100644 index 8022bed71f5..00000000000 --- a/knowledge-fs/apps/api/src/source-credential-backfill-options.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { randomUUID } from "node:crypto"; - -import { - type SourceCredentialBackfillRepository, - type SourceCredentialBackfillRuntime, - type SourceRepository, - type SourceSecretStore, - createSourceCredentialBackfillRuntime, -} from "@knowledge/api"; - -export interface ApiSourceCredentialBackfillEnv { - readonly KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_CLAIM_BATCH?: string | undefined; - readonly KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_DISCOVERY_BATCH?: string | undefined; - readonly KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_INTERVAL_MS?: string | undefined; - readonly KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_LEASE_MS?: string | undefined; - readonly KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_MAX_RETRIES?: string | undefined; -} - -export interface ApiSourceCredentialBackfillAssembly { - readonly runtime: SourceCredentialBackfillRuntime; - start(): void; - stop(): void; -} - -/** - * Installed only when all three durable boundaries exist: the database job repository, the - * database source repository, and the encrypted SecretStore. There is intentionally no in-memory - * job fallback because that would make rollout completion replica-local and crash-sensitive. - */ -export function createApiSourceCredentialBackfillAssembly(input: { - readonly env?: ApiSourceCredentialBackfillEnv | undefined; - readonly onError?: - | ((input: { readonly error: unknown; readonly jobId?: string | undefined }) => void) - | undefined; - readonly repository?: SourceCredentialBackfillRepository | undefined; - readonly secretStore?: SourceSecretStore | undefined; - readonly sources?: Pick | undefined; -}): ApiSourceCredentialBackfillAssembly | undefined { - if (!input.repository || !input.secretStore || !input.sources) { - return undefined; - } - const env = input.env ?? process.env; - const runtime = createSourceCredentialBackfillRuntime({ - discoveryBatchSize: positiveEnv( - env.KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_DISCOVERY_BATCH, - 100, - "KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_DISCOVERY_BATCH", - ), - intervalMs: positiveEnv( - env.KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_INTERVAL_MS, - 1_000, - "KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_INTERVAL_MS", - ), - leaseMs: positiveEnv( - env.KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_LEASE_MS, - 30_000, - "KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_LEASE_MS", - ), - maxClaimBatchSize: positiveEnv( - env.KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_CLAIM_BATCH, - 10, - "KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_CLAIM_BATCH", - ), - maxRetryCount: nonnegativeEnv( - env.KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_MAX_RETRIES, - 5, - "KNOWLEDGE_SOURCE_CREDENTIAL_BACKFILL_MAX_RETRIES", - ), - onError: ({ error, job }) => input.onError?.({ error, ...(job ? { jobId: job.id } : {}) }), - repository: input.repository, - secretStore: input.secretStore, - sources: input.sources, - workerId: `source-credential-backfill-${process.pid}-${randomUUID()}`, - }); - let started = false; - return { - runtime, - start: () => { - if (started) { - return; - } - started = true; - runtime.start(); - }, - stop: () => { - if (!started) { - return; - } - runtime.stop(); - started = false; - }, - }; -} - -function positiveEnv(value: string | undefined, fallback: number, name: string): number { - if (value === undefined || value.trim() === "") { - return fallback; - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < 1) { - throw new Error(`${name} must be a positive safe integer`); - } - return parsed; -} - -function nonnegativeEnv(value: string | undefined, fallback: number, name: string): number { - if (value === undefined || value.trim() === "") { - return fallback; - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < 0) { - throw new Error(`${name} must be a non-negative safe integer`); - } - return parsed; -} diff --git a/knowledge-fs/apps/api/src/source-oauth-provider-options.test.ts b/knowledge-fs/apps/api/src/source-oauth-provider-options.test.ts deleted file mode 100644 index 8463fec7bdb..00000000000 --- a/knowledge-fs/apps/api/src/source-oauth-provider-options.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { createApiSourceOAuthProviderOptions } from "./source-oauth-provider-options"; - -const providerConfig = { - authorizationUrl: "https://accounts.example.test/oauth/authorize", - clientAuthentication: "basic", - clientId: "client-a", - clientSecretEnv: "OAUTH_CLIENT_SECRET_A", - providerId: "plugin-daemon-online-document", - refreshIdempotencySupported: true, - revokeUrl: "https://accounts.example.test/oauth/revoke", - tokenUrl: "https://accounts.example.test/oauth/token", -}; - -afterEach(() => vi.unstubAllGlobals()); - -describe("source OAuth provider options", () => { - it("is unavailable unless an explicit provider registry is configured", () => { - const options = createApiSourceOAuthProviderOptions({}); - expect(options.providerIds.size).toBe(0); - expect(options.registry.get("plugin-daemon-online-document")).toBeUndefined(); - }); - - it("fails closed for inline/missing secrets, insecure endpoints, and non-idempotent refresh", () => { - expect(() => - createApiSourceOAuthProviderOptions({ - SOURCE_OAUTH_PROVIDERS_JSON: JSON.stringify([providerConfig]), - }), - ).toThrow(/client secret is not configured/u); - expect(() => - createApiSourceOAuthProviderOptions({ - OAUTH_CLIENT_SECRET_A: "secret", - SOURCE_OAUTH_PROVIDERS_JSON: JSON.stringify([ - { ...providerConfig, tokenUrl: "http://accounts.example.test/oauth/token" }, - ]), - }), - ).toThrow(/HTTPS URL/u); - expect(() => - createApiSourceOAuthProviderOptions({ - OAUTH_CLIENT_SECRET_A: "secret", - SOURCE_OAUTH_PROVIDERS_JSON: JSON.stringify([ - { ...providerConfig, refreshIdempotencySupported: false }, - ]), - }), - ).toThrow(/guarantee refresh idempotency/u); - }); - - it("implements PKCE exchange, stable refresh idempotency, and durable revoke calls", async () => { - const calls: Array<{ body: string; headers: Headers; url: string }> = []; - vi.stubGlobal( - "fetch", - vi.fn(async (url: string, init: RequestInit) => { - calls.push({ - body: String(init.body), - headers: init.headers as Headers, - url, - }); - return new Response( - JSON.stringify({ - access_token: "access-a", - expires_in: 3600, - refresh_token: "refresh-a", - scope: "read write", - token_type: "Bearer", - }), - { headers: { "content-type": "application/json" }, status: 200 }, - ); - }), - ); - const options = createApiSourceOAuthProviderOptions({ - OAUTH_CLIENT_SECRET_A: "secret-a", - SOURCE_OAUTH_PROVIDERS_JSON: JSON.stringify([providerConfig]), - }); - const provider = options.registry.get(providerConfig.providerId); - expect(provider).toBeDefined(); - const authorization = await provider?.authorizationUrl({ - codeChallenge: "challenge-a", - redirectUri: "https://api.example.test/source-oauth/callback", - scopes: ["read", "write"], - state: "state-a", - }); - expect(authorization).toContain("code_challenge_method=S256"); - expect(authorization).toContain("state=state-a"); - - await provider?.exchange({ - code: "code-a", - codeVerifier: "verifier-a", - redirectUri: "https://api.example.test/source-oauth/callback", - }); - await provider?.refresh({ - idempotencyKey: "source-refresh:connection-a:3", - refreshToken: "refresh-a", - }); - await provider?.revoke({ refreshToken: "refresh-a" }); - - expect(calls[0]?.body).toContain("code_verifier=verifier-a"); - expect(calls[1]?.headers.get("idempotency-key")).toBe("source-refresh:connection-a:3"); - expect(calls[2]?.url).toBe(providerConfig.revokeUrl); - expect(calls[2]?.body).toContain("token_type_hint=refresh_token"); - expect(calls.every((call) => call.headers.get("authorization")?.startsWith("Basic "))).toBe( - true, - ); - }); -}); diff --git a/knowledge-fs/apps/api/src/source-oauth-provider-options.ts b/knowledge-fs/apps/api/src/source-oauth-provider-options.ts deleted file mode 100644 index b4445e56ecd..00000000000 --- a/knowledge-fs/apps/api/src/source-oauth-provider-options.ts +++ /dev/null @@ -1,251 +0,0 @@ -import type { - SourceOAuthProvider, - SourceOAuthProviderRegistry, - SourceOAuthTokens, -} from "@knowledge/api"; - -export interface ApiSourceOAuthProviderOptions { - readonly providerIds: ReadonlySet; - readonly registry: SourceOAuthProviderRegistry; -} - -interface OAuthProviderConfiguration { - readonly authorizationUrl: string; - readonly clientAuthentication: "basic" | "body"; - readonly clientId: string; - readonly clientSecret: string; - readonly providerId: string; - readonly refreshIdempotencySupported: true; - readonly revokeUrl: string; - readonly tokenUrl: string; -} - -/** - * Explicit production registry. Secrets never live in SOURCE_OAUTH_PROVIDERS_JSON: each entry - * names a separate environment variable containing its client secret. - */ -export function createApiSourceOAuthProviderOptions( - env: NodeJS.ProcessEnv, -): ApiSourceOAuthProviderOptions { - const raw = env.SOURCE_OAUTH_PROVIDERS_JSON?.trim(); - if (!raw) return emptyOptions(); - let decoded: unknown; - try { - decoded = JSON.parse(raw); - } catch { - throw new Error("SOURCE_OAUTH_PROVIDERS_JSON must be valid JSON"); - } - if (!Array.isArray(decoded)) { - throw new Error("SOURCE_OAUTH_PROVIDERS_JSON must be an array"); - } - const providers = new Map(); - for (const entry of decoded) { - const configuration = readConfiguration(entry, env); - if (providers.has(configuration.providerId)) { - throw new Error(`Duplicate OAuth source provider ${configuration.providerId}`); - } - providers.set(configuration.providerId, createOAuthProvider(configuration)); - } - return { - providerIds: new Set(providers.keys()), - registry: { get: (providerId) => providers.get(providerId) }, - }; -} - -function emptyOptions(): ApiSourceOAuthProviderOptions { - return { - providerIds: new Set(), - registry: { get: () => undefined }, - }; -} - -function readConfiguration(value: unknown, env: NodeJS.ProcessEnv): OAuthProviderConfiguration { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("OAuth source provider configuration must be an object"); - } - const record = value as Record; - const providerId = identifier(record.providerId, "providerId"); - const clientId = text(record.clientId, "clientId", 512); - const clientSecretEnv = identifier(record.clientSecretEnv, "clientSecretEnv", true); - const clientSecret = env[clientSecretEnv]?.trim(); - if (!clientSecret) { - throw new Error(`OAuth source provider ${providerId} client secret is not configured`); - } - const clientAuthentication = record.clientAuthentication ?? "basic"; - if (clientAuthentication !== "basic" && clientAuthentication !== "body") { - throw new Error(`OAuth source provider ${providerId} clientAuthentication is invalid`); - } - return { - authorizationUrl: httpsUrl(record.authorizationUrl, "authorizationUrl"), - clientAuthentication, - clientId, - clientSecret, - providerId, - refreshIdempotencySupported: requireTrue(record.refreshIdempotencySupported, providerId), - revokeUrl: httpsUrl(record.revokeUrl, "revokeUrl"), - tokenUrl: httpsUrl(record.tokenUrl, "tokenUrl"), - }; -} - -function createOAuthProvider(configuration: OAuthProviderConfiguration): SourceOAuthProvider { - const tokenRequest = async ( - fields: Readonly>, - signal?: AbortSignal, - idempotencyKey?: string, - ): Promise => { - const form = new URLSearchParams(fields); - const headers = new Headers({ - accept: "application/json", - "content-type": "application/x-www-form-urlencoded", - }); - if (configuration.clientAuthentication === "basic") { - headers.set( - "authorization", - `Basic ${Buffer.from(`${configuration.clientId}:${configuration.clientSecret}`, "utf8").toString("base64")}`, - ); - } else { - form.set("client_id", configuration.clientId); - form.set("client_secret", configuration.clientSecret); - } - if (idempotencyKey) headers.set("idempotency-key", idempotencyKey); - const response = await fetch(configuration.tokenUrl, { - body: form, - headers, - method: "POST", - ...(signal ? { signal } : {}), - }); - if (!response.ok) { - throw new Error(`OAuth token endpoint returned ${response.status}`); - } - return parseTokens(await response.json()); - }; - - return { - authorizationUrl: async ({ codeChallenge, redirectUri, scopes, state }) => { - const url = new URL(configuration.authorizationUrl); - url.searchParams.set("client_id", configuration.clientId); - url.searchParams.set("code_challenge", codeChallenge); - url.searchParams.set("code_challenge_method", "S256"); - url.searchParams.set("redirect_uri", redirectUri); - url.searchParams.set("response_type", "code"); - url.searchParams.set("scope", [...new Set(scopes)].join(" ")); - url.searchParams.set("state", state); - return url.toString(); - }, - exchange: ({ code, codeVerifier, redirectUri, signal }) => - tokenRequest( - { - client_id: configuration.clientId, - code, - code_verifier: codeVerifier, - grant_type: "authorization_code", - redirect_uri: redirectUri, - }, - signal, - ), - refresh: ({ idempotencyKey, refreshToken, signal }) => - tokenRequest( - { - client_id: configuration.clientId, - grant_type: "refresh_token", - refresh_token: refreshToken, - }, - signal, - idempotencyKey, - ), - revoke: async ({ accessToken, refreshToken, signal }) => { - const token = refreshToken ?? accessToken; - if (!token) return; - const form = new URLSearchParams({ - client_id: configuration.clientId, - token, - token_type_hint: refreshToken ? "refresh_token" : "access_token", - }); - const headers = new Headers({ "content-type": "application/x-www-form-urlencoded" }); - if (configuration.clientAuthentication === "basic") { - headers.set( - "authorization", - `Basic ${Buffer.from(`${configuration.clientId}:${configuration.clientSecret}`, "utf8").toString("base64")}`, - ); - } else { - form.set("client_secret", configuration.clientSecret); - } - const response = await fetch(configuration.revokeUrl, { - body: form, - headers, - method: "POST", - ...(signal ? { signal } : {}), - }); - if (!response.ok) throw new Error(`OAuth revoke endpoint returned ${response.status}`); - }, - }; -} - -function parseTokens(value: unknown): SourceOAuthTokens { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error("OAuth token response is invalid"); - } - const record = value as Record; - const accessToken = text(record.access_token, "access_token", 16_384); - const expiresIn = record.expires_in; - if ( - expiresIn !== undefined && - (typeof expiresIn !== "number" || !Number.isFinite(expiresIn) || expiresIn <= 0) - ) { - throw new Error("OAuth expires_in is invalid"); - } - const scope = - typeof record.scope === "string" - ? record.scope.trim().split(/\s+/u).filter(Boolean) - : undefined; - return { - accessToken, - ...(typeof expiresIn === "number" - ? { expiresAt: new Date(Date.now() + Math.floor(expiresIn * 1_000)).toISOString() } - : {}), - ...(typeof record.refresh_token === "string" && record.refresh_token.trim() - ? { refreshToken: record.refresh_token } - : {}), - ...(scope ? { scopes: scope } : {}), - ...(typeof record.token_type === "string" && record.token_type.trim() - ? { tokenType: record.token_type } - : {}), - }; -} - -function httpsUrl(value: unknown, field: string): string { - const raw = text(value, field, 2_048); - let url: URL; - try { - url = new URL(raw); - } catch { - throw new Error(`OAuth source provider ${field} is invalid`); - } - if (url.protocol !== "https:" || url.username || url.password || url.hash) { - throw new Error( - `OAuth source provider ${field} must be an HTTPS URL without credentials or fragment`, - ); - } - return url.toString(); -} - -function identifier(value: unknown, field: string, envName = false): string { - const raw = text(value, field, 255); - const pattern = envName ? /^[A-Z_][A-Z0-9_]*$/u : /^[a-z0-9][a-z0-9._-]{0,127}$/u; - if (!pattern.test(raw)) throw new Error(`OAuth source provider ${field} is invalid`); - return raw; -} - -function text(value: unknown, field: string, maxLength: number): string { - if (typeof value !== "string" || !value.trim() || value.trim().length > maxLength) { - throw new Error(`OAuth source provider ${field} is invalid`); - } - return value.trim(); -} - -function requireTrue(value: unknown, providerId: string): true { - if (value !== true) { - throw new Error(`OAuth source provider ${providerId} must guarantee refresh idempotency`); - } - return true; -} diff --git a/knowledge-fs/apps/api/src/source-secret-options.test.ts b/knowledge-fs/apps/api/src/source-secret-options.test.ts deleted file mode 100644 index 979d13cf984..00000000000 --- a/knowledge-fs/apps/api/src/source-secret-options.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { createMemoryObjectStorageAdapter } from "@knowledge/adapters"; -import { describe, expect, it } from "vitest"; - -import { - assertApiSourceSecretDurability, - createApiSourceSecretStore, -} from "./source-secret-options"; - -describe("assertApiSourceSecretDurability", () => { - it("rejects production memory object storage when the secret store is configured", () => { - expect(() => - assertApiSourceSecretDurability({ - objectStorageKind: "memory", - production: true, - secretStoreConfigured: true, - usesDatabaseLifecycleLedger: true, - }), - ).toThrow(/durable object storage/u); - }); - - it("rejects a production in-memory lifecycle ledger", () => { - expect(() => - assertApiSourceSecretDurability({ - objectStorageKind: "s3-compatible", - production: true, - secretStoreConfigured: true, - usesDatabaseLifecycleLedger: false, - }), - ).toThrow(/database-backed source secret lifecycle ledger/u); - }); - - it("allows durable production assembly and all non-production or disabled assemblies", () => { - expect(() => - assertApiSourceSecretDurability({ - objectStorageKind: "s3-compatible", - production: true, - secretStoreConfigured: true, - usesDatabaseLifecycleLedger: true, - }), - ).not.toThrow(); - expect(() => - assertApiSourceSecretDurability({ - objectStorageKind: "memory", - production: false, - secretStoreConfigured: true, - usesDatabaseLifecycleLedger: false, - }), - ).not.toThrow(); - expect(() => - assertApiSourceSecretDurability({ - objectStorageKind: "memory", - production: true, - secretStoreConfigured: false, - usesDatabaseLifecycleLedger: false, - }), - ).not.toThrow(); - }); -}); - -describe("createApiSourceSecretStore", () => { - const storage = createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 128_000 }); - - it("stays disabled without a key and validates configured bounds", () => { - expect(createApiSourceSecretStore(storage, {})).toBeUndefined(); - expect(() => - createApiSourceSecretStore(storage, { - KNOWLEDGE_SOURCE_SECRET_KEY: Buffer.alloc(32, 1).toString("base64"), - KNOWLEDGE_SOURCE_SECRET_MAX_BYTES: "0", - }), - ).toThrow(/positive safe integer/u); - }); - - it("constructs an encrypted store from a 32-byte deployment key", async () => { - const store = createApiSourceSecretStore(storage, { - KNOWLEDGE_SOURCE_SECRET_KEY: Buffer.alloc(32, 2).toString("hex"), - }); - await expect( - store?.put({ - credentials: { token: "secret" }, - knowledgeSpaceId: "space-1", - ref: "source-secret:v1:018f0d60-7a49-7cc2-9c1b-5b36f18f2c44", - sourceId: "source-1", - tenantId: "tenant-1", - }), - ).resolves.toMatchObject({ ref: expect.stringMatching(/^source-secret:v1:/u) }); - }); -}); diff --git a/knowledge-fs/apps/api/src/source-secret-options.ts b/knowledge-fs/apps/api/src/source-secret-options.ts deleted file mode 100644 index 5e794146cd4..00000000000 --- a/knowledge-fs/apps/api/src/source-secret-options.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { - type KnowledgeGatewayOptions, - type SourceSecretStore, - createEncryptedObjectSourceSecretStore, - parseSourceSecretEncryptionKey, -} from "@knowledge/api"; - -export interface SourceSecretEnv { - readonly KNOWLEDGE_SOURCE_SECRET_KEY?: string | undefined; - readonly KNOWLEDGE_SOURCE_SECRET_MAX_BYTES?: string | undefined; - readonly KNOWLEDGE_SOURCE_SECRET_PREFIX?: string | undefined; -} - -export interface ApiSourceSecretDurabilityInput { - readonly objectStorageKind: KnowledgeGatewayOptions["adapter"]["objectStorage"]["kind"]; - readonly production: boolean; - readonly secretStoreConfigured: boolean; - readonly usesDatabaseLifecycleLedger: boolean; -} - -/** - * Source credentials must survive process loss as one durable unit: both their encrypted payloads - * and the lifecycle ledger that eventually removes retired payloads. Local development may use - * process-local adapters, but production must fail startup instead of silently accepting them. - */ -export function assertApiSourceSecretDurability({ - objectStorageKind, - production, - secretStoreConfigured, - usesDatabaseLifecycleLedger, -}: ApiSourceSecretDurabilityInput): void { - if (!production || !secretStoreConfigured) { - return; - } - if (objectStorageKind === "memory") { - throw new Error( - "Production Source SecretStore requires durable object storage; memory storage is not allowed", - ); - } - if (!usesDatabaseLifecycleLedger) { - throw new Error( - "Production Source SecretStore requires a database-backed source secret lifecycle ledger", - ); - } -} - -/** - * Secret storage is opt-in by key. When absent, credential-bearing source writes fail closed at the - * gateway while ordinary credential-free sources continue to work. - */ -export function createApiSourceSecretStore( - storage: KnowledgeGatewayOptions["adapter"]["objectStorage"], - env: SourceSecretEnv = process.env, -): SourceSecretStore | undefined { - const rawKey = env.KNOWLEDGE_SOURCE_SECRET_KEY?.trim(); - if (!rawKey) { - return undefined; - } - return createEncryptedObjectSourceSecretStore({ - encryptionKey: parseSourceSecretEncryptionKey(rawKey), - maxSecretBytes: positiveInteger( - env.KNOWLEDGE_SOURCE_SECRET_MAX_BYTES, - 64 * 1024, - "KNOWLEDGE_SOURCE_SECRET_MAX_BYTES", - ), - objectKeyPrefix: env.KNOWLEDGE_SOURCE_SECRET_PREFIX?.trim() || "__knowledge-secrets/source/v1/", - storage, - }); -} - -function positiveInteger(value: string | undefined, fallback: number, name: string): number { - if (value === undefined || value.trim() === "") { - return fallback; - } - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < 1) { - throw new Error(`${name} must be a positive safe integer`); - } - return parsed; -} diff --git a/knowledge-fs/apps/api/src/standalone-datasource-invocation-client.test.ts b/knowledge-fs/apps/api/src/standalone-datasource-invocation-client.test.ts deleted file mode 100644 index ef53b8e74f3..00000000000 --- a/knowledge-fs/apps/api/src/standalone-datasource-invocation-client.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { Source } from "@knowledge/core"; -import type { - PluginDaemonDatasourceClient, - PluginDaemonDatasourceInput, -} from "@knowledge/plugin-daemon-client"; -import { describe, expect, it } from "vitest"; - -import { createStandaloneDatasourceInvocationClient } from "./standalone-datasource-invocation-client"; - -const SOURCE: Source = { - createdAt: "2026-07-03T00:00:00.000Z", - id: "00000000-0000-4000-8000-000000000001", - knowledgeSpaceId: "10000000-0000-4000-8000-000000000001", - metadata: { - credentials: { api_key: "standalone-secret" }, - datasource: "crawl", - parameters: { limit: 5 }, - pluginId: "langgenius/firecrawl_datasource", - provider: "firecrawl", - }, - name: "Standalone crawl", - permissionScope: [], - status: "active", - type: "web", - updatedAt: "2026-07-03T00:00:00.000Z", - uri: "https://example.com", - version: 1, -}; - -describe("createStandaloneDatasourceInvocationClient", () => { - it("preserves the legacy direct daemon contract only in the standalone adapter", async () => { - const calls: PluginDaemonDatasourceInput[] = []; - const daemon: PluginDaemonDatasourceClient = { - dispatchDatasourceStream(input) { - calls.push(input); - return chunks({ result: { status: "completed" } }); - }, - }; - const adapter = createStandaloneDatasourceInvocationClient({ client: daemon }); - - await expect( - collect( - adapter.dispatch({ - operation: "get_website_crawl", - source: SOURCE, - tenantId: "tenant-1", - userId: "user-1", - }), - ), - ).resolves.toEqual([{ result: { status: "completed" } }]); - expect(calls).toEqual([ - { - data: { - credentials: { api_key: "standalone-secret" }, - datasource: "crawl", - datasource_parameters: { limit: 5, url: "https://example.com" }, - provider: "firecrawl", - }, - method: "get_website_crawl", - pluginId: "langgenius/firecrawl_datasource", - tenantId: "tenant-1", - userId: "user-1", - }, - ]); - }); -}); - -async function* chunks(...values: unknown[]): AsyncGenerator { - for (const value of values) yield value; -} - -async function collect(input: AsyncIterable): Promise { - const values: unknown[] = []; - for await (const value of input) values.push(value); - return values; -} diff --git a/knowledge-fs/apps/api/src/standalone-datasource-invocation-client.ts b/knowledge-fs/apps/api/src/standalone-datasource-invocation-client.ts deleted file mode 100644 index f505ac30ff2..00000000000 --- a/knowledge-fs/apps/api/src/standalone-datasource-invocation-client.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { - readOnlineDocumentSourceConfig, - readOnlineDriveSourceConfig, - readSourceCredentialConfig, - readWebsiteCrawlSourceConfig, -} from "@knowledge/api"; -import type { PluginDaemonDatasourceClient } from "@knowledge/plugin-daemon-client"; - -import type { ApiDatasourceInvocationClient } from "./datasource-invocation-client"; - -/** Legacy standalone adapter. It is the only application adapter allowed to receive source secrets. */ -export function createStandaloneDatasourceInvocationClient(input: { - readonly client: PluginDaemonDatasourceClient; -}): ApiDatasourceInvocationClient { - return { - dispatch(invocation) { - switch (invocation.operation) { - case "get_website_crawl": { - const config = readWebsiteCrawlSourceConfig(invocation.source); - return input.client.dispatchDatasourceStream({ - data: { - credentials: config.credentials, - datasource: config.datasource, - datasource_parameters: config.parameters, - provider: config.provider, - }, - method: invocation.operation, - pluginId: config.pluginId, - tenantId: invocation.tenantId, - ...(invocation.userId ? { userId: invocation.userId } : {}), - ...(invocation.signal ? { signal: invocation.signal } : {}), - }); - } - case "get_online_document_pages": { - const config = readOnlineDocumentSourceConfig(invocation.source); - return input.client.dispatchDatasourceStream({ - data: { - credentials: config.credentials, - datasource: config.datasource, - datasource_parameters: { - ...config.parameters, - ...(invocation.cursor === undefined ? {} : { cursor: invocation.cursor }), - ...(invocation.limit === undefined ? {} : { limit: invocation.limit }), - }, - provider: config.provider, - }, - method: invocation.operation, - pluginId: config.pluginId, - tenantId: invocation.tenantId, - ...(invocation.userId ? { userId: invocation.userId } : {}), - ...(invocation.signal ? { signal: invocation.signal } : {}), - }); - } - case "get_online_document_page_content": { - const config = readOnlineDocumentSourceConfig(invocation.source); - return input.client.dispatchDatasourceStream({ - data: { - credentials: config.credentials, - datasource: config.datasource, - page: { - page_id: invocation.page.pageId, - type: invocation.page.type, - workspace_id: invocation.page.workspaceId, - }, - provider: config.provider, - }, - method: invocation.operation, - pluginId: config.pluginId, - tenantId: invocation.tenantId, - ...(invocation.userId ? { userId: invocation.userId } : {}), - ...(invocation.signal ? { signal: invocation.signal } : {}), - }); - } - case "online_drive_browse_files": { - const config = readOnlineDriveSourceConfig(invocation.source); - return input.client.dispatchDatasourceStream({ - data: { - credentials: config.credentials, - datasource: config.datasource, - provider: config.provider, - request: { - ...(invocation.bucket === undefined ? {} : { bucket: invocation.bucket }), - ...(invocation.continuationToken === undefined - ? {} - : { continuation_token: invocation.continuationToken }), - max_keys: invocation.maxKeys ?? 20, - prefix: invocation.prefix ?? "", - }, - }, - method: invocation.operation, - pluginId: config.pluginId, - tenantId: invocation.tenantId, - ...(invocation.userId ? { userId: invocation.userId } : {}), - ...(invocation.signal ? { signal: invocation.signal } : {}), - }); - } - case "online_drive_download_file": { - const config = readOnlineDriveSourceConfig(invocation.source); - return input.client.dispatchDatasourceStream({ - data: { - credentials: config.credentials, - datasource: config.datasource, - provider: config.provider, - request: { bucket: invocation.file.bucket ?? "", id: invocation.file.id }, - }, - method: invocation.operation, - pluginId: config.pluginId, - tenantId: invocation.tenantId, - ...(invocation.userId ? { userId: invocation.userId } : {}), - ...(invocation.signal ? { signal: invocation.signal } : {}), - }); - } - case "validate_credentials": { - const config = readSourceCredentialConfig(invocation.source); - return input.client.dispatchDatasourceStream({ - data: { credentials: config.credentials, provider: config.provider }, - method: invocation.operation, - pluginId: config.pluginId, - tenantId: invocation.tenantId, - ...(invocation.userId ? { userId: invocation.userId } : {}), - ...(invocation.signal ? { signal: invocation.signal } : {}), - }); - } - } - }, - }; -} diff --git a/knowledge-fs/apps/api/src/visual-embedding-options.test.ts b/knowledge-fs/apps/api/src/visual-embedding-options.test.ts index 1b1efa94693..ca9257974d2 100644 --- a/knowledge-fs/apps/api/src/visual-embedding-options.test.ts +++ b/knowledge-fs/apps/api/src/visual-embedding-options.test.ts @@ -13,7 +13,7 @@ const PLUGIN_ENV = { KNOWLEDGE_VISUAL_EMBEDDING_MODEL: "clip-multimodal", KNOWLEDGE_VISUAL_EMBEDDING_PLUGIN_ID: "langgenius/clip", KNOWLEDGE_VISUAL_EMBEDDING_PLUGIN_PROVIDER: "clip", - KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER: "plugin-daemon", + KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER: "dify-model-runtime", } as const; describe("createApiVisualEmbeddingOptions", () => { @@ -185,7 +185,7 @@ describe("createApiVisualEmbeddingOptions", () => { const adapter = createNodePlatformAdapter({ env: {} }); expect(() => createApiVisualEmbeddingOptions({ - env: { KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER: "plugin-daemon" }, + env: { KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER: "dify-model-runtime" }, objectStorage: adapter.objectStorage, }), ).toThrow("KNOWLEDGE_VISUAL_EMBEDDING_MODEL is required for visual embeddings"); @@ -193,7 +193,7 @@ describe("createApiVisualEmbeddingOptions", () => { createApiVisualEmbeddingOptions({ env: { KNOWLEDGE_VISUAL_EMBEDDING_MODEL: "clip-multimodal", - KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER: "plugin-daemon", + KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER: "dify-model-runtime", }, objectStorage: adapter.objectStorage, }), diff --git a/knowledge-fs/apps/api/src/visual-embedding-options.ts b/knowledge-fs/apps/api/src/visual-embedding-options.ts index d881660c0a4..7873d7336a8 100644 --- a/knowledge-fs/apps/api/src/visual-embedding-options.ts +++ b/knowledge-fs/apps/api/src/visual-embedding-options.ts @@ -38,7 +38,7 @@ export interface ApiVisualEmbeddingOptions { /** * Resolves the image-byte visual embedding provider. Opt-in: returns `undefined` unless - * `KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER=dify-model-runtime` (legacy `plugin-daemon` is accepted). + * `KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER=dify-model-runtime`. * * Mirrors dify's multimodal RAG split exactly: * - image bytes route through Dify's multimodal embedding model instance @@ -207,13 +207,11 @@ function visualEmbeddingEnabled(value: string | undefined): boolean { return false; } - if (normalized === "dify-model-runtime" || normalized === "plugin-daemon") { + if (normalized === "dify-model-runtime") { return true; } - throw new Error( - "KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER must be dify-model-runtime, plugin-daemon, or off", - ); + throw new Error("KNOWLEDGE_VISUAL_EMBEDDING_PROVIDER must be dify-model-runtime or off"); } function normalizedQueryMode(value: string | undefined): "fallback" | "off" | "primary" { diff --git a/knowledge-fs/apps/api/vitest.config.ts b/knowledge-fs/apps/api/vitest.config.ts index 8fb6f2dcff4..d6f12d8df0b 100644 --- a/knowledge-fs/apps/api/vitest.config.ts +++ b/knowledge-fs/apps/api/vitest.config.ts @@ -1,3 +1,9 @@ +import { resolve } from "node:path"; + import { defineConfig } from "vitest/config"; -export default defineConfig({}); +export default defineConfig({ + test: { + setupFiles: [resolve(import.meta.dirname, "../../test/setup-dify-object-storage.ts")], + }, +}); diff --git a/knowledge-fs/docs/api-reference.md b/knowledge-fs/docs/api-reference.md index 0d2be23d678..b12e37d2285 100644 --- a/knowledge-fs/docs/api-reference.md +++ b/knowledge-fs/docs/api-reference.md @@ -72,7 +72,7 @@ the live spec and this companion document differ. **Auth**: Bearer; scope `knowledge-spaces:read`. **Path params**: `id` (uuid). **Responses**: -- `200`: `KnowledgeSpaceManifest` — `{ id, knowledgeSpaceId, tenantId, manifestVersion: int, embeddingProfile?: { pluginId, provider, model, vectorSpaceId, revision, dimension? }, embeddingProfileFrozenAt?: datetime, minClientVersion, nodeSchemaVersion: int, parserPolicyVersion, projectionSetVersion, objectKeyPrefix, metadataDialect: enum(portable|postgres|tidb), storageProvider: enum(memory-dev|r2|s3-compatible), consistencyPolicy: { defaultClass: enum(path-consistent|snapshot-consistent|cache-consistent|eventual-preview), snapshotTtlSeconds, cacheTtlSeconds? }, encryptionPolicy: { strategy: enum(provider-managed|customer-managed|none), keyRef? }, retentionPolicy: { artifactVersionsToKeep, failedCommitRetentionDays, traceRetentionDays }, quotaPolicy: { maxActiveJobCount, maxActiveSessionCount, maxArtifactBytes, maxGraphEntityCount, maxGraphRelationCount, maxNodeCount, maxProjectionCount, maxRawDocumentBytes, maxSegmentCount, maxTraceBytes: int|null, providerBudgets: { maxEmbeddingTokensPerDay, maxLlmTokensPerDay, maxParserPagesPerDay, maxRerankRequestsPerDay: int|null } }, metadata, createdAt, updatedAt }`. `embeddingProfileFrozenAt` is set atomically when the first document ingestion is admitted. +- `200`: `KnowledgeSpaceManifest` — `{ id, knowledgeSpaceId, tenantId, manifestVersion: int, embeddingProfile?: { pluginId, provider, model, vectorSpaceId, revision, dimension? }, embeddingProfileFrozenAt?: datetime, minClientVersion, nodeSchemaVersion: int, parserPolicyVersion, projectionSetVersion, objectKeyPrefix, metadataDialect: enum(portable|postgres|tidb), storageProvider: enum(dify|memory-dev|r2|s3-compatible), consistencyPolicy: { defaultClass: enum(path-consistent|snapshot-consistent|cache-consistent|eventual-preview), snapshotTtlSeconds, cacheTtlSeconds? }, encryptionPolicy: { strategy: enum(provider-managed|customer-managed|none), keyRef? }, retentionPolicy: { artifactVersionsToKeep, failedCommitRetentionDays, traceRetentionDays }, quotaPolicy: { maxActiveJobCount, maxActiveSessionCount, maxArtifactBytes, maxGraphEntityCount, maxGraphRelationCount, maxNodeCount, maxProjectionCount, maxRawDocumentBytes, maxSegmentCount, maxTraceBytes: int|null, providerBudgets: { maxEmbeddingTokensPerDay, maxLlmTokensPerDay, maxParserPagesPerDay, maxRerankRequestsPerDay: int|null } }, metadata, createdAt, updatedAt }`. `embeddingProfileFrozenAt` is set atomically when the first document ingestion is admitted. New spaces use `dify`; the other storage values are retained only to decode legacy manifests during coexistence. - `404`; `401`/`403`. ### `PUT /knowledge-spaces/{id}/embedding-profile` @@ -262,7 +262,8 @@ knowledge-space retrieval profile's `defaultMode`, or `fast` for a legacy space optional, default `[]`); `sessionId` (uuid, optional). `auto` is a public **routing selector**, not a fourth retrieval pipeline. Only an explicit -`mode: "auto"` invokes the knowledge space's published `reasoningModel` through plugin-daemon to +`mode: "auto"` invokes the knowledge space's published `reasoningModel` through Dify's model +runtime to select exactly one concrete pipeline. Omitting `mode` does not invoke the router; it uses the published `defaultMode`. Explicit `fast`, `research`, and `deep` requests also bypass routing. diff --git a/knowledge-fs/docs/operator-manual.md b/knowledge-fs/docs/operator-manual.md index 2d16855fae6..1d82d7916b4 100644 --- a/knowledge-fs/docs/operator-manual.md +++ b/knowledge-fs/docs/operator-manual.md @@ -1,17 +1,19 @@ # KnowledgeFS Operator Manual -This manual is for people running KnowledgeFS in development, staging, or production. It complements the API reference and deployment guide with daily operating procedures, quality gates, incident response, and performance guardrails. +This manual is for people running KnowledgeFS as an internal Dify backend in development, +staging, or production. KnowledgeFS has no independent deployment mode. ## Operating Model -KnowledgeFS is split into independently observable services: +KnowledgeFS is an internal Dify service with independently observable dependencies: | Service | Responsibility | |---|---| | Admin Console | Human workflows, upload/evaluation dashboards, Retrieval Studio, trace diagnostics. | | Hono API | Auth, ingestion, retrieval, KnowledgeFS, queries, evaluation routes, traces, MCP tools. | | Database | Tenant-scoped metadata, generated artifacts, nodes, projections, traces, evaluation data. | -| Object storage | Raw uploaded document bytes. | +| Dify inner API | Model instances, datasource plugins, and unified object storage. | +| Object storage | Dify-owned raw uploaded document bytes. | | Parser service | Unstructured-compatible parsing for complex document formats. | | Queue runtime | Async document compilation, bulk jobs, cleanup, and research work when configured. | | TypeScript compute | Pure bounded compute: chunking, token counting, RRF, packing, diff. | @@ -28,7 +30,7 @@ curl -fsS "$API/openapi.json" >/dev/null pnpm eval:regression ``` -For Standalone environments: +For the local developer harness only: ```bash docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps ps @@ -38,7 +40,8 @@ docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml - Expected health: - API returns healthy platform adapter status. -- Parser, embedding, LLM, reranker, object storage, database, cache, and job components are either healthy or explicitly marked unavailable for the environment. +- Dify model, datasource, and object-storage configuration is healthy. +- Parser, database, cache, and enabled job components are healthy. - Retrieval regression gate passes recall, citation-hit, no-answer, citation accuracy, and faithfulness thresholds. ## Release Checklist @@ -58,11 +61,12 @@ git diff --check ``` `docker:api:bundle-smoke` deliberately starts the built API bundle with `NODE_ENV=test`. It proves -that the container can boot, serve `/health`, and report `components.compute === true`; it does not -exercise production fail-closed startup, database repositories, durable compilation, object -storage, or providers. Production promotion still requires the deployed/Compose-backed health and -tenant-scoped upload/query checks below. The legacy `docker:api:http-smoke` command is only an alias -for this isolated check. +that the container can boot and serve `/health`. It also requires `ok === false` and +`components.objectStorage === false`, proving that an isolated container stays unhealthy without +Dify instead of falling back to standalone storage. It does not exercise production database +repositories, durable compilation, Dify object storage, or providers. Production promotion still +requires the Dify-connected health and tenant-scoped upload/query checks below. The legacy +`docker:api:http-smoke` command is only an alias for this isolated check. Confirm: @@ -86,7 +90,8 @@ Operational rules: - Never put `tenantId` in client requests expecting it to be trusted. - Treat cross-tenant 404s as expected behavior. -- Rotate `AUTH_JWT_SECRET` or provider secrets through the environment secret manager, not through committed files. +- Rotate Dify capability verification material through the environment secret manager; provider + secrets remain in Dify and must never be copied into KnowledgeFS. - Never log bearer tokens. ## Ingestion Operations @@ -231,7 +236,9 @@ Object storage: - Raw documents are stored under tenant/space/document prefixes. - Object metadata includes asset id, KnowledgeSpace id, tenant id, hash, and uploader when available. -- Production should use S3-compatible object storage, MinIO, or R2. Bounded memory storage is for development only. +- Dify integrated mode reaches Dify's configured unified storage through the authenticated inner + API and must not receive separate provider credentials. +- KnowledgeFS must not connect directly to an object store or accept object-storage credentials. Retention: @@ -249,7 +256,7 @@ Treat performance regressions as correctness failures: - Every database read path needs an explicit `maxRows` or route-level limit. - Avoid N+1 queries; prefer repository methods that join or batch required data. - Cache keys must include tenant, subject or permission snapshot, strategy, model, and index versions where relevant. -- Queue, retention, in-memory fallback, and Admin diagnostic surfaces must keep explicit max sizes. +- Queue, retention, test adapters, and diagnostic surfaces must keep explicit max sizes. - Never add a hot path that fetches object storage bytes after upload when bytes are already in memory. ## Incident Response diff --git a/knowledge-fs/docs/production-deployment.md b/knowledge-fs/docs/production-deployment.md index 2ecfa5489a8..61a4b6f5048 100644 --- a/knowledge-fs/docs/production-deployment.md +++ b/knowledge-fs/docs/production-deployment.md @@ -1,521 +1,153 @@ -# Production Deployment Guide +# KnowledgeFS Production Deployment -This guide covers the two supported production shapes for KnowledgeFS: +KnowledgeFS has one supported production topology: an internal backend service deployed as part of +Dify. It has no independent SaaS, private-cloud, or single-host deployment mode. -- **SaaS:** Cloudflare Pages for the Next.js Admin Console, Cloudflare Workers for the Hono Knowledge Gateway, R2 for object storage, KV for cache/session state, TiDB Cloud for relational search/index data, and Unstructured API for complex document parsing. -- **Standalone:** Docker Compose or equivalent orchestration with a separate Admin service, Hono API service, PostgreSQL + pgvector, MinIO, Redis or bounded in-memory cache, and Unstructured API. +The existing Dify knowledge-base feature and KnowledgeFS intentionally coexist during rollout. +This deployment does not migrate, replace, or delete existing Dataset/Document data. -The deployment boundary is intentional: Next.js owns only the human Admin Console and thin UI BFF routes, while Hono owns all platform APIs, retrieval, ingestion, KnowledgeFS, MCP, auth, provider orchestration, and persistence. +## Dependency ownership -> **Current promotion gate:** the production API entrypoint assembles the Capability v2 verifier -> only from an explicitly enabled, public-only JWKS profile and otherwise fails closed. `/ready` -> must return `200` with all durable dependencies configured before any traffic is enabled; a -> `503` is a stop signal, not an expected production steady state. The Docker and Kubernetes -> artifacts described below remain an inert deployment baseline until the migration and cutover -> gates are completed. +| Capability | Owner | KnowledgeFS access | +|---|---|---| +| Model configuration and credentials | Dify model manager / Plugin Daemon | Dify inner model API | +| Datasource configuration, OAuth, and credentials | Dify datasource plugins | Dify inner datasource API | +| Physical object storage | Dify `STORAGE_TYPE` implementation | Dify inner storage API | +| KnowledgeFS relational state | KnowledgeFS database | `DATABASE_URL` | +| Complex document parsing | Unstructured-compatible service | `UNSTRUCTURED_API_URL` | +| Capability signing | Dify | Public JWKS only in KnowledgeFS | -Related operating documents: +KnowledgeFS must never receive model-provider keys, datasource secrets, direct Plugin Daemon +credentials, or object-storage provider credentials. -- [API reference](api-reference.md) for route, scope, and error semantics. -- [Operator manual](operator-manual.md) for daily checks, release process, incident response, rollback, observability, and performance guardrails. -- [Dify integration alert runbook](../../docs/design/knowledge-fs-integration-alert-runbook.md) for the seven cross-service authorization, lifecycle, upload, stream, deletion, and shadow alerts. +## Dify Compose -## Release Gates +The canonical service definitions are: -Run the full verification set before promoting a release candidate: +- `docker/docker-compose.yaml` +- `docker/docker-compose-template.yaml` +- `docker/envs/core-services/knowledge-fs.env.example` + +The Compose service: + +- starts by default with the rest of Dify; +- builds `knowledge-fs/apps/api/Dockerfile` when a prebuilt image is unavailable; +- remains on the internal `default` network and exposes only port `8787` to peer services; +- receives `DIFY_INNER_API_URL=http://api:5001`; +- receives the same inner API key used by Dify's plugin boundary; +- waits for the Dify API and its database dependency; +- uses `/health` for liveness and `/ready` for traffic readiness. + +`KNOWLEDGE_INTEGRATED_MODE_ENABLED` controls Workspace provisioning/cutover behavior only. Whether +it is `false` or `true`, model, datasource, and object-storage calls always go through Dify. + +## Operator-owned environment + +The tracked KnowledgeFS environment example intentionally contains only settings that belong to +the service: + +| Variable | Purpose | +|---|---| +| `DATABASE_URL` | KnowledgeFS PostgreSQL connection string. | +| `KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME` | Durable document worker rollout. | +| `KNOWLEDGE_FS_CAPABILITY_V2_ENABLED` | Capability-v2 verifier rollout. | +| `KNOWLEDGE_FS_CAPABILITY_V2_PUBLIC_JWKS` | Public verification key set issued by Dify. | +| `UNSTRUCTURED_API_URL` | Parser endpoint for complex formats. | +| `UNSTRUCTURED_API_KEY` | Optional parser authentication. | + +Compose injects `DIFY_INNER_API_URL` and `DIFY_INNER_API_KEY`; do not duplicate them in the +operator-owned env file. Do not add `MINIO_*`, cloud object-storage credentials, provider API keys, +`PLUGIN_DAEMON_*`, datasource tokens, or OAuth client secrets. + +## Database release + +Apply checked-in KnowledgeFS migrations through the controlled migration runner before scaling a +new binary: + +```bash +pnpm db:migrations:check +pnpm local:db:migrate +``` + +Use the environment's normal migration job in production rather than running the local command +from an application container. The KnowledgeFS migration runner owns only KnowledgeFS tables. It +must not mutate existing Dify Dataset/Document tables or perform a production data migration. + +Keep destructive legacy-removal flags disabled until the separately approved zero-traffic, +backup/restore, DBA, and CAB gates are complete. + +## Readiness contract + +Production `/ready` fails closed unless all enabled capabilities are assembled. The base checks +include: + +- an authentication verifier; +- Dify model-runtime configuration; +- Dify datasource-runtime configuration; +- Dify object-storage configuration; +- durable database repositories required by enabled workers and product routes. + +`/health` is liveness and component diagnostics; it is not permission to receive production +traffic. A service with `/health=200` and `/ready=503` must remain out of rotation. + +Direct upload remains disabled because the Dify storage bridge deliberately does not expose +provider-specific presign or multipart primitives. Upload bytes pass through the bounded +KnowledgeFS API and Dify inner storage API. + +## Release validation + +Before publishing an image: ```bash pnpm install --frozen-lockfile -pnpm check -pnpm build -pnpm lint -pnpm compose:config -docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml --profile apps config +pnpm typecheck +pnpm test +pnpm lint:backend +pnpm openapi:export:test +pnpm db:migrations:check +pnpm dify:compose:config pnpm docker:api:build pnpm docker:api:bundle-smoke git diff --check ``` -`docker:api:bundle-smoke` is an isolated image-bundle gate. It overrides the container to -`NODE_ENV=test`, checks `/health`, and requires `components.compute === true`. It does **not** -validate production fail-closed startup or the configured database, durable compilation, object -storage, parser, datasource plugin-daemon, and Dify model-runtime dependencies. Treat the deployed SaaS or -Standalone smoke flows later in this guide as the production configuration gate. +The isolated bundle smoke is not a production dependency test. In a Dify-connected environment, +also verify: -Optional live storage smoke for Standalone object storage: +1. `/health` and `/ready`. +2. A tenant-scoped KnowledgeSpace create/read. +3. A bounded document upload and object read through Dify storage. +4. Embedding, rerank, LLM, and model-catalog calls through Dify model instances. +5. Datasource validation/browse through a Dify-managed `credentialId`. +6. No model, datasource, OAuth, Plugin Daemon, or storage credentials appear in KnowledgeFS + environment variables, requests, logs, or database rows. +7. Existing Dify knowledge-base flows remain unchanged. -```bash -docker compose --env-file infra/local/.env -f infra/local/compose.yaml up -d minio minio-bootstrap -pnpm test:minio -``` +## Workspace rollout -The bounded compute runtime is ordinary TypeScript bundled with the API; no generated runtime -artifact or language-specific toolchain is required during deployment. +Roll out Workspace by Workspace. Keep the integrated-mode/capability flags disabled by default, +then enable only after the selected Workspace has: -## Runtime Configuration +- durable KnowledgeFS provisioning state; +- capability verification; +- successful model, datasource, and storage smoke; +- rollback evidence and monitoring ownership. -Shared API settings: +The rollout flag changes admission and provisioning behavior. It does not switch transports and +does not authorize a fallback runtime. -| Variable | Required | Purpose | -|---|---:|---| -| `NODE_ENV` | Yes | Use `production` for deployed services. | -| `PORT` | Standalone API | Hono API port, default `8787` locally. | -| `KNOWLEDGE_DEV_AUTH_TOKEN` | Development/test only | Static local verifier token. It is ignored in production and must not be treated as a production credential. | -| `DIFY_INNER_API_URL`, `DIFY_INNER_API_KEY` | Integrated production baseline | Dify API inner endpoint and key matching `INNER_API_KEY_FOR_PLUGIN`. Model and datasource calls use this boundary; KnowledgeFS never receives provider credential bytes. Missing values keep `/ready` closed. | -| `DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES`, `DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS` | Optional | Bounds integrated datasource response size and request duration. | -| `PLUGIN_DAEMON_URL`, `PLUGIN_DAEMON_KEY` | Standalone only | Legacy direct datasource transport. It is ignored when `KNOWLEDGE_INTEGRATED_MODE_ENABLED=true`. | -| `UNSTRUCTURED_API_URL` | Ingestion for complex files | Base URL for Unstructured API. | -| `UNSTRUCTURED_API_KEY` | SaaS or protected Unstructured | Optional API key sent by the parser client. | +## Kubernetes -Object storage: - -| Target | Variables | -|---|---| -| MinIO / S3-compatible | `MINIO_ENDPOINT`, `MINIO_BUCKET`, `MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, optional `MINIO_REGION` | -| Cloudflare R2 | `R2_ACCOUNT_ID`, `R2_BUCKET`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, optional `R2_REGION` | - -Database and cache: - -| Target | Variables | -|---|---| -| PostgreSQL standalone | `DATABASE_URL` pointing to PostgreSQL + pgvector. | -| TiDB Cloud SaaS | `DATABASE_URL` or future TiDB serverless binding once runtime driver wiring is introduced. | -| Cache/session state | Current runtime uses adapter-backed cache. Use KV/Redis wiring when available; otherwise bounded in-memory fallback is development-only. | - -Durable document compilation is controlled by `KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME` and is off -by default. Setting it to `on`, `true`, or `1` assembles the database attempt/outbox control plane, -candidate-only worker/evaluator, publication coordinator, dispatcher, and runtime consumer. Startup -fails closed unless database repositories, the compute runtime, and the per-knowledge-space plugin -embedding resolver are all available; it never falls back to the legacy in-memory writer. The API -and compilation consumer currently run in the same process. The remaining bounded settings are -`KNOWLEDGE_DOCUMENT_COMPILATION_BATCH_SIZE`, `KNOWLEDGE_DOCUMENT_COMPILATION_LEASE_MS`, -`KNOWLEDGE_DOCUMENT_COMPILATION_MAX_ATTEMPTS`, -`KNOWLEDGE_DOCUMENT_COMPILATION_OUTBOX_VISIBILITY_MS`, -`KNOWLEDGE_DOCUMENT_COMPILATION_RETRY_BASE_MS`, -`KNOWLEDGE_DOCUMENT_COMPILATION_RETRY_MAX_MS`, and -`KNOWLEDGE_DOCUMENT_COMPILATION_TICK_MS`. - -When database repositories are enabled, `KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME=on` is mandatory: -startup rejects the unsafe synchronous-upload combination because it can only create legacy -`NULL`-generation rows and no immutable publication head. `NODE_ENV=production` also rejects the -process-local repository fallback. Migration `0009_legacy_space_bootstrap` installs a fail-closed -ledger for every pre-cutover space. The runtime freezes a bounded document/version/SHA-256 set, -rebuilds one document generation at a time, verifies the final publication, ready flattened -PageIndex and FTS/Graph ownership closure, and only then opens query readiness. Intermediate child -heads are intentionally unavailable (queries return 503); upload, delete, reindex, and ordinary -compilation remain fenced (409) until completion. Operators can inspect/start/retry the tenant- -scoped ledger at `/knowledge-spaces/{id}/publication-bootstrap` and -`/knowledge-spaces/{id}/publication-bootstrap/retry`. Never delete or bypass a failed ledger. -Document writes also hold a durable, space-exclusive mutation lease across object and metadata -writes so snapshot capture cannot interleave after admission. These leases intentionally have no -automatic expiry: after a writer crash, prove the process has stopped and reconcile its staged -commit/object state before manually removing an orphan lease; time-based eviction is unsafe. - -Admin Console: - -| Variable | Required | Purpose | -|---|---:|---| -| `NEXT_PUBLIC_API_BASE_URL` | Yes | Browser-visible HTTPS URL for the Hono API. Plain HTTP is development/loopback-only. | - -Dify direct query, Research SSE, upload control, and presigned object URLs are also HTTPS-only in -production browsers, except for explicit loopback development. Terminate TLS before exposing any -Capability-bearing route or object-upload URL. - -## SaaS Deployment - -### 1. Provision services - -Provision the SaaS backing services before deploying code: - -1. Cloudflare R2 bucket for document objects. -2. Cloudflare KV namespace for cache/session state once KV runtime wiring is enabled. -3. TiDB Cloud database for relational tables, FTS-like indexes, retrieval metadata, traces, and generated artifacts. -4. Unstructured API endpoint for PDF/DOCX/PPTX and other complex formats. -5. Cloudflare Pages project for `apps/admin`. -6. Cloudflare Workers project for the Hono API runtime. - -Run migration drift checks before applying any database migration: - -```bash -pnpm db:migrations:check -``` - -The checked-in artifacts live in `packages/database/migrations`. Apply the TiDB artifact to the SaaS database only through the controlled database release process for the environment. - -Migration `0017_durable_deletion` adds permanent writer tombstones and the checkpointed -Space/Source/Document deletion ledger. Deploy it in two phases. First, apply `0017` and roll out -the new API/worker build to every writer with `DURABLE_DELETION_ENABLED=off`. Verify that no older -writer remains. Only then set all of the following on the API deployment and restart it: - -```bash -DURABLE_DELETION_ENABLED=true -DURABLE_DELETION_WRITER_FENCE_VERSION=0017 -DURABLE_DELETION_HMAC_KEY_BASE64= -``` - -The HMAC key is part of the retained deletion/idempotency audit contract. Keep it stable for as -long as deletion jobs, retry audits, or idempotency ledgers are retained; do not silently rotate -it. Enabling deletion without the exact writer-fence declaration, database repositories, secret -cleanup capability, or a valid key fails startup. Leaving the gate off keeps destructive routes -unavailable while all read/write paths still honor any existing tombstones. - -`0017` also adds nullable tenant/space ownership to historical `evidence_bundles`. The migration -backfills only bundles whose AnswerTrace/Research references agree on exactly one scope; ambiguous -rows stay quarantined with NULL ownership. Before enabling deletion, run the bounded maintenance -operation `purgeUnscopedEvidenceBundlesPage` until it returns zero, then retain this zero-result -audit with the release evidence: - -```sql -SELECT COUNT(*) AS unscoped_evidence_bundles -FROM evidence_bundles -WHERE tenant_id IS NULL OR knowledge_space_id IS NULL; -``` - -Startup repeats this readiness check whenever durable deletion is enabled and fails closed if the -count is nonzero. Do not assign an ambiguous bundle to a guessed space; purge it and let scoped -writers recreate future evidence under the exact tenant/space boundary. - -Migration `0005_publication_generation_nonzero` requires TiDB 7.2 or newer with -`tidb_enable_check_constraint` enabled. The migration runner verifies both conditions before it -executes the migration and fails closed when CHECK constraints would not be enforced. Configure -the database cluster accordingly before starting the release. - -Migration `0006_document_compilation_attempts` additionally requires -TiDB 8.5 or newer, `@@GLOBAL.tidb_enable_foreign_key=ON`, and -`@@SESSION.foreign_key_checks=ON`. Version 8.5 is the first release where TiDB foreign keys are -generally available. The runner validates these conditions before DDL, rejects historical -Knowledge Space tenant IDs longer than 255 characters before narrowing the column, and checks -`SHOW CREATE TABLE` afterward so TiDB cannot silently retain `FOREIGN KEY INVALID` declarations. -Run migrations through this runner rather than executing the TiDB artifact in a session with -different constraint settings. - -Migration `0012_tidb_baseline_repair` is a mandatory forward repair for TiDB environments. Before -the first supported production release, the clean-install TiDB artifacts `0001`, `0002`, `0004`, -`0006`, and `0007` were corrected in place to remove unsupported TEXT/JSON/expression/FULLTEXT key -definitions and CHECK/foreign-key combinations. An experimental environment may already have -recorded those migration IDs, so changing only the historical files cannot repair that database. -`0012` reapplies every material type, generated-column, index, CHECK, and compilation foreign-key -correction under a new immutable ID. Its PostgreSQL pair is an intentional no-op. - -Take a schema snapshot and a normal database backup before applying `0012`. The migration performs -no destructive data cleanup: an overlong value, duplicate logical identity, or orphaned foreign-key -row aborts DDL and leaves `0012` unrecorded. Reconcile the reported data explicitly and rerun; do -not truncate values, delete an arbitrary duplicate, disable CHECK constraints, or turn off foreign -keys to force the release through. After the runner succeeds, retain these audit results with the -release evidence: - -```sql -SELECT migration_id, dialect, applied_at -FROM schema_migrations -WHERE migration_id = '0012_tidb_baseline_repair' AND dialect = 'tidb'; - -SHOW CREATE TABLE index_projections; -SHOW CREATE TABLE knowledge_nodes; -SHOW CREATE TABLE document_compilation_attempts; -SHOW CREATE TABLE document_compilation_outbox; - -SELECT table_name, column_name, column_type, extra, generation_expression -FROM information_schema.columns -WHERE table_schema = DATABASE() - AND column_name IN ('model_key', 'publication_generation_key') -ORDER BY table_name, column_name; - -SELECT table_name, index_name -FROM information_schema.statistics -WHERE table_schema = DATABASE() - AND ( - index_name = '' - OR index_name IN ( - 'resource_mounts_permission_scope_idx', - 'knowledge_nodes_permission_scope_idx', - 'index_projections_fts_document_idx', - 'graph_entities_permission_scope_idx', - 'graph_relations_permission_scope_idx' - ) - ); -``` - -The audit must show `model_key` and every `publication_generation_key` as virtual generated -columns; generated-column-backed identity indexes; no permission-scope JSON or `fts_document` -FULLTEXT index (the final audit query must return no rows); exactly the three named attempt foreign -keys and one named outbox foreign key from -`0012`; and none of -`document_compilation_attempts_document_version_ck`, -`document_compilation_attempts_candidate_pair_ck`, or -`document_compilation_attempts_candidate_checkpoint_ck`. The runner independently validates the -foreign-key names/count and rejects `FOREIGN KEY INVALID` after all pending migrations. - -### 2. Build the Hono API for Workers - -The repository currently has the portable Hono gateway and adapter contracts, but it does not yet commit a Workers-specific `wrangler.toml`. Use this guide as the required shape for that later runtime wiring: - -```toml -name = "knowledge-fs-api" -main = "apps/api/src/worker.ts" -compatibility_date = "2026-05-11" - -[[r2_buckets]] -binding = "DOCUMENT_OBJECTS" -bucket_name = "knowledge-fs-documents" - -[[kv_namespaces]] -binding = "KNOWLEDGE_CACHE" -id = "" -``` - -Expected Worker backing-service secrets: - -```bash -wrangler secret put R2_ACCESS_KEY_ID -wrangler secret put R2_SECRET_ACCESS_KEY -wrangler secret put DATABASE_URL -wrangler secret put UNSTRUCTURED_API_KEY -``` - -Deploy only after the Workers entrypoint exists and the release gates pass: - -```bash -pnpm build -wrangler deploy -``` - -The sample does not configure authentication by itself. A production Worker must select the -Capability v2 profile, receive only the public JWKS plus the expected issuer and audience, and -return `200` from `/ready` before promotion. - -### 3. Deploy the Admin Console to Pages - -The Admin Console must call the Hono API rather than importing platform internals. - -Set Pages environment variables: - -```text -NEXT_PUBLIC_API_BASE_URL=https:// -NODE_ENV=production -``` - -Build command: - -```bash -pnpm install --frozen-lockfile -pnpm --filter @knowledge/admin build -``` - -Output mode is currently Next.js standalone-oriented. If deploying to Cloudflare Pages, add the Pages adapter in a dedicated slice and keep all core behavior behind the Hono API. - -### 4. SaaS smoke checks - -After deployment: - -```bash -curl -fsS https:///health -curl -sS -o /dev/null -w '%{http_code}\n' https:///ready -curl -fsS https:///openapi.json -``` - -`/ready` must print `200` before continuing. Stop on `503`; it means the Capability v2 verifier or -another required durable dependency is missing or invalid. Do not run the authenticated product -flow or route traffic while readiness is closed. - -Then verify a tenant-scoped authenticated flow with a non-production test tenant: - -1. Create a KnowledgeSpace. -2. Upload a Markdown or HTML document. -3. Confirm parse status becomes `parsed`. -4. Run a query and verify citations, trace id, and session id headers. -5. Confirm cross-tenant access returns 404 or 403 as appropriate. - -## Standalone Deployment - -### 1. Build images - -The API image is already defined in `apps/api/Dockerfile`: - -```bash -pnpm docker:api:build -``` - -The Admin service currently runs as a Compose development container. For production Standalone, build a separate Admin image from the Next.js standalone output or run the Next server in an equivalent process manager. Keep it as a separate service from the API. - -### 2. Configure Compose - -The local Compose file already models the production service separation: - -- `api`: Hono API service. -- `admin`: Next.js Admin Console. -- `postgres`: PostgreSQL + pgvector. -- `minio`: S3-compatible object storage. -- `minio-bootstrap`: one-shot bucket creation. -- `unstructured`: self-hosted parser service. - -Validate the resolved deployment plan: - -```bash -docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml --profile apps config -``` - -For a production environment, override local defaults with environment-specific secrets: - -```text -DATABASE_URL=postgresql://:@postgres:5432/ -MINIO_ENDPOINT=http://minio:9000 -MINIO_BUCKET=knowledge-fs -MINIO_ACCESS_KEY= -MINIO_SECRET_KEY= -UNSTRUCTURED_API_URL=http://unstructured:8000 -NEXT_PUBLIC_API_BASE_URL=https:// -``` - -Do not commit environment files containing production secrets. - -Production auth is selected explicitly with Capability v2. The KFS process receives only public -JWKS material; Dify keeps the private signing key. Keep every runtime capability disabled while -installing the deployment, then verify `/ready` before enabling any per-Workspace cutover: - -```text -# KFS API: deployment capability only; these values do not authorize a Workspace by themselves. -KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=true -KNOWLEDGE_FS_CAPABILITY_V2_PUBLIC_JWKS= -KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane -KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs -KNOWLEDGE_INTEGRATED_MODE_ENABLED=true -# Emergency deployment-wide freeze only. Per-Workspace cutover uses durable KFS activation rows. -KNOWLEDGE_LEGACY_ACL_READ_ONLY=false -# P9-only final profile. It must remain false during P8 and the legacy-route zero-call window. -KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED=false -KNOWLEDGE_DIRECT_UPLOAD_ENABLED=off -KNOWLEDGE_DIRECT_UPLOAD_ALLOWED_ORIGINS=https:// -KNOWLEDGE_DIRECT_STREAM_ENABLED=off -KNOWLEDGE_DIRECT_STREAM_ALLOWED_ORIGINS=https:// - -# Dify API: feature availability remains separate from the per-Workspace cutover ledger. -KNOWLEDGE_FS_ENABLED=false -KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false -KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false -KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false -KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false -``` - -The Dify product route and product Capability brokers also require the tenant's atomic cutover row -to have `product_routes_enabled`, `capability_v2_enabled`, `integrated_mode_enabled`, and -`legacy_acl_read_only` set together. Dify first sends the signed -`freezeDifyWorkspaceIntegration` command and persists its exact, replay-safe KFS ACK before it -applies the final delta. Immediately before the local cutover CAS, Dify sends the signed -`activateDifyWorkspaceIntegration` command and requires an exact KFS ACK for the activation id, -revision, and final-delta digest. KFS then admits Capability-v2 product traffic only for activated -Workspaces and rejects legacy API-key traffic for those Workspaces; inactive Workspaces retain the -legacy path. Activation is durable and one-way, so a lost ACK is replay-safe and rollback stops -Dify product traffic without re-enabling stale KFS ACL. Global environment flags mean only that -code is deployed; `KNOWLEDGE_LEGACY_ACL_READ_ONLY=true` is an emergency deployment-wide freeze, -never the rollout allowlist. Lifecycle internal-worker capabilities remain independent so deletion, -revoke, and reconciliation still converge while product traffic is stopped. - -Only the reviewed P9 removal deployment may set -`KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED=true`. The API then requires Capability v2, durable freeze -and activation repositories, integrated mode, and read-only legacy mutations at startup; it does -not register legacy member/access-policy/API-key routes and rejects every `kfs_` token. Treat this -as a one-way deployment after the global zero-call window, verified backup/restore drill, and -cleanup authorization—not as a per-Workspace rollout switch. - -Direct upload has two independent CORS boundaries. KFS upload-session control routes admit only -exact origins from `KNOWLEDGE_DIRECT_UPLOAD_ALLOWED_ORIGINS`, `POST`, `Authorization`, and -`Content-Type`; wildcard origins and credential cookies are rejected. The S3-compatible bucket must -separately allow those exact origins to send `PUT` with `Content-Type` and -`x-amz-checksum-sha256`, expose `ETag`, and disable credential-cookie support. Apply that bucket CORS -policy before enabling direct upload, then verify both a KFS control-route preflight and a -presigned-object preflight from the deployed Dify origin. KFS readiness cannot prove browser CORS -at the independent bucket endpoint. - -The browser computes the declared full-object SHA-256 incrementally before session creation. Each -multipart part also carries its own checksum, but an S3 multipart checksum is composite and must -not be compared with that full-object digest. After multipart completion KFS therefore streams the -stored object once through a bounded SHA-256 verifier before publishing the document; it never -buffers the complete object in the Node process. Capacity planning must include this verification -read and its object-store egress/latency. - -### Dify Compose and Kubernetes P0 baseline - -From the repository `docker/` directory, the optional `knowledge-fs` profile adds only an internal -API service. It shares Dify's default Compose network and existing `plugin_daemon`, exposes no host -port or nginx route, and leaves Dify's `KNOWLEDGE_FS_ENABLED` product flag at `false`. - -```bash -cp envs/core-services/knowledge-fs.env.example envs/core-services/knowledge-fs.env -docker compose --profile knowledge-fs config -docker compose --profile knowledge-fs build knowledge_fs -``` - -The Kubernetes reference is `infra/kubernetes/dify-integration-baseline.yaml`. It commits the -Deployment at zero replicas, uses `/health` for startup/liveness and `/ready` for readiness, and -defines only a ClusterIP Service plus same-namespace ingress policy. It creates no Ingress, -LoadBalancer, NodePort, migration Job, or product route. See `infra/kubernetes/README.md` before -adapting it to a downstream chart. - -Both shapes require a dedicated KnowledgeFS database and object-storage bucket. Never reuse the -Dify application database or Dataset/Document data, and run KnowledgeFS migrations only as a -separate controlled operation. - -### 3. Apply database migrations - -Check migration drift in CI and before deployment: - -```bash -pnpm db:migrations:check -``` - -Apply every pending PostgreSQL artifact from `packages/database/migrations` in migration-id order -through the migration runner or the controlled deployment system. Do not apply only `0001` to an -existing environment. Re-run health and smoke checks after migrations complete. - -### 4. Start services - -For an environment using the repository Compose file: - -```bash -docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps up -d -``` - -The API service depends on PostgreSQL health, Unstructured startup, and successful MinIO bucket bootstrap. If the bootstrap service fails, do not start the API against a missing object bucket. - -### 5. Standalone smoke checks - -```bash -curl -fsS http://localhost:8787/health -curl -sS -o /dev/null -w '%{http_code}\n' http://localhost:8787/ready -curl -fsS http://localhost:8787/openapi.json -``` - -`/ready` must print `200`. Do not proceed to tenant-scoped smoke checks or traffic enablement on -`503`; reconcile the Capability v2 public JWKS profile and every reported durable dependency first. - -Run the same tenant-scoped upload and query smoke used for SaaS. Also verify MinIO bucket contents and PostgreSQL row counts through operational tooling, not through ad hoc application-side scans. +`infra/kubernetes/dify-integration-baseline.yaml` is an inert reference with zero replicas, an +internal `ClusterIP`, fail-closed probes, and no public ingress. A downstream Dify deployment may +adopt it only while preserving the same ownership boundaries. ## Rollback -Rollback order should preserve data integrity: +Rollback the KnowledgeFS image or disable the affected Workspace cutover. Preserve KnowledgeFS +database rows and Dify-owned objects unless a reviewed recovery procedure says otherwise. Do not +rotate the Dify inner key, delete existing knowledge-base data, or redirect KnowledgeFS to a direct +storage/plugin endpoint as a rollback shortcut. -1. Stop traffic to Admin and API. -2. Roll back API first if the issue is ingestion, retrieval, auth, or persistence. -3. Roll back Admin first only for UI-only regressions. -4. Do not roll back database migrations without an explicit down-migration and data impact review. -5. Keep object storage data; do not bulk-delete uploaded objects during rollback. - -## Operational Guardrails - -- Keep API and Admin deploys separately observable, even when they are released together. -- Keep object reads, cache entries, query context, and streaming responses bounded. -- Use tenant id, subject id, permission snapshot, model version, and index version in cache keys where relevant. -- Never expose JWTs, raw file bytes, document text, or provider prompts in traces or logs. -- Treat missing database indexes and N+1 query paths as release blockers. -- Run retrieval regression gates before production promotion. - -## Current Gaps - -The current repository has production-ready contracts and local/CI gates, but these items still need dedicated implementation slices before fully automated production deploys: - -- Commit Workers entrypoint and `wrangler.toml`. -- Add Cloudflare KV and TiDB runtime driver wiring. -- Add production Admin Dockerfile or Pages adapter configuration. -- Automate public JWKS distribution and current/previous signing-key rotation evidence for each environment. -- Add secret-management automation for each environment. -- Add deployment pipeline jobs beyond validation gates. +After rollback, rerun Dify-connected health and tenant smoke, confirm the existing knowledge-base +feature is unaffected, and record the release and rollback evidence. diff --git a/knowledge-fs/docs/project-overview.md b/knowledge-fs/docs/project-overview.md index 69d0dd9894c..41926458fce 100644 --- a/knowledge-fs/docs/project-overview.md +++ b/knowledge-fs/docs/project-overview.md @@ -1,11 +1,12 @@ # Summary -KnowledgeFS is a TypeScript knowledge platform for retrieval-augmented systems. It provides a Hono-based Knowledge API, a Next.js Admin Console, portable infrastructure adapters, virtual KnowledgeFS command surfaces, MCP tools, retrieval pipelines, parser routing, background jobs, traces, and bounded in-process compute primitives. +KnowledgeFS is Dify's TypeScript knowledge backend. It provides a Hono-based Knowledge API, +virtual KnowledgeFS command surfaces, MCP tools, retrieval pipelines, parser routing, background +jobs, traces, and bounded in-process compute primitives. -The project has two intended deployment shapes: - -- SaaS: Cloudflare Workers, Cloudflare Pages, R2, KV, TiDB Cloud, and external parser services. -- Standalone/private deployment: Node.js API, Next.js Admin, PostgreSQL with pgvector, MinIO or S3-compatible object storage, Redis or bounded cache, and self-hosted parser services. +KnowledgeFS has one deployment shape: an internal service within Dify. Dify owns physical object +storage, model/plugin credentials, and datasource credentials/invocation; KnowledgeFS consumes +those capabilities through authenticated inner APIs. The system should be understood as a knowledge control plane and API server, not as a traditional L7 API gateway. The Admin Console remains a thin human-facing surface, while the Hono API owns auth, ingestion, retrieval, KnowledgeFS, MCP, jobs, provider orchestration, persistence, and observability. @@ -23,13 +24,15 @@ The core intent is to turn documents and external sources into structured, searc - Expose evidence through API, Admin UI, MCP tools, and KnowledgeFS-like commands. - Record traces, answer evidence, job history, and evaluation metrics for operations and debugging. -KnowledgeFS also aims to keep platform choices portable. Database, object storage, cache, job queue, parser, embedding, reranking, and generation providers should sit behind adapters so a deployment can choose SaaS infrastructure or a private enterprise stack without rewriting product logic. +Infrastructure remains behind adapters so domain logic does not depend on Dify's selected storage +provider or a specific database/parser implementation. Adapter portability is an implementation +boundary, not permission to deploy KnowledgeFS without Dify. # Goals - Provide a tenant-scoped knowledge platform for document ingestion, retrieval, answer generation, evaluation, and agent access. - Keep the Hono API as the main business boundary and keep the Admin Console thin. -- Support both SaaS and private/standalone deployment modes. +- Run as a Dify-owned internal backend with fail-closed inner-API dependencies. - Expose virtual filesystem-style knowledge inspection through bounded commands such as `ls`, `tree`, `cat`, `stat`, `grep`, `find`, `diff`, and `open_node`. - Make KnowledgeFS virtual and storage-agnostic: the backing store may be object storage, PostgreSQL, TiDB, index projections, or other repositories. - Keep chunking, token counting, RRF fusion, evidence packing, and text diff in the shared bounded TypeScript compute package. @@ -49,7 +52,7 @@ KnowledgeFS also aims to keep platform choices portable. Database, object storag - The Hono Knowledge API is not meant to be a generic reverse proxy or traditional API gateway. - Compute helpers must remain pure and must not host IO, network calls, database calls, or workflow orchestration. - Unstructured should not be treated as a mandatory hard dependency for every parser path. -- Inline in-memory adapters are not production infrastructure; they are development and test fallbacks. +- Inline in-memory adapters are test utilities, not deployment infrastructure. - PostgreSQL-backed queues are not assumed to be the right answer for every scale profile. - Workflow history, traces, partial results, parser artifacts, and projections should not be retained forever. - Database migration between PostgreSQL and TiDB is not assumed to be a trivial table copy. @@ -75,18 +78,14 @@ The Admin Console owns human workflows and diagnostics only. It should call the ## Deployment Design -SaaS deployment targets Cloudflare Workers/Pages, R2, KV, TiDB Cloud, and hosted parser services. +Dify Compose or a downstream Dify Kubernetes deployment starts the Node.js Hono API as an +internal service. KnowledgeFS has its own relational state and may use an out-of-process parser, +but it does not own a storage bucket, model credentials, datasource credentials, or a direct +Plugin Daemon connection. -Private deployment targets Docker Compose, Kubernetes, or equivalent orchestration with: - -- Node.js Hono API service. -- Next.js Admin service. -- PostgreSQL with pgvector. -- MinIO or enterprise S3-compatible object storage. -- Redis or another bounded cache. -- Self-hosted parser service. - -The same API contract should be preserved across both deployment shapes. Runtime-specific differences belong in adapters and deployment wiring. +Every model, rerank, LLM, datasource, and object-storage call crosses the authenticated Dify inner +API. `KNOWLEDGE_INTEGRATED_MODE_ENABLED` is limited to Workspace rollout/provisioning and must not +select a different runtime implementation. ## KnowledgeFS Design @@ -162,16 +161,14 @@ Database-specific details should remain behind repositories and migration artifa # Open Questions - Should the external name "Knowledge Gateway" be changed to "Knowledge API" or "Knowledge Control Plane" to avoid confusion with traditional API gateways? -- Which private deployment target should be treated as the first production reference: Docker Compose, Kubernetes, or an enterprise PaaS profile? - What is the first-class enterprise auth target after JWT secret auth: OIDC/JWKS, SAML, LDAP bridge, or a customer-specific identity proxy? -- Which parser provider should be the default for complex PDFs and Office files in private deployments? +- Which parser provider should be the default for complex PDFs and Office files in Dify deployments? - What timeout, memory, and isolation policy should parser workers use for OCR-heavy or malformed documents? - Which ResourceMount providers should be supported first beyond upload and object storage? - Should write-capable KnowledgeFS mounts be delayed until read-only inspection is fully stable? - What retention defaults should be used for traces, partial results, job history, parser artifacts, raw documents, and inactive projections? -- At what queue volume should a deployment move from PostgreSQL-backed queueing to Redis, Cloudflare Queues, or Temporal? +- At what queue volume should a deployment move from PostgreSQL-backed queueing to another Dify-operated queue or Temporal? - Should graph index traversal remain in the primary relational database, or should high-scale graph workloads move to a specialized graph/search backend later? -- What is the official migration playbook between PostgreSQL and TiDB deployments? - How should legacy tools that require real filesystem paths be supported: optional FUSE, sidecar projection, temporary workspace materialization, or API-only access? # Decisions @@ -190,4 +187,5 @@ Database-specific details should remain behind repositories and migration artifa - Retention and cleanup are required product behavior for production readiness. - Graph, vector, FTS, semantic, and summary indexes are rebuildable projections. - Cross-database migration should be handled through controlled migration plus reindex/rebuild flows. -- Private deployment should be supported without Cloudflare dependencies through Node.js, PostgreSQL, MinIO/S3, Redis/cache, and self-hosted parser services. +- KnowledgeFS is deployed only as part of Dify; provider credentials and physical object storage + remain exclusively owned by Dify. diff --git a/knowledge-fs/infra/aws_terraform/README.md b/knowledge-fs/infra/aws_terraform/README.md deleted file mode 100644 index 07daad03640..00000000000 --- a/knowledge-fs/infra/aws_terraform/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# knowledge-fs on AWS — Terraform - -Infrastructure-as-code for running the **Standalone** deployment target of `knowledge-fs` -on AWS. This directory currently holds **only the target architecture diagram**. -The Terraform code and deployment runbook are intentionally **not written yet** — see -[Status](#status). - -## Target architecture - -``` - ┌──────────────────────────── VPC ────────────────────────────┐ - │ │ - client ──TLS──▶│ ┌──────────────── EC2 ───────────────┐ ┌──────────────┐ │ - (HTTP / MCP) │ │ api (:8787, compiled JS, non-root)│ │ Aurora │ │ - │ │ ├─ retrieval / ingestion │TCP │ Serverless │ │ - │ │ ├─ pg-boss jobs ──────────────────┼───▶│ v2 │ │ - │ │ └─ in-process cache (single node) │5432│ PostgreSQL │ │ - │ │ unstructured (:8000, doc parsing) │ │ + pgvector │ │ - │ └──────────────────┬──────────────────┘ └──────────────┘ │ - │ │ HTTPS (S3 API) │ - │ │ creds via EC2 IAM instance role │ - └─────────────────────┼─────────────────────────────────────────┘ - ▼ - ┌───────────┐ - │ AWS S3 │ bucket: knowledge-fs - └───────────┘ -``` - -## Component mapping - -| knowledge-fs component | AWS service | Notes | -|-------------------------------|--------------------------------------|-------| -| API gateway (`apps/api`) | EC2 (Docker container) | `compose.middleware.yaml` minus DB/MinIO; runs compiled JS as non-root | -| Document parsing (unstructured)| EC2 (same instance, Docker) | ML image — size the box for it; can split to its own instance later | -| PostgreSQL + pgvector | **Aurora Serverless v2 (PostgreSQL)**| Standard TCP endpoint (not the Data API); `CREATE EXTENSION vector` | -| Job queue (pg-boss) | Same Aurora cluster | Postgres-backed; no extra service | -| Object storage | **AWS S3** | `MINIO_*` env points at S3; credentials via **IAM instance role** | -| Cache | In-process (single EC2) | No ElastiCache today — see [Open decisions](#open-decisions) | -| Admin console (`apps/admin`) | _not deployed in this topology_ | Optional Next.js UI; add as a second container if a web console is needed | - -## Environment wiring - -```bash -# Aurora Serverless v2 — standard cluster endpoint, TCP -DATABASE_URL=postgresql://:@:5432/knowledge_fs - -# S3 via the S3-compatible object-storage path. NO MINIO_ACCESS_KEY / MINIO_SECRET_KEY: -# the EC2 IAM instance role supplies credentials through the AWS SDK default chain. -MINIO_ENDPOINT=https://s3..amazonaws.com -MINIO_REGION= -MINIO_BUCKET=knowledge-fs - -# Unstructured runs locally on the same EC2 host -UNSTRUCTURED_API_URL=http://127.0.0.1:8000 -``` - -## EC2 metadata requirement for S3 IAM role - -The API container relies on the AWS SDK default credential chain to read the EC2 -instance role. Because the API runs inside Docker, the EC2 instance metadata -service must allow the IMDSv2 token response to cross the container network hop. -The Terraform EC2 resource/runbook must set: - -```hcl -metadata_options { - http_endpoint = "enabled" - http_tokens = "required" - http_put_response_hop_limit = 2 -} -``` - -Without the hop limit of `2`, a containerized API can have a valid instance role -but still fail to obtain S3 credentials from IMDSv2. - -## Open decisions - -These are deliberately deferred until the Terraform code is written: - -1. **Single EC2 = no HA / single point of failure.** Fine to start. Scaling the API to - multiple instances later requires a **shared cache (ElastiCache)** because the cache is - in-process today, and cache keys carry tenant + permission scope. -2. **Aurora SSL.** If the cluster parameter group sets `rds.force_ssl`, either use - `?sslmode=no-verify` or front the cluster with **RDS Proxy** (which also smooths - Serverless v2 connection scaling). Full CA verification needs a small adapter change. -3. **Admin console placement** — skip, co-locate on the EC2 host, or run separately. -4. **Compute choice** — plain EC2 + Docker for now; ECS Fargate / App Runner are - alternatives if container orchestration becomes preferable. - -## Status - -- [x] Target architecture diagram (this document) -- [ ] Terraform modules (VPC, EC2, Aurora Serverless v2, S3, IAM instance role, security groups) -- [ ] Deployment runbook diff --git a/knowledge-fs/infra/kubernetes/README.md b/knowledge-fs/infra/kubernetes/README.md index 1dcf78e6c25..9c270153c83 100644 --- a/knowledge-fs/infra/kubernetes/README.md +++ b/knowledge-fs/infra/kubernetes/README.md @@ -7,8 +7,8 @@ only defines the KnowledgeFS boundary that a downstream chart must preserve: - one `knowledge-fs-api` Deployment and internal `ClusterIP` Service; - `/health` for startup/liveness and `/ready` for traffic readiness; - same-namespace ingress only from pods labelled `app.kubernetes.io/part-of=dify`; -- Dify model and datasource runtime discovery at `http://api:5001` plus a Secret-provided - `DIFY_INNER_API_KEY` matching Dify's `INNER_API_KEY_FOR_PLUGIN`; +- Dify model, datasource, and unified object-storage access at `http://api:5001` plus a + Secret-provided `DIFY_INNER_API_KEY` matching Dify's `INNER_API_KEY_FOR_PLUGIN`; - no Ingress, LoadBalancer, NodePort, data migration job, or product route. The Deployment is committed with `replicas: 0`. Keep it there until Capability v2 supplies a @@ -17,9 +17,9 @@ streaming explicitly disabled, so any unchanged pod reports `503` from `/ready` is `200`. Before a later controlled scale-up, replace the image with an immutable digest and create the -`knowledge-fs-runtime` Secret with dedicated `DATABASE_URL`, `MINIO_ENDPOINT`, `MINIO_BUCKET`, -`MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`, and `DIFY_INNER_API_KEY`. Model and datasource credentials -stay in Dify/plugin-daemon and are never copied into KnowledgeFS. +`knowledge-fs-runtime` Secret with dedicated `DATABASE_URL` and `DIFY_INNER_API_KEY`. Model, +datasource, and object-storage credentials stay in Dify and are never copied into KnowledgeFS. +The configured Dify storage backend must support recursive `scan`. Add only the public Capability-v2 JWKS to the KFS Secret; its private signing key belongs exclusively to Dify. Set the ConfigMap capability flags only after migrations and readiness checks pass, and keep Dify's product/lifecycle flags disabled until the per-Workspace cutover gate @@ -27,5 +27,6 @@ is ready. Apply KnowledgeFS migrations separately through its migration runner; creates, reuses, or migrates Dify Dataset/Document data. The NetworkPolicy intentionally limits ingress only. Egress remains cluster/platform-owned because -database, object storage, parser, and Dify inner-runtime destinations differ between -deployments. A downstream default-deny policy must explicitly allow those dependencies. +database, parser, and Dify inner-runtime destinations differ between deployments. In integrated +mode KnowledgeFS does not connect directly to the object store. A downstream default-deny policy +must explicitly allow those dependencies. diff --git a/knowledge-fs/infra/local/.env.example b/knowledge-fs/infra/local/.env.example index b72b7446c51..492dc9700f4 100644 --- a/knowledge-fs/infra/local/.env.example +++ b/knowledge-fs/infra/local/.env.example @@ -9,22 +9,6 @@ DURABLE_DELETION_WRITER_FENCE_VERSION= # Keep this stable for the lifetime of deletion idempotency ledgers; do not rotate silently. DURABLE_DELETION_HMAC_KEY_BASE64= -MINIO_ACCESS_KEY=knowledge -MINIO_API_PORT=9000 -MINIO_BUCKET=knowledge-fs -MINIO_CONSOLE_PORT=9001 -MINIO_ENDPOINT=http://127.0.0.1:9000 -MINIO_REGION=us-east-1 -MINIO_ROOT_PASSWORD=knowledge-secret -MINIO_ROOT_USER=knowledge -MINIO_SECRET_KEY=knowledge-secret - -R2_ACCESS_KEY_ID= -R2_ACCOUNT_ID= -R2_BUCKET= -R2_REGION=auto -R2_SECRET_ACCESS_KEY= - UNSTRUCTURED_PORT=8000 UNSTRUCTURED_API_URL=http://127.0.0.1:8000 UNSTRUCTURED_API_KEY= @@ -32,47 +16,15 @@ UNSTRUCTURED_MAX_RESPONSE_BYTES= UNSTRUCTURED_MAX_RETRIES= UNSTRUCTURED_RETRY_DELAY_MS= -# Dify-managed model runtime. Required for embedding, rerank, LLM, and model preflight calls. +# Dify is required. It owns object storage, model/plugin credentials, and datasource invocation. DIFY_INNER_API_URL=http://host.docker.internal:5001 DIFY_INNER_API_KEY= DIFY_DATASOURCE_RUNTIME_MAX_RESPONSE_BYTES=8388608 DIFY_DATASOURCE_RUNTIME_REQUEST_TIMEOUT_MS=60000 DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES=8388608 DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS=60000 - -# Optional provider keys. OpenAI/Anthropic power LLM extraction; OpenAI/Cohere/Voyage/Gemini power embeddings; Cohere/Voyage power reranking. -OPENAI_API_KEY= -OPENAI_BASE_URL= -OPENAI_EMBEDDING_BASE_URL= -OPENAI_MODEL= -ANTHROPIC_API_KEY= -ANTHROPIC_BASE_URL= -COHERE_API_KEY= -COHERE_BASE_URL= -GEMINI_API_KEY= -GEMINI_BASE_URL= -VOYAGE_API_KEY= -VOYAGE_BASE_URL= -KNOWLEDGE_EMBEDDING_PROVIDER= -# Deployment default for newly-created/legacy knowledge spaces. Each space can persist an -# independent plugin/provider/model selection through its embedding-profile API. -KNOWLEDGE_EMBEDDING_MODEL= -KNOWLEDGE_EMBEDDING_PLUGIN_ID= -KNOWLEDGE_EMBEDDING_PLUGIN_PROVIDER= -# Required only for the test-only static provider; Dify model dimensions are inferred. -KNOWLEDGE_EMBEDDING_DIMENSION= -KNOWLEDGE_RERANK_PROVIDER= -KNOWLEDGE_RERANK_MODEL= -KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER= -KNOWLEDGE_ENTITY_EXTRACTION_MODEL= -KNOWLEDGE_ENTITY_EXTRACTION_MAX_ENTITIES_PER_NODE= -KNOWLEDGE_ENTITY_EXTRACTION_MAX_NODES_PER_RUN= -KNOWLEDGE_ENTITY_EXTRACTION_MAX_OUTPUT_TOKENS= -KNOWLEDGE_RELATION_EXTRACTION_MODEL= -KNOWLEDGE_RELATION_EXTRACTION_MAX_RELATIONS_PER_NODE= -KNOWLEDGE_RELATION_EXTRACTION_MAX_OUTPUT_TOKENS= -KNOWLEDGE_COMMUNITY_SUMMARY_MODEL= -KNOWLEDGE_COMMUNITY_SUMMARY_MAX_OUTPUT_TOKENS= +# Rollout/cutover gate only; it never enables a standalone runtime. +KNOWLEDGE_INTEGRATED_MODE_ENABLED=false API_PORT=8788 ADMIN_PORT=3000 diff --git a/knowledge-fs/infra/local/README.md b/knowledge-fs/infra/local/README.md index 85c101d30da..ea9b8d74323 100644 --- a/knowledge-fs/infra/local/README.md +++ b/knowledge-fs/infra/local/README.md @@ -1,91 +1,49 @@ # Local Development Environment -This compose stack is the Sprint 1 local scaffold for the standalone target. +KnowledgeFS has no standalone deployment mode. This directory is a developer harness for running +the KnowledgeFS backend against an existing Dify API. Dify remains the owner of object storage, +model/plugin credentials, and datasource invocation. -## Services +## Prerequisites -- PostgreSQL with pgvector on port `5432`. The `vector` extension is enabled automatically on a fresh data volume by `infra/local/postgres-init/01-enable-pgvector.sql` (mounted into `/docker-entrypoint-initdb.d`), so `pnpm local:db:migrate` succeeds without a manual step. -- MinIO S3-compatible object storage on ports `9000` and `9001`. -- A one-shot MinIO bootstrap container that creates `${MINIO_BUCKET:-knowledge-fs}`. -- Self-hosted Unstructured API on port `8000`. -- Optional API and Admin app containers behind the `apps` profile. +- A running Dify API reachable through `DIFY_INNER_API_URL`. +- The matching Dify inner API key in `DIFY_INNER_API_KEY`. +- Docker for the local PostgreSQL and Unstructured dependencies. -## Commands +Copy `infra/local/.env.example` to the ignored `infra/local/.env` and set the Dify URL and key. +Do not add MinIO, cloud-storage credentials, model-provider keys, or datasource credentials to the +KnowledgeFS environment. + +## Local dependencies ```bash pnpm dev:infra ``` -Starts PostgreSQL, MinIO, the MinIO bucket bootstrap, and Unstructured from `infra/local/compose.middleware.yaml`. -This is the preferred local mode when the API and Admin Console should run from the checked-out source tree on the host. +This starts: + +- PostgreSQL with pgvector on port `5432`. +- Unstructured API on port `8000`. + +The `vector` extension is enabled automatically on a fresh PostgreSQL volume through +`infra/local/postgres-init/01-enable-pgvector.sql`. + +Apply migrations explicitly after PostgreSQL starts: ```bash pnpm local:db:migrate ``` -Applies checked-in PostgreSQL migrations to the configured `DATABASE_URL` before running the source API against database-backed repositories. Migrations are **not** run by the API container, the Docker entrypoint, or on server startup — run this explicitly after the database is up. The initial migration relies on the `vector` extension; on a fresh volume the init script above provides it. If your Postgres volume predates that script, enable it once with: +If the PostgreSQL volume predates the init script, enable the extension once: ```bash docker compose --env-file infra/local/.env -f infra/local/compose.yaml exec postgres \ psql -U "${POSTGRES_USER:-knowledge_fs}" -d "${POSTGRES_DB:-knowledge_fs}" -c 'CREATE EXTENSION IF NOT EXISTS vector;' ``` -### Full Docker Compose startup +## Source-run workflow -```bash -docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml --profile apps config -docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps up -d --build -``` - -`pnpm dev:stack` runs the same full stack in the foreground when you want attached Compose logs. - -Starts PostgreSQL, MinIO, the MinIO bucket bootstrap, Unstructured, the production API container, and the production Admin Console container. - -Default local endpoints: - -- Admin Console / control panel: `http://localhost:3000` -- API health: `http://localhost:8788/health` -- API readiness: `http://localhost:8788/ready` -- Admin BFF health: `http://localhost:3000/api/bff/health` -- MinIO console: `http://localhost:9001` -- Unstructured API: `http://localhost:8000` - -If a default host port is already in use, override only the conflicting port for the startup command: - -```bash -API_PORT=8787 ADMIN_PORT=3003 UNSTRUCTURED_PORT=8002 docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps up -d --build -``` - -The API service is built from `apps/api/Dockerfile` as `knowledge-fs-api:local` and runs the standalone Hono server. The Admin service is built from `apps/admin/Dockerfile` as `knowledge-fs-admin:local` and runs the Next.js standalone server. - -If Docker Hub auth or rate limiting blocks another build and the app images already exist locally, start from cached local images without rebuilding: - -```bash -docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps up -d --no-build -``` - -Use the same port overrides with `--no-build` when needed: - -```bash -API_PORT=8787 ADMIN_PORT=3003 UNSTRUCTURED_PORT=8002 docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps up -d --no-build -``` - -Verify the full stack after startup: - -```bash -docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps ps -curl http://localhost:${API_PORT:-8788}/health -curl --fail http://localhost:${API_PORT:-8788}/ready -curl http://localhost:${ADMIN_PORT:-3000}/api/bff/health -``` - -The API container uses `/ready` for its Compose health check, and Admin waits for that check to -pass. In this development profile, the explicit local verifier is installed and the Dify/plugin-daemon -transports are optional until their corresponding model or datasource features are exercised. `/health` -remains a liveness endpoint and always uses HTTP 200 to expose component diagnostics. - -For the source-run local happy path, keep `pnpm dev:infra` running and start these in separate -terminals: +Keep `pnpm dev:infra` and the Dify API running, then start these commands in separate terminals: ```bash pnpm local:db:migrate @@ -94,64 +52,53 @@ pnpm --filter @knowledge/admin dev pnpm local:happy-path ``` -`pnpm dev:api` loads `infra/local/.env` automatically, so it sees the same database, object storage, and local auth settings used by `pnpm local:db:migrate`. +`pnpm dev:api` automatically loads `infra/local/.env`. `pnpm local:happy-path` checks health, +workspace bootstrap, Markdown upload, parse-artifact reads, and bounded query evidence. Configure +`LOCAL_SMOKE_ADMIN_BASE` if the local Admin development server is not on +`http://127.0.0.1:3000`. -The smoke command validates Compose config, Admin build, API health, Admin BFF health, workspace bootstrap, -single Markdown upload through the Admin BFF proxy, document status read, parse artifact read, and a bounded query evidence -check without manual database edits. Set `LOCAL_SMOKE_ADMIN_BASE` when the Admin dev server is not running at `http://127.0.0.1:3000`. -Set `LOCAL_SMOKE_SKIP_ADMIN_BUILD=1` to skip the Admin build when you only want to recheck the live API/upload/artifact path. -Set `LOCAL_SMOKE_RUN_MIGRATIONS=1 pnpm local:happy-path` when you want the smoke run to apply checked-in PostgreSQL migrations before API health, upload, artifact, and query checks. -Run `pnpm local:happy-path:durable` when you want the smoke to require `DATABASE_URL`, MinIO env, healthy database, and healthy object storage instead of silently accepting memory fallback. -Run `pnpm local:happy-path:api` when you want the same bounded API upload, artifact, and query evidence checks without requiring the Admin dev server or Admin BFF. +Useful variants: + +```bash +LOCAL_SMOKE_RUN_MIGRATIONS=1 pnpm local:happy-path +pnpm local:happy-path:durable +pnpm local:happy-path:api +``` + +The durable variant requires `DATABASE_URL`, `DIFY_INNER_API_URL`, and `DIFY_INNER_API_KEY`, then +requires both database and Dify-backed object-storage health to pass. The API-only variant skips +the local Admin BFF. + +## Containerized developer harness + +The `apps` profile can build the backend and optional local Admin harness while still depending on +the external Dify API: + +```bash +docker compose --env-file infra/local/.env.example -f infra/local/compose.yaml --profile apps config +docker compose --env-file infra/local/.env -f infra/local/compose.yaml --profile apps up -d --build +``` + +Default local endpoints: + +- Admin developer harness: `http://localhost:3000` +- KnowledgeFS API health: `http://localhost:8788/health` +- KnowledgeFS API readiness: `http://localhost:8788/ready` +- Unstructured API: `http://localhost:8000` + +This Compose file is not a supported KnowledgeFS deployment topology. Production KnowledgeFS must +be started by Dify's Compose or Kubernetes deployment so the internal URL, authentication, storage, +models, datasources, and lifecycle are wired together. + +## Validation ```bash pnpm compose:config -``` - -Validates the resolved compose configuration without starting containers. - -```bash pnpm compose:middleware:config pnpm compose:middleware:test -``` - -Validates the middleware-only Compose file and asserts that it does not include `api` or `admin`. - -```bash pnpm compose:apps:test ``` -Validates the full app profile contract for API image build wiring, middleware readiness dependencies, and the Admin source-run BFF base URL without starting containers. - -```bash -docker compose --env-file infra/local/.env -f infra/local/compose.yaml up -d minio minio-bootstrap -pnpm test:minio -``` - -Runs the live MinIO object-storage smoke test against the bootstrapped local bucket. - -## Notes - -- Runtime secrets are intentionally local defaults only. Put overrides in `infra/local/.env`, which is ignored by git. -- Copy `infra/local/.env.example` to `infra/local/.env` for a fresh local setup. The tracked example leaves provider API keys blank. -- With the default `infra/local/.env.example` values, source-run Node can use PostgreSQL through `DATABASE_URL`, database-backed core repositories unless `KNOWLEDGE_DATABASE_REPOSITORIES=off`, MinIO through `MINIO_ENDPOINT`, `MINIO_BUCKET`, `MINIO_ACCESS_KEY`, and `MINIO_SECRET_KEY`, and local Admin-to-API auth through `KNOWLEDGE_DEV_AUTH_TOKEN`. -- In the full Docker stack the Admin container also receives `KNOWLEDGE_DEV_AUTH_TOKEN` so its server-side BFF can authenticate to the API. Because the Admin image runs with `NODE_ENV=production`, without this token `getAdminServerToken()` returns `null` and the token-gated panels (Control plane, Operations diagnostics, Publish readiness) render as "Unavailable". The default matches the API service so both resolve to `dev-token` unless overridden in `infra/local/.env`. -- First run against an empty database: after `pnpm local:db:migrate`, the Admin Console still has no knowledge space, so per-space panels show "Unavailable". Create one (or upload a document, which bootstraps a space) to populate them. The default active workspace is the space whose slug is `workspace`: - - ```bash - curl -X POST -H "Authorization: Bearer ${KNOWLEDGE_DEV_AUTH_TOKEN:-dev-token}" -H 'Content-Type: application/json' \ - -d '{"name":"workspace","slug":"workspace"}' "http://localhost:${API_PORT:-8788}/knowledge-spaces" - ``` - - Publish readiness stays empty ("No document selected") until a document is uploaded through the Upload intake panel. -- Dense-vector indexing/search is disabled until `KNOWLEDGE_EMBEDDING_PROVIDER` is set to `openai`, `cohere`, `voyage`, or `static`. Set `KNOWLEDGE_EMBEDDING_MODEL` when you need a non-default model. -- For OpenAI-compatible embeddings, prefer `OPENAI_EMBEDDING_BASE_URL`; `OPENAI_BASE_URL` is accepted as a fallback and a trailing `/v1` is normalized before the embedding client appends `/v1/embeddings`. -- The API production image bundles the TypeScript compute package, so containerized ingestion can create KnowledgeNodes without an external runtime artifact. -- R2-compatible runtime wiring is available through `R2_ACCOUNT_ID`, `R2_BUCKET`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, and optional `R2_REGION`. -- The API and Admin containers run from standalone local images. Use `infra/local/compose.middleware.yaml` when you want middleware containers only and API/Admin from the host source tree. -- Build the API image directly with `pnpm docker:api:build` and the Admin image with `pnpm docker:admin:build` when you want to validate Dockerfiles outside Compose. -- The Unstructured API image follows the upstream self-hosted API image path. -- The live MinIO smoke test is intentionally separate from `pnpm check` so normal CI does not require long-running local containers. -- Run `pnpm docker:api:bundle-smoke` to start the built API image under `NODE_ENV=test` and verify `/health` reports `components.compute === true`. This isolated bundle check does not validate production fail-closed configuration or durable dependencies. `pnpm docker:api:http-smoke` is retained only as a compatibility alias. -- Run `pnpm docker:admin:http-smoke` after `pnpm docker:admin:build` to start the production Admin image and verify the Next.js standalone homepage renders. -- Run `pnpm docker:apps:smoke` to build both app images, run the isolated API bundle check, and run the Admin image homepage check. Validate production API wiring through the Compose-backed durable happy path and tenant-scoped upload/query checks. +The middleware Compose contains only PostgreSQL and Unstructured. The app-profile checks also +assert that no MinIO, cloud-storage credential, provider credential, or direct Plugin Daemon +configuration reaches the KnowledgeFS API. diff --git a/knowledge-fs/infra/local/compose.middleware.yaml b/knowledge-fs/infra/local/compose.middleware.yaml index 4ea80e89ea1..cbe7b14a345 100644 --- a/knowledge-fs/infra/local/compose.middleware.yaml +++ b/knowledge-fs/infra/local/compose.middleware.yaml @@ -18,41 +18,10 @@ services: - postgres-data:/var/lib/postgresql/data - ./postgres-init:/docker-entrypoint-initdb.d:ro - minio: - image: minio/minio:latest - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-knowledge-secret} - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-knowledge} - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 10 - ports: - - "${MINIO_API_PORT:-9000}:9000" - - "${MINIO_CONSOLE_PORT:-9001}:9001" - volumes: - - minio-data:/data - - minio-bootstrap: - image: minio/mc:latest - depends_on: - minio: - condition: service_healthy - entrypoint: ["/bin/sh", "-lc"] - command: - - mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}" && mc mb --ignore-existing "local/$${MINIO_BUCKET}" - environment: - MINIO_BUCKET: ${MINIO_BUCKET:-knowledge-fs} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-knowledge-secret} - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-knowledge} - unstructured: image: downloads.unstructured.io/unstructured-io/unstructured-api:latest ports: - "${UNSTRUCTURED_PORT:-8000}:8000" volumes: - minio-data: postgres-data: diff --git a/knowledge-fs/infra/local/compose.yaml b/knowledge-fs/infra/local/compose.yaml index bebb9bad407..a39cc4eea48 100644 --- a/knowledge-fs/infra/local/compose.yaml +++ b/knowledge-fs/infra/local/compose.yaml @@ -18,36 +18,6 @@ services: - postgres-data:/var/lib/postgresql/data - ./postgres-init:/docker-entrypoint-initdb.d:ro - minio: - image: minio/minio:latest - command: server /data --console-address ":9001" - environment: - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-knowledge-secret} - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-knowledge} - healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 5s - timeout: 5s - retries: 10 - ports: - - "${MINIO_API_PORT:-9000}:9000" - - "${MINIO_CONSOLE_PORT:-9001}:9001" - volumes: - - minio-data:/data - - minio-bootstrap: - image: minio/mc:latest - depends_on: - minio: - condition: service_healthy - entrypoint: ["/bin/sh", "-lc"] - command: - - mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}" && mc mb --ignore-existing "local/$${MINIO_BUCKET}" - environment: - MINIO_BUCKET: ${MINIO_BUCKET:-knowledge-fs} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-knowledge-secret} - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-knowledge} - unstructured: image: downloads.unstructured.io/unstructured-io/unstructured-api:latest ports: @@ -62,8 +32,6 @@ services: depends_on: postgres: condition: service_healthy - minio-bootstrap: - condition: service_completed_successfully unstructured: condition: service_started environment: @@ -75,46 +43,11 @@ services: DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES: ${DIFY_MODEL_RUNTIME_MAX_RESPONSE_BYTES:-8388608} DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS: ${DIFY_MODEL_RUNTIME_REQUEST_TIMEOUT_MS:-60000} KNOWLEDGE_INTEGRATED_MODE_ENABLED: ${KNOWLEDGE_INTEGRATED_MODE_ENABLED:-false} - MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-knowledge} - MINIO_BUCKET: ${MINIO_BUCKET:-knowledge-fs} - MINIO_ENDPOINT: http://minio:9000 - MINIO_REGION: ${MINIO_REGION:-us-east-1} - MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-knowledge-secret} NODE_ENV: development PORT: 8787 KNOWLEDGE_DEV_AUTH_TOKEN: ${KNOWLEDGE_DEV_AUTH_TOKEN:-dev-token} KNOWLEDGE_DEV_SUBJECT_ID: ${KNOWLEDGE_DEV_SUBJECT_ID:-dev-user} KNOWLEDGE_DEV_TENANT_ID: ${KNOWLEDGE_DEV_TENANT_ID:-tenant-dev} - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} - ANTHROPIC_BASE_URL: ${ANTHROPIC_BASE_URL:-} - COHERE_API_KEY: ${COHERE_API_KEY:-} - COHERE_BASE_URL: ${COHERE_BASE_URL:-} - GEMINI_API_KEY: ${GEMINI_API_KEY:-} - GEMINI_BASE_URL: ${GEMINI_BASE_URL:-} - KNOWLEDGE_ANSWER_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_ANSWER_MAX_OUTPUT_TOKENS:-} - KNOWLEDGE_ANSWER_MODEL: ${KNOWLEDGE_ANSWER_MODEL:-} - KNOWLEDGE_ANSWER_PROVIDER: ${KNOWLEDGE_ANSWER_PROVIDER:-} - KNOWLEDGE_COMMUNITY_SUMMARY_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_COMMUNITY_SUMMARY_MAX_OUTPUT_TOKENS:-} - KNOWLEDGE_COMMUNITY_SUMMARY_MODEL: ${KNOWLEDGE_COMMUNITY_SUMMARY_MODEL:-} - KNOWLEDGE_ENTITY_EXTRACTION_MAX_ENTITIES_PER_NODE: ${KNOWLEDGE_ENTITY_EXTRACTION_MAX_ENTITIES_PER_NODE:-} - KNOWLEDGE_ENTITY_EXTRACTION_MAX_NODES_PER_RUN: ${KNOWLEDGE_ENTITY_EXTRACTION_MAX_NODES_PER_RUN:-} - KNOWLEDGE_EMBEDDING_DIMENSION: ${KNOWLEDGE_EMBEDDING_DIMENSION:-} - KNOWLEDGE_EMBEDDING_MODEL: ${KNOWLEDGE_EMBEDDING_MODEL:-} - KNOWLEDGE_EMBEDDING_PROVIDER: ${KNOWLEDGE_EMBEDDING_PROVIDER:-} - KNOWLEDGE_ENTITY_EXTRACTION_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_ENTITY_EXTRACTION_MAX_OUTPUT_TOKENS:-} - KNOWLEDGE_ENTITY_EXTRACTION_MODEL: ${KNOWLEDGE_ENTITY_EXTRACTION_MODEL:-} - KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER: ${KNOWLEDGE_ENTITY_EXTRACTION_PROVIDER:-} - KNOWLEDGE_RERANK_MODEL: ${KNOWLEDGE_RERANK_MODEL:-} - KNOWLEDGE_RERANK_PROVIDER: ${KNOWLEDGE_RERANK_PROVIDER:-} - KNOWLEDGE_RELATION_EXTRACTION_MAX_OUTPUT_TOKENS: ${KNOWLEDGE_RELATION_EXTRACTION_MAX_OUTPUT_TOKENS:-} - KNOWLEDGE_RELATION_EXTRACTION_MAX_RELATIONS_PER_NODE: ${KNOWLEDGE_RELATION_EXTRACTION_MAX_RELATIONS_PER_NODE:-} - KNOWLEDGE_RELATION_EXTRACTION_MODEL: ${KNOWLEDGE_RELATION_EXTRACTION_MODEL:-} - OPENAI_API_KEY: ${OPENAI_API_KEY:-} - OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} - OPENAI_EMBEDDING_BASE_URL: ${OPENAI_EMBEDDING_BASE_URL:-} - OPENAI_MODEL: ${OPENAI_MODEL:-} - VOYAGE_API_KEY: ${VOYAGE_API_KEY:-} - VOYAGE_BASE_URL: ${VOYAGE_BASE_URL:-} UNSTRUCTURED_API_URL: http://unstructured:8000 ports: - "${API_PORT:-8788}:8787" @@ -151,5 +84,4 @@ services: - "${ADMIN_PORT:-3000}:3000" volumes: - minio-data: postgres-data: diff --git a/knowledge-fs/package.json b/knowledge-fs/package.json index 5fd8178a228..3970f0d72a0 100644 --- a/knowledge-fs/package.json +++ b/knowledge-fs/package.json @@ -46,13 +46,12 @@ "openapi:export:test": "node --test scripts/export-openapi.test.mjs scripts/export-capability-v2-operations.test.mjs", "p9:bundle": "node scripts/build-p9-removal-bundle.mjs", "p9:bundle:test": "node --test scripts/build-p9-removal-bundle.test.mjs", - "security:dependencies": "pnpm audit --prod --audit-level high", + "security:dependencies": "node scripts/audit-backend-dependencies.mjs", "security:secrets": "node scripts/secret-scan.mjs", "swagger": "node tools/swagger/server.mjs", "swagger:test": "node --test tools/swagger/server.test.mjs", "test": "turbo run test", "test:coverage": "turbo run test:coverage", - "test:minio": "MINIO_ENDPOINT=${MINIO_ENDPOINT:-http://127.0.0.1:9000} MINIO_BUCKET=${MINIO_BUCKET:-knowledge-fs} MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-knowledge} MINIO_SECRET_KEY=${MINIO_SECRET_KEY:-knowledge-secret} MINIO_REGION=${MINIO_REGION:-us-east-1} RUN_MINIO_INTEGRATION=1 pnpm --filter @knowledge/adapters test:minio", "typecheck": "turbo run typecheck" }, "devDependencies": { diff --git a/knowledge-fs/packages/adapters/package.json b/knowledge-fs/packages/adapters/package.json index 0ce9f1c4b77..9dee6e2e841 100644 --- a/knowledge-fs/packages/adapters/package.json +++ b/knowledge-fs/packages/adapters/package.json @@ -4,19 +4,15 @@ "type": "module", "exports": { ".": "./src/index.ts", - "./cloudflare": "./src/cloudflare.ts", "./node": "./src/node.ts" }, "scripts": { "build": "tsc --noEmit", "test": "vitest run", "test:coverage": "vitest run --coverage", - "test:minio": "vitest run src/object-storage.integration.test.ts", "typecheck": "tsc --noEmit" }, "dependencies": { - "@aws-sdk/client-s3": "^3.1045.0", - "@aws-sdk/s3-request-presigner": "3.1045.0", "@knowledge/core": "workspace:*", "@knowledge/database": "workspace:*", "pg": "^8.21.0" diff --git a/knowledge-fs/packages/adapters/src/adapters.test.ts b/knowledge-fs/packages/adapters/src/adapters.test.ts index 29f9a95a63b..6381ff96438 100644 --- a/knowledge-fs/packages/adapters/src/adapters.test.ts +++ b/knowledge-fs/packages/adapters/src/adapters.test.ts @@ -1,8 +1,7 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { createCloudflarePlatformAdapter } from "./cloudflare"; import { createCloudflareJobQueueAdapter } from "./cloudflare-job-queue"; -import { buildNodeS3ClientConfig, createNodePlatformAdapter } from "./node"; +import { createNodePlatformAdapter } from "./node"; import { createPgBossJobQueueAdapter } from "./pg-boss-job-queue"; import { type PostgresPoolLike, @@ -11,202 +10,46 @@ import { } from "./postgres"; describe("platform adapter skeletons", () => { - it("creates a Cloudflare adapter with the SaaS runtime target", async () => { - const adapter = createCloudflarePlatformAdapter({ env: {} }); - - await expect(adapter.health()).resolves.toMatchObject({ - ok: true, - runtime: "cloudflare-workers", - }); - }); - - it("creates a Node adapter with the standalone runtime target", async () => { - const adapter = createNodePlatformAdapter({ env: {} }); + it("creates a Dify-dependent Node adapter", async () => { + const difyStorageFetch = vi + .fn() + .mockResolvedValue(Response.json({ ok: true })); + const adapter = createNodePlatformAdapter({ difyStorageFetch, env: {} }); await expect(adapter.health()).resolves.toMatchObject({ ok: true, runtime: "node-docker", }); + expect(adapter.objectStorage.kind).toBe("dify"); + expect(difyStorageFetch.mock.calls[0]?.[0].toString()).toBe( + "http://localhost:5001/inner/api/knowledge-fs/storage/health", + ); }); - it("uses S3-compatible object storage for Node when complete MinIO env is provided", async () => { - const client = new FakeS3Client(); + it("uses Dify unified storage regardless of rollout mode and ignores MinIO env", async () => { + const difyStorageFetch = vi + .fn() + .mockResolvedValue(Response.json({ ok: true })); const adapter = createNodePlatformAdapter({ + difyStorageFetch, env: { - MINIO_ACCESS_KEY: "knowledge", - MINIO_BUCKET: "knowledge-fs", + DIFY_INNER_API_KEY: "inner-key", + DIFY_INNER_API_URL: "http://api:5001", + KNOWLEDGE_INTEGRATED_MODE_ENABLED: "false", + MINIO_BUCKET: "must-not-be-used", MINIO_ENDPOINT: "http://minio:9000", - MINIO_REGION: "us-east-1", - MINIO_SECRET_KEY: "knowledge-secret", }, - objectStorageClient: client, - }); - const body = new Uint8Array([1, 2, 3]); - - await expect( - adapter.objectStorage.putObject({ - body, - contentType: "application/octet-stream", - key: "tenant-1/object.bin", - }), - ).resolves.toMatchObject({ - key: "tenant-1/object.bin", - sizeBytes: 3, }); - expect(adapter.objectStorage.kind).toBe("s3-compatible"); - expect(client.commands).toEqual([ - { - input: { - Body: body, - Bucket: "knowledge-fs", - ContentType: "application/octet-stream", - Key: "tenant-1/object.bin", - Metadata: {}, - }, - name: "PutObjectCommand", - }, - ]); - }); - - it("builds a Node S3 client config with static credentials when access key and secret are set", () => { - const config = buildNodeS3ClientConfig( - { - MINIO_ACCESS_KEY: "knowledge", - MINIO_REGION: "us-east-1", - MINIO_SECRET_KEY: "knowledge-secret", - }, - "http://minio:9000", - ); - - expect(config).toMatchObject({ - endpoint: "http://minio:9000", - forcePathStyle: true, - region: "us-east-1", - }); - expect(config.credentials).toEqual({ - accessKeyId: "knowledge", - secretAccessKey: "knowledge-secret", - }); - }); - - it("omits static credentials from the Node S3 client config so AWS resolves the IAM instance role", () => { - const config = buildNodeS3ClientConfig( - { - MINIO_REGION: "us-east-1", - }, - "https://s3.us-east-1.amazonaws.com", - ); - - expect(config).toMatchObject({ - endpoint: "https://s3.us-east-1.amazonaws.com", - forcePathStyle: true, - region: "us-east-1", - }); - expect("credentials" in config).toBe(false); - }); - - it("defaults the Node S3 region to us-east-1 when MINIO_REGION is unset", () => { - const config = buildNodeS3ClientConfig({}, "https://s3.us-east-1.amazonaws.com"); - - expect(config.region).toBe("us-east-1"); - }); - - it("uses S3-compatible object storage for Node when credentials are absent (IAM instance role)", async () => { - const client = new FakeS3Client(); - const adapter = createNodePlatformAdapter({ - env: { - MINIO_BUCKET: "knowledge-fs", - MINIO_ENDPOINT: "https://s3.us-east-1.amazonaws.com", - MINIO_REGION: "us-east-1", - }, - objectStorageClient: client, - }); - - await expect( - adapter.objectStorage.putObject({ - body: new Uint8Array([1, 2, 3]), - key: "tenant-1/role.bin", - }), - ).resolves.toMatchObject({ key: "tenant-1/role.bin", sizeBytes: 3 }); - - expect(adapter.objectStorage.kind).toBe("s3-compatible"); - expect(client.commands).toHaveLength(1); - }); - - it("keeps Node object storage in memory when the MinIO endpoint is missing", async () => { - const client = new FakeS3Client(); - const adapter = createNodePlatformAdapter({ - env: { - MINIO_ACCESS_KEY: "knowledge", - MINIO_BUCKET: "knowledge-fs", - MINIO_SECRET_KEY: "knowledge-secret", - }, - objectStorageClient: client, - }); - - await expect( - adapter.objectStorage.putObject({ - body: new Uint8Array([4, 5, 6]), - key: "tenant-1/fallback.bin", - }), - ).resolves.toMatchObject({ - key: "tenant-1/fallback.bin", - sizeBytes: 3, - }); + expect(adapter.objectStorage.kind).toBe("dify"); await expect(adapter.objectStorage.health()).resolves.toBe(true); - expect(adapter.objectStorage.kind).toBe("memory"); - expect(client.commands).toEqual([]); - }); - - it("keeps Cloudflare cache and incomplete R2 fallback honest as memory adapters", async () => { - const adapter = createCloudflarePlatformAdapter({ - env: { - R2_BUCKET: "knowledge-r2", - }, - objectStorageClient: new FakeS3Client(), - }); - - expect(adapter.cache.kind).toBe("memory"); - expect(adapter.objectStorage.kind).toBe("memory"); - await expect(adapter.health()).resolves.toMatchObject({ - ok: true, - components: { - cache: true, - objectStorage: true, - }, - }); - }); - - it("uses R2 object storage for Cloudflare when complete R2 env is provided", async () => { - const client = new FakeS3Client(); - const adapter = createCloudflarePlatformAdapter({ - env: { - R2_ACCESS_KEY_ID: "r2-access-key", - R2_ACCOUNT_ID: "account-id", - R2_BUCKET: "knowledge-r2", - R2_SECRET_ACCESS_KEY: "r2-secret-key", - }, - objectStorageClient: client, - }); - - await adapter.objectStorage.putObject({ - body: new Uint8Array([7, 8, 9]), - key: "tenant-1/r2-object.bin", - }); - - expect(adapter.objectStorage.kind).toBe("r2"); - expect(client.commands).toEqual([ - { - input: { - Body: new Uint8Array([7, 8, 9]), - Bucket: "knowledge-r2", - Key: "tenant-1/r2-object.bin", - Metadata: {}, - }, - name: "PutObjectCommand", - }, - ]); + expect(difyStorageFetch).toHaveBeenCalledOnce(); + expect(difyStorageFetch.mock.calls[0]?.[0].toString()).toBe( + "http://api:5001/inner/api/knowledge-fs/storage/health", + ); + expect(new Headers(difyStorageFetch.mock.calls[0]?.[1]?.headers).get("X-Inner-Api-Key")).toBe( + "inner-key", + ); }); it("sends Cloudflare queue messages and stores durable job state", async () => { @@ -414,29 +257,7 @@ describe("platform adapter skeletons", () => { await expect(queue.status(job.id)).resolves.toMatchObject({ status: "queued" }); }); - it("wires injected Cloudflare queue bindings through the platform factory", async () => { - const queueBinding = new FakeCloudflareQueueBinding(); - const stateStore = new FakeCloudflareJobStateStore(); - const adapter = createCloudflarePlatformAdapter({ - env: {}, - jobQueue: queueBinding, - jobStateStore: stateStore, - }); - - const job = await adapter.jobs.enqueue({ - payload: { documentId: "doc-1" }, - type: "compile.document", - }); - - expect(adapter.jobs.kind).toBe("cloudflare-queues"); - expect(queueBinding.messages).toHaveLength(1); - expect(stateStore.records.get(job.id)).toMatchObject({ - id: job.id, - status: "queued", - }); - }); - - it("sends pg-boss jobs and stores standalone job status", async () => { + it("sends pg-boss jobs and stores durable job status", async () => { const boss = new FakePgBossClient(); const queue = createPgBossJobQueueAdapter({ boss, @@ -686,14 +507,14 @@ describe("platform adapter skeletons", () => { await expect(configuredAdapter.database.close?.()).resolves.toBeUndefined(); - const s3Adapter = createNodePlatformAdapter({ + const difyAdapter = createNodePlatformAdapter({ env: { MINIO_BUCKET: "knowledge-fs", MINIO_ENDPOINT: "http://minio:9000", }, }); - expect(s3Adapter.objectStorage.kind).toBe("s3-compatible"); + expect(difyAdapter.objectStorage.kind).toBe("dify"); }); it("covers PostgreSQL result, rollback, release, and health fallbacks", async () => { @@ -797,22 +618,6 @@ describe("platform adapter skeletons", () => { }); }); -class FakeS3Client { - readonly commands: { input: unknown; name: string }[] = []; - - async send(command: { - readonly input: unknown; - readonly constructor: { readonly name: string }; - }) { - this.commands.push({ - input: command.input, - name: command.constructor.name, - }); - - return {}; - } -} - class FakeCloudflareQueueBinding { readonly messages: { body: unknown; options: unknown }[] = []; diff --git a/knowledge-fs/packages/adapters/src/cloudflare.ts b/knowledge-fs/packages/adapters/src/cloudflare.ts deleted file mode 100644 index 7a16b1121fd..00000000000 --- a/knowledge-fs/packages/adapters/src/cloudflare.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { S3Client } from "@aws-sdk/client-s3"; -import { type PlatformAdapter, collectPlatformHealth } from "@knowledge/core"; - -import { createMemoryCacheAdapter } from "./cache"; -import { - type CloudflareJobStateStore, - type CloudflareQueueBinding, - createCloudflareJobQueueAdapter, -} from "./cloudflare-job-queue"; -import { createSchemaDatabaseAdapter } from "./database"; -import { - type S3ObjectStorageClient, - createMemoryObjectStorageAdapter, - createS3ObjectStorageAdapter, -} from "./object-storage"; - -type RuntimeEnv = Readonly>; - -export interface CloudflarePlatformAdapterOptions { - readonly env?: RuntimeEnv; - readonly jobQueue?: CloudflareQueueBinding; - readonly jobStateStore?: CloudflareJobStateStore; - readonly objectStorageClient?: S3ObjectStorageClient; -} - -const maxObjectBytes = 64 * 1024 * 1024; -const maxMemoryObjects = 10_000; -const maxMemoryObjectBytes = maxObjectBytes * maxMemoryObjects; - -export function createCloudflarePlatformAdapter( - options: CloudflarePlatformAdapterOptions = {}, -): PlatformAdapter { - const env = options.env ?? {}; - const adapter: PlatformAdapter = { - runtime: "cloudflare-workers", - database: createSchemaDatabaseAdapter({ kind: "tidb" }), - objectStorage: createCloudflareObjectStorageAdapter(env, options.objectStorageClient), - cache: createMemoryCacheAdapter({ maxEntries: 10_000 }), - jobs: createCloudflareJobQueueAdapter({ - ...(options.jobQueue ? { queue: options.jobQueue } : {}), - ...(options.jobStateStore ? { state: options.jobStateStore } : {}), - maxBatchSize: 100, - maxQueuedJobs: 10_000, - }), - health: async () => collectPlatformHealth(adapter), - }; - - return adapter; -} - -function createCloudflareObjectStorageAdapter( - env: RuntimeEnv, - objectStorageClient?: S3ObjectStorageClient, -) { - const accessKeyId = env.R2_ACCESS_KEY_ID?.trim(); - const accountId = env.R2_ACCOUNT_ID?.trim(); - const bucket = env.R2_BUCKET?.trim(); - const secretAccessKey = env.R2_SECRET_ACCESS_KEY?.trim(); - - if (accessKeyId && accountId && bucket && secretAccessKey) { - const client = - objectStorageClient ?? - new S3Client({ - credentials: { - accessKeyId, - secretAccessKey, - }, - endpoint: env.R2_ENDPOINT?.trim() || `https://${accountId}.r2.cloudflarestorage.com`, - region: env.R2_REGION?.trim() || "auto", - }); - - return createS3ObjectStorageAdapter({ - bucket, - client, - kind: "r2", - maxObjectBytes, - }); - } - - return createMemoryObjectStorageAdapter({ - kind: "memory", - maxObjectBytes, - maxObjects: maxMemoryObjects, - maxTotalBytes: maxMemoryObjectBytes, - }); -} diff --git a/knowledge-fs/packages/adapters/src/database.test.ts b/knowledge-fs/packages/adapters/src/database.test.ts index ffde280f620..12a8d1aee82 100644 --- a/knowledge-fs/packages/adapters/src/database.test.ts +++ b/knowledge-fs/packages/adapters/src/database.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "vitest"; -import { createCloudflarePlatformAdapter } from "./cloudflare"; import { createSchemaDatabaseAdapter } from "./database"; import { createNodePlatformAdapter } from "./node"; import { type PostgresPoolLike, createPostgresDatabaseExecutor } from "./postgres"; @@ -605,17 +604,6 @@ describe("platform database skeletons", () => { type: "postgres", }); }); - - it("wires the Cloudflare adapter to the TiDB schema database contract", async () => { - const adapter = createCloudflarePlatformAdapter(); - - expect(adapter.database.kind).toBe("tidb"); - await expect(adapter.database.getSchemaSummary()).resolves.toMatchObject({ dialect: "tidb" }); - await expect(adapter.database.getCapabilities()).resolves.toMatchObject({ - fullTextCjkNative: true, - type: "tidb", - }); - }); }); describe("PostgreSQL database executor", () => { diff --git a/knowledge-fs/packages/adapters/src/dify-object-storage.test.ts b/knowledge-fs/packages/adapters/src/dify-object-storage.test.ts new file mode 100644 index 00000000000..db5e980f3cc --- /dev/null +++ b/knowledge-fs/packages/adapters/src/dify-object-storage.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createDifyObjectStorageAdapter } from "./dify-object-storage"; + +const metadata = { + checksumSha256Base64: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + contentType: "text/plain", + key: "tenant-1/spaces/space-1/file.txt", + metadata: { tenantId: "tenant-1" }, + sizeBytes: 4, +}; + +describe("Dify object storage adapter", () => { + it("uses the authenticated Dify inner API for object operations", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json(metadata)) + .mockResolvedValueOnce(Response.json(metadata)) + .mockResolvedValueOnce(Response.json({ nextCursor: metadata.key, objects: [metadata] })) + .mockResolvedValueOnce(new Response(new Uint8Array([1, 2, 3, 4]))) + .mockResolvedValueOnce(new Response(null, { status: 204 })) + .mockResolvedValueOnce(Response.json({ ok: true })); + const adapter = createDifyObjectStorageAdapter({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch, + }); + + await expect( + adapter.putObject({ + body: new Uint8Array([1, 2, 3, 4]), + contentType: "text/plain", + key: metadata.key, + metadata: { tenantId: "tenant-1" }, + }), + ).resolves.toEqual(metadata); + await expect(adapter.headObject(metadata.key)).resolves.toEqual(metadata); + await expect( + adapter.listObjects({ limit: 1, prefix: "tenant-1/spaces/space-1/" }), + ).resolves.toEqual({ nextCursor: metadata.key, objects: [metadata] }); + await expect(adapter.getObject(metadata.key)).resolves.toEqual(new Uint8Array([1, 2, 3, 4])); + await expect(adapter.deleteObject(metadata.key)).resolves.toBeUndefined(); + await expect(adapter.health()).resolves.toBe(true); + + expect(adapter.kind).toBe("dify"); + expect(adapter.directUpload).toBeUndefined(); + for (const call of fetch.mock.calls) { + expect(new Headers(call[1]?.headers).get("X-Inner-Api-Key")).toBe("inner-key"); + } + expect(fetch.mock.calls[0]?.[0].toString()).toContain( + "/inner/api/knowledge-fs/storage/object?key=tenant-1%2Fspaces%2Fspace-1%2Ffile.txt", + ); + }); + + it("maps missing objects to null and rejects oversized response bodies", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 404 })) + .mockResolvedValueOnce( + new Response(new Uint8Array([1, 2, 3]), { + headers: { "Content-Length": "3" }, + }), + ); + const adapter = createDifyObjectStorageAdapter({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch, + maxObjectBytes: 2, + }); + + await expect(adapter.getObject("tenant-1/missing")).resolves.toBeNull(); + await expect(adapter.getObject("tenant-1/large")).rejects.toThrow("exceeds maxObjectBytes=2"); + }); + + it("returns false when Dify storage health is unavailable", async () => { + const adapter = createDifyObjectStorageAdapter({ + apiKey: "inner-key", + baseUrl: "http://api:5001", + fetch: vi.fn().mockRejectedValue(new Error("offline")), + }); + + await expect(adapter.health()).resolves.toBe(false); + }); +}); diff --git a/knowledge-fs/packages/adapters/src/dify-object-storage.ts b/knowledge-fs/packages/adapters/src/dify-object-storage.ts new file mode 100644 index 00000000000..3663596a67a --- /dev/null +++ b/knowledge-fs/packages/adapters/src/dify-object-storage.ts @@ -0,0 +1,277 @@ +import type { + ListObjectsResult, + ObjectMetadata, + ObjectStorageAdapter, + PutObjectInput, +} from "@knowledge/core"; + +export interface DifyObjectStorageOptions { + readonly apiKey: string; + readonly baseUrl: string; + readonly fetch?: typeof globalThis.fetch; + readonly maxObjectBytes?: number; +} + +const defaultMaxObjectBytes = 64 * 1024 * 1024; +const metadataHeader = "X-Knowledge-FS-Metadata"; +const checksumHeader = "X-Knowledge-FS-Checksum-Sha256"; +const contentTypeHeader = "X-Knowledge-FS-Content-Type"; + +/** + * Uses Dify's authenticated inner API as the only physical object-storage owner. The adapter + * deliberately omits direct-upload capabilities because Dify's portable + * storage contract does not expose provider-specific multipart or presign primitives. + */ +export function createDifyObjectStorageAdapter({ + apiKey, + baseUrl, + fetch = globalThis.fetch, + maxObjectBytes = defaultMaxObjectBytes, +}: DifyObjectStorageOptions): ObjectStorageAdapter { + const normalizedBaseUrl = requiredBaseUrl(baseUrl); + const normalizedApiKey = requiredString(apiKey, "Dify inner API key"); + positiveSafeInteger(maxObjectBytes, "maxObjectBytes"); + + const request = (path: string, init: RequestInit = {}) => + fetch(new URL(path, normalizedBaseUrl), { + ...init, + headers: { + ...headersRecord(init.headers), + "X-Inner-Api-Key": normalizedApiKey, + }, + }); + + return { + kind: "dify", + deleteObject: async (key) => { + const response = await request( + objectPath("/inner/api/knowledge-fs/storage/object", { key }), + { + method: "DELETE", + }, + ); + assertStatus(response, [204]); + }, + getObject: async (key) => { + const response = await request(objectPath("/inner/api/knowledge-fs/storage/object", { key })); + if (response.status === 404) return null; + assertStatus(response, [200]); + return readBoundedBody(response, maxObjectBytes); + }, + getObjectStream: async (key) => { + const response = await request(objectPath("/inner/api/knowledge-fs/storage/object", { key })); + if (response.status === 404) return null; + assertStatus(response, [200]); + return boundedResponseStream(response, maxObjectBytes); + }, + health: async () => { + try { + const response = await request("/inner/api/knowledge-fs/storage/health"); + if (!response.ok) return false; + const payload = asRecord(await response.json()); + return payload?.ok === true; + } catch { + return false; + } + }, + headObject: async (key) => { + const response = await request( + objectPath("/inner/api/knowledge-fs/storage/object/metadata", { key }), + ); + if (response.status === 404) return null; + assertStatus(response, [200]); + return parseObjectMetadata(await response.json()); + }, + listObjects: async ({ cursor, limit, prefix }) => { + const response = await request( + objectPath("/inner/api/knowledge-fs/storage/objects", { + ...(cursor ? { cursor } : {}), + limit: String(limit), + prefix, + }), + ); + assertStatus(response, [200]); + return parseObjectList(await response.json()); + }, + putObject: async (input) => { + if (input.body.byteLength > maxObjectBytes) { + throw new Error(`Object ${input.key} exceeds maxObjectBytes=${maxObjectBytes}`); + } + const response = await request( + objectPath("/inner/api/knowledge-fs/storage/object", { key: input.key }), + { + body: requestBody(input.body), + headers: { + ...(input.checksumSha256Base64 ? { [checksumHeader]: input.checksumSha256Base64 } : {}), + ...(input.contentType ? { [contentTypeHeader]: input.contentType } : {}), + [metadataHeader]: Buffer.from(JSON.stringify(input.metadata ?? {})).toString( + "base64url", + ), + }, + method: "PUT", + }, + ); + assertStatus(response, [200]); + return parseObjectMetadata(await response.json()); + }, + }; +} + +function requestBody(body: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(body.byteLength); + copy.set(body); + return copy.buffer; +} + +function objectPath(path: string, query: Readonly>): string { + const search = new URLSearchParams(query); + return `${path}?${search.toString()}`; +} + +function requiredBaseUrl(value: string): string { + const normalized = requiredString(value, "Dify inner API URL"); + let parsed: URL; + try { + parsed = new URL(normalized); + } catch { + throw new Error("Dify inner API URL is invalid"); + } + if ( + (parsed.protocol !== "http:" && parsed.protocol !== "https:") || + parsed.username || + parsed.password + ) { + throw new Error("Dify inner API URL is invalid"); + } + return parsed.href.endsWith("/") ? parsed.href : `${parsed.href}/`; +} + +function requiredString(value: string, name: string): string { + const normalized = value.trim(); + if (!normalized) throw new Error(`${name} is required`); + return normalized; +} + +function positiveSafeInteger(value: number, name: string): void { + if (!Number.isSafeInteger(value) || value < 1) { + throw new Error(`${name} must be a positive safe integer`); + } +} + +function headersRecord(headers: HeadersInit | undefined): Record { + return Object.fromEntries(new Headers(headers).entries()); +} + +function assertStatus(response: Response, expected: readonly number[]): void { + if (!expected.includes(response.status)) { + throw new Error(`Dify object storage request failed with status ${response.status}`); + } +} + +function parseObjectList(value: unknown): ListObjectsResult { + const record = asRecord(value); + if (!record || !Array.isArray(record.objects)) { + throw new Error("Dify object storage list response is invalid"); + } + const objects = record.objects.map(parseObjectMetadata); + const nextCursor = optionalString(record.nextCursor); + return { + objects, + ...(nextCursor ? { nextCursor } : {}), + }; +} + +function parseObjectMetadata(value: unknown): ObjectMetadata { + const record = asRecord(value); + const metadata = asStringRecord(record?.metadata); + const key = optionalString(record?.key); + const sizeBytes = record?.sizeBytes; + if ( + !record || + !metadata || + !key || + !Number.isSafeInteger(sizeBytes) || + typeof sizeBytes !== "number" || + sizeBytes < 0 + ) { + throw new Error("Dify object storage metadata response is invalid"); + } + const checksumSha256Base64 = optionalString(record.checksumSha256Base64); + const contentType = optionalString(record.contentType); + return { + ...(checksumSha256Base64 ? { checksumSha256Base64 } : {}), + ...(contentType ? { contentType } : {}), + key, + metadata, + sizeBytes, + }; +} + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function asStringRecord(value: unknown): Record | undefined { + const record = asRecord(value); + if (!record || Object.values(record).some((item) => typeof item !== "string")) return undefined; + return Object.fromEntries(Object.entries(record).map(([key, item]) => [key, item as string])); +} + +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +async function readBoundedBody(response: Response, maxObjectBytes: number): Promise { + const stream = boundedResponseStream(response, maxObjectBytes); + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + chunks.push(chunk.value); + totalBytes += chunk.value.byteLength; + } + const body = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; +} + +function boundedResponseStream( + response: Response, + maxObjectBytes: number, +): ReadableStream { + const declaredLength = Number(response.headers.get("content-length")); + if (Number.isFinite(declaredLength) && declaredLength > maxObjectBytes) { + throw new Error(`Dify object storage response exceeds maxObjectBytes=${maxObjectBytes}`); + } + const source = response.body; + if (!source) return new ReadableStream({ start: (controller) => controller.close() }); + const reader = source.getReader(); + let totalBytes = 0; + return new ReadableStream({ + cancel: (reason) => reader.cancel(reason), + async pull(controller) { + const chunk = await reader.read(); + if (chunk.done) { + controller.close(); + return; + } + totalBytes += chunk.value.byteLength; + if (totalBytes > maxObjectBytes) { + await reader.cancel(); + controller.error( + new Error(`Dify object storage response exceeds maxObjectBytes=${maxObjectBytes}`), + ); + return; + } + controller.enqueue(chunk.value); + }, + }); +} diff --git a/knowledge-fs/packages/adapters/src/index.ts b/knowledge-fs/packages/adapters/src/index.ts index 43985fa5cf0..6ae1aca1943 100644 --- a/knowledge-fs/packages/adapters/src/index.ts +++ b/knowledge-fs/packages/adapters/src/index.ts @@ -1,10 +1,10 @@ export * from "./cache"; -export * from "./cloudflare"; export * from "./cloudflare-job-queue"; export * from "./database"; +export * from "./dify-object-storage"; export * from "./job-queue"; +export * from "./memory-object-storage"; export * from "./migration-runner"; export * from "./node"; -export * from "./object-storage"; export * from "./pg-boss-job-queue"; export * from "./postgres"; diff --git a/knowledge-fs/packages/adapters/src/memory-object-storage.test.ts b/knowledge-fs/packages/adapters/src/memory-object-storage.test.ts new file mode 100644 index 00000000000..48e2e4fae49 --- /dev/null +++ b/knowledge-fs/packages/adapters/src/memory-object-storage.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; + +import { createMemoryObjectStorageAdapter } from "./memory-object-storage"; + +describe("memory object storage adapter", () => { + it("advertises that test storage cannot issue direct-upload URLs", () => { + const storage = createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 64 }); + expect(storage.directUpload).toBeUndefined(); + }); + + it("stores, streams, heads, lists, and deletes copied object state", async () => { + const storage = createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 64 }); + const body = new TextEncoder().encode("knowledge object"); + const metadata = { sha256: "original" }; + + const stored = await storage.putObject({ + body, + checksumSha256Base64: "checksum", + contentType: "text/plain", + key: "tenant-1/b.txt", + metadata, + }); + metadata.sha256 = "mutated"; + (stored.metadata as Record).sha256 = "returned-mutation"; + await storage.putObject({ body: new Uint8Array([1]), key: "tenant-1/a.txt" }); + + await expect(storage.getObject("tenant-1/b.txt")).resolves.toEqual(body); + await expect(readStream(await storage.getObjectStream("tenant-1/b.txt"))).resolves.toEqual( + body, + ); + await expect(storage.getObjectStream("tenant-1/missing.txt")).resolves.toBeNull(); + await expect(storage.headObject("tenant-1/b.txt")).resolves.toEqual({ + checksumSha256Base64: "checksum", + contentType: "text/plain", + key: "tenant-1/b.txt", + metadata: { sha256: "original" }, + sizeBytes: body.byteLength, + }); + await expect(storage.listObjects({ limit: 1, prefix: "tenant-1/" })).resolves.toMatchObject({ + nextCursor: "tenant-1/a.txt", + objects: [{ key: "tenant-1/a.txt" }], + }); + + await storage.deleteObject("tenant-1/b.txt"); + await expect(storage.getObject("tenant-1/b.txt")).resolves.toBeNull(); + }); + + it("enforces object, count, total-byte, and list bounds", async () => { + const storage = createMemoryObjectStorageAdapter({ + kind: "local", + maxObjectBytes: 2, + maxObjects: 1, + maxTotalBytes: 2, + }); + + await expect( + storage.putObject({ body: new Uint8Array([1, 2, 3]), key: "too-large" }), + ).rejects.toThrow("Object too-large exceeds maxObjectBytes=2"); + await storage.putObject({ body: new Uint8Array([1, 2]), key: "first" }); + await expect(storage.putObject({ body: new Uint8Array([1]), key: "second" })).rejects.toThrow( + "Object storage maxObjects=1 exceeded", + ); + await expect(storage.listObjects({ limit: 0, prefix: "" })).rejects.toThrow( + "Object list limit must be at least 1", + ); + }); + + it("accounts for replacement bytes and preserves optional checksums", async () => { + const storage = createMemoryObjectStorageAdapter({ + kind: "memory", + maxObjectBytes: 4, + maxObjects: 1, + maxTotalBytes: 2, + }); + + await storage.putObject({ body: new Uint8Array([1, 2]), key: "object" }); + await expect( + storage.putObject({ + body: new Uint8Array([3]), + checksumSha256Base64: "checksum", + key: "object", + }), + ).resolves.toEqual({ + checksumSha256Base64: "checksum", + key: "object", + metadata: {}, + sizeBytes: 1, + }); + }); + + it("rejects invalid capacity bounds", () => { + expect(() => createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 0 })).toThrow( + "Object storage maxObjectBytes must be at least 1", + ); + expect(() => + createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 1, maxObjects: 0 }), + ).toThrow("Object storage maxObjects must be at least 1"); + expect(() => + createMemoryObjectStorageAdapter({ + kind: "memory", + maxObjectBytes: 1, + maxTotalBytes: 0, + }), + ).toThrow("Object storage maxTotalBytes must be at least 1"); + }); +}); + +async function readStream(stream: ReadableStream | null): Promise { + if (!stream) throw new Error("Expected stream"); + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + while (true) { + const result = await reader.read(); + if (result.done) break; + chunks.push(result.value); + totalBytes += result.value.byteLength; + } + + const bytes = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} diff --git a/knowledge-fs/packages/adapters/src/memory-object-storage.ts b/knowledge-fs/packages/adapters/src/memory-object-storage.ts new file mode 100644 index 00000000000..9864bb13a0d --- /dev/null +++ b/knowledge-fs/packages/adapters/src/memory-object-storage.ts @@ -0,0 +1,151 @@ +import type { + ListObjectsInput, + ListObjectsResult, + ObjectMetadata, + ObjectStorageAdapter, +} from "@knowledge/core"; + +export interface MemoryObjectStorageOptions { + readonly kind: Extract; + readonly maxObjectBytes: number; + readonly maxObjects?: number; + readonly maxTotalBytes?: number; +} + +interface StoredObject { + readonly body: Uint8Array; + readonly metadata: ObjectMetadata; +} + +/** In-process object storage for bounded unit tests; production Node assembly always uses Dify. */ +export function createMemoryObjectStorageAdapter({ + kind, + maxObjectBytes, + maxObjects = 1_000, + maxTotalBytes = maxObjectBytes * maxObjects, +}: MemoryObjectStorageOptions): ObjectStorageAdapter { + if (maxObjectBytes < 1) { + throw new Error("Object storage maxObjectBytes must be at least 1"); + } + if (maxObjects < 1) { + throw new Error("Object storage maxObjects must be at least 1"); + } + if (maxTotalBytes < 1) { + throw new Error("Object storage maxTotalBytes must be at least 1"); + } + + const objects = new Map(); + let totalBytes = 0; + + return { + kind, + deleteObject: async (key) => { + const stored = objects.get(key); + if (stored) { + totalBytes -= stored.body.byteLength; + objects.delete(key); + } + }, + getObject: async (key) => { + const stored = objects.get(key); + return stored ? copyBytes(stored.body) : null; + }, + getObjectStream: async (key) => { + const stored = objects.get(key); + return stored ? bytesToStream(copyBytes(stored.body)) : null; + }, + health: async () => true, + headObject: async (key) => { + const stored = objects.get(key); + return stored ? cloneObjectMetadata(stored.metadata) : null; + }, + listObjects: async (input) => listObjects(objects, input), + putObject: async (input) => { + if (input.body.byteLength > maxObjectBytes) { + throw new Error(`Object ${input.key} exceeds maxObjectBytes=${maxObjectBytes}`); + } + + const body = copyBytes(input.body); + const existing = objects.get(input.key); + const nextObjectCount = objects.size + (existing ? 0 : 1); + const nextTotalBytes = totalBytes - (existing?.body.byteLength ?? 0) + body.byteLength; + if (nextObjectCount > maxObjects) { + throw new Error(`Object storage maxObjects=${maxObjects} exceeded`); + } + if (nextTotalBytes > maxTotalBytes) { + throw new Error(`Object storage maxTotalBytes=${maxTotalBytes} exceeded`); + } + + const metadata: ObjectMetadata = { + ...(input.checksumSha256Base64 ? { checksumSha256Base64: input.checksumSha256Base64 } : {}), + ...(input.contentType ? { contentType: input.contentType } : {}), + key: input.key, + metadata: cloneMetadata(input.metadata), + sizeBytes: body.byteLength, + }; + objects.set(input.key, { body, metadata }); + totalBytes = nextTotalBytes; + return cloneObjectMetadata(metadata); + }, + }; +} + +function listObjects( + objects: ReadonlyMap, + { cursor, limit, prefix }: ListObjectsInput, +): ListObjectsResult { + if (limit < 1) { + throw new Error("Object list limit must be at least 1"); + } + + const keys = [...objects.keys()] + .filter((key) => key.startsWith(prefix) && (!cursor || key > cursor)) + .sort() + .slice(0, limit + 1); + const pageKeys = keys.slice(0, limit); + const pageObjects = pageKeys + .map((key) => objects.get(key)?.metadata) + .filter(isObjectMetadata) + .map(cloneObjectMetadata); + const nextCursor = keys.length > limit ? pageKeys.at(-1) : undefined; + + return { + objects: pageObjects, + ...(nextCursor ? { nextCursor } : {}), + }; +} + +function copyBytes(bytes: Uint8Array): Uint8Array { + return new Uint8Array(bytes); +} + +function bytesToStream(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); +} + +function cloneMetadata( + metadata: Readonly> | undefined, +): Readonly> { + return { ...(metadata ?? {}) }; +} + +function cloneObjectMetadata(metadata: ObjectMetadata): ObjectMetadata { + return { + ...(metadata.checksumSha256Base64 + ? { checksumSha256Base64: metadata.checksumSha256Base64 } + : {}), + ...(metadata.contentType ? { contentType: metadata.contentType } : {}), + key: metadata.key, + metadata: cloneMetadata(metadata.metadata), + sizeBytes: metadata.sizeBytes, + }; +} + +function isObjectMetadata(value: ObjectMetadata | undefined): value is ObjectMetadata { + return Boolean(value); +} diff --git a/knowledge-fs/packages/adapters/src/node.ts b/knowledge-fs/packages/adapters/src/node.ts index 833b116c64f..14b9320166b 100644 --- a/knowledge-fs/packages/adapters/src/node.ts +++ b/knowledge-fs/packages/adapters/src/node.ts @@ -1,14 +1,9 @@ -import { S3Client, type S3ClientConfig } from "@aws-sdk/client-s3"; import { type PlatformAdapter, collectPlatformHealth } from "@knowledge/core"; import { createMemoryCacheAdapter } from "./cache"; import { createSchemaDatabaseAdapter } from "./database"; +import { createDifyObjectStorageAdapter } from "./dify-object-storage"; import { createInlineJobQueueAdapter } from "./job-queue"; -import { - type S3ObjectStorageClient, - createMemoryObjectStorageAdapter, - createS3ObjectStorageAdapter, -} from "./object-storage"; import { type PgBossClient, createPgBossJobQueueAdapter } from "./pg-boss-job-queue"; import { type PostgresPoolLike, @@ -21,14 +16,14 @@ type RuntimeEnv = Readonly>; export interface NodePlatformAdapterOptions { readonly databasePool?: PostgresPoolLike; + readonly difyStorageFetch?: typeof globalThis.fetch; readonly env?: RuntimeEnv; readonly jobBoss?: PgBossClient; - readonly objectStorageClient?: S3ObjectStorageClient; } const maxObjectBytes = 64 * 1024 * 1024; -const maxMemoryObjects = 10_000; -const maxMemoryObjectBytes = maxObjectBytes * maxMemoryObjects; +const defaultDifyInnerApiUrl = "http://localhost:5001"; +const defaultDifyInnerApiKey = "QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1"; export function createNodePlatformAdapter( options: NodePlatformAdapterOptions = {}, @@ -38,7 +33,7 @@ export function createNodePlatformAdapter( const adapter: PlatformAdapter = { runtime: "node-docker", database, - objectStorage: createNodeObjectStorageAdapter(env, options.objectStorageClient), + objectStorage: createNodeObjectStorageAdapter(env, options.difyStorageFetch), cache: createMemoryCacheAdapter({ maxEntries: 10_000 }), jobs: options.jobBoss ? createPgBossJobQueueAdapter({ @@ -83,48 +78,15 @@ function createNodeDatabaseAdapter(env: RuntimeEnv, databasePool?: PostgresPoolL return createSchemaDatabaseAdapter({ kind: "postgres" }); } -/** - * Builds the S3 client config for the Node object-storage adapter. Static - * credentials are only included when both `MINIO_ACCESS_KEY` and - * `MINIO_SECRET_KEY` are present; otherwise they are omitted so the AWS SDK - * resolves credentials through its default provider chain (e.g. an EC2 IAM - * instance role or ECS task role). - */ -export function buildNodeS3ClientConfig(env: RuntimeEnv, endpoint: string): S3ClientConfig { - const accessKeyId = env.MINIO_ACCESS_KEY?.trim(); - const secretAccessKey = env.MINIO_SECRET_KEY?.trim(); - - return { - endpoint, - forcePathStyle: true, - region: env.MINIO_REGION?.trim() || "us-east-1", - ...(accessKeyId && secretAccessKey ? { credentials: { accessKeyId, secretAccessKey } } : {}), - }; -} - function createNodeObjectStorageAdapter( env: RuntimeEnv, - objectStorageClient?: S3ObjectStorageClient, + difyStorageFetch?: typeof globalThis.fetch, ) { - const bucket = env.MINIO_BUCKET?.trim(); - const endpoint = env.MINIO_ENDPOINT?.trim(); - - if (bucket && endpoint) { - const client = objectStorageClient ?? new S3Client(buildNodeS3ClientConfig(env, endpoint)); - - return createS3ObjectStorageAdapter({ - bucket, - client, - kind: "s3-compatible", - maxObjectBytes, - }); - } - - return createMemoryObjectStorageAdapter({ - kind: "memory", + return createDifyObjectStorageAdapter({ + apiKey: env.DIFY_INNER_API_KEY?.trim() || defaultDifyInnerApiKey, + baseUrl: env.DIFY_INNER_API_URL?.trim() || defaultDifyInnerApiUrl, + ...(difyStorageFetch ? { fetch: difyStorageFetch } : {}), maxObjectBytes, - maxObjects: maxMemoryObjects, - maxTotalBytes: maxMemoryObjectBytes, }); } diff --git a/knowledge-fs/packages/adapters/src/object-storage.integration.test.ts b/knowledge-fs/packages/adapters/src/object-storage.integration.test.ts deleted file mode 100644 index b364a203b93..00000000000 --- a/knowledge-fs/packages/adapters/src/object-storage.integration.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { randomUUID } from "node:crypto"; - -import { describe, expect, it } from "vitest"; - -import { createNodePlatformAdapter } from "./node"; - -const describeMinio = process.env.RUN_MINIO_INTEGRATION === "1" ? describe : describe.skip; - -describeMinio("MinIO object storage integration", () => { - it("round-trips objects through the Node platform S3-compatible adapter", async () => { - const adapter = createNodePlatformAdapter({ env: readMinioEnv() }); - const key = `integration-smoke/${Date.now()}-${randomUUID()}.txt`; - const body = new TextEncoder().encode("knowledge minio smoke"); - - expect(adapter.objectStorage.kind).toBe("s3-compatible"); - await expect(adapter.objectStorage.health()).resolves.toBe(true); - - try { - const putMetadata = await adapter.objectStorage.putObject({ - body, - contentType: "text/plain", - key, - metadata: { smoke: "minio" }, - }); - - expect(putMetadata).toEqual({ - contentType: "text/plain", - key, - metadata: { smoke: "minio" }, - sizeBytes: body.byteLength, - }); - - const firstRead = await adapter.objectStorage.getObject(key); - expect(firstRead).toEqual(body); - - if (!firstRead) { - throw new Error("Expected MinIO object body to be readable"); - } - - firstRead[0] = 0; - await expect(adapter.objectStorage.getObject(key)).resolves.toEqual(body); - await expect(adapter.objectStorage.headObject(key)).resolves.toEqual({ - contentType: "text/plain", - key, - metadata: { smoke: "minio" }, - sizeBytes: body.byteLength, - }); - - const listed = await adapter.objectStorage.listObjects({ - limit: 10, - prefix: "integration-smoke/", - }); - - expect(listed.objects.some((object) => object.key === key)).toBe(true); - - await adapter.objectStorage.deleteObject(key); - - await expect(adapter.objectStorage.getObject(key)).resolves.toBeNull(); - await expect(adapter.objectStorage.headObject(key)).resolves.toBeNull(); - } finally { - await adapter.objectStorage.deleteObject(key).catch(() => undefined); - } - }); -}); - -function readMinioEnv(): Readonly> { - return { - MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY ?? "knowledge", - MINIO_BUCKET: process.env.MINIO_BUCKET ?? "knowledge-fs", - MINIO_ENDPOINT: process.env.MINIO_ENDPOINT ?? "http://127.0.0.1:9000", - MINIO_REGION: process.env.MINIO_REGION ?? "us-east-1", - MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY ?? "knowledge-secret", - }; -} diff --git a/knowledge-fs/packages/adapters/src/object-storage.test.ts b/knowledge-fs/packages/adapters/src/object-storage.test.ts deleted file mode 100644 index 5d2f5dc90ff..00000000000 --- a/knowledge-fs/packages/adapters/src/object-storage.test.ts +++ /dev/null @@ -1,1424 +0,0 @@ -import { createHash } from "node:crypto"; - -import { describe, expect, it } from "vitest"; - -import { createMemoryObjectStorageAdapter, createS3ObjectStorageAdapter } from "./object-storage"; - -describe("memory object storage adapter", () => { - it("advertises that local memory storage cannot issue direct-upload URLs", () => { - const storage = createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 64 }); - - expect(storage.directUpload).toBeUndefined(); - }); - - it("stores, reads, heads, and deletes immutable object bytes", async () => { - const storage = createMemoryObjectStorageAdapter({ - kind: "local", - maxObjectBytes: 64, - }); - const body = new TextEncoder().encode("knowledge object"); - - await storage.putObject({ - body, - contentType: "text/plain", - key: "tenant-1/documents/object.txt", - metadata: { sha256: "a".repeat(64) }, - }); - - await expect(storage.headObject("tenant-1/documents/object.txt")).resolves.toMatchObject({ - contentType: "text/plain", - key: "tenant-1/documents/object.txt", - metadata: { sha256: "a".repeat(64) }, - sizeBytes: body.byteLength, - }); - await expect(storage.getObject("tenant-1/documents/object.txt")).resolves.toEqual(body); - - await storage.deleteObject("tenant-1/documents/object.txt"); - - await expect(storage.headObject("tenant-1/documents/object.txt")).resolves.toBeNull(); - await expect(storage.getObject("tenant-1/documents/object.txt")).resolves.toBeNull(); - }); - - it("lists objects with explicit limits and stable cursors", async () => { - const storage = createMemoryObjectStorageAdapter({ kind: "s3-compatible", maxObjectBytes: 64 }); - - await storage.putObject({ body: new Uint8Array([1]), key: "tenant-1/a.txt" }); - await storage.putObject({ body: new Uint8Array([2]), key: "tenant-1/b.txt" }); - await storage.putObject({ body: new Uint8Array([3]), key: "tenant-1/c.txt" }); - await storage.putObject({ body: new Uint8Array([4]), key: "tenant-2/a.txt" }); - - const firstPage = await storage.listObjects({ limit: 2, prefix: "tenant-1/" }); - const cursor = firstPage.nextCursor; - - if (!cursor) { - throw new Error("Expected first page to include a next cursor"); - } - - const secondPage = await storage.listObjects({ - cursor, - limit: 2, - prefix: "tenant-1/", - }); - - expect(firstPage.objects.map((object) => object.key)).toEqual([ - "tenant-1/a.txt", - "tenant-1/b.txt", - ]); - expect(firstPage.nextCursor).toBe("tenant-1/b.txt"); - expect(secondPage.objects.map((object) => object.key)).toEqual(["tenant-1/c.txt"]); - expect(secondPage.nextCursor).toBeUndefined(); - }); - - it("rejects objects larger than the configured memory cap", async () => { - const storage = createMemoryObjectStorageAdapter({ kind: "local", maxObjectBytes: 2 }); - - await expect( - storage.putObject({ body: new Uint8Array([1, 2, 3]), key: "tenant-1/too-large.bin" }), - ).rejects.toThrow("Object tenant-1/too-large.bin exceeds maxObjectBytes=2"); - }); - - it("can stream memory objects without forcing callers through getObject", async () => { - const storage = createMemoryObjectStorageAdapter({ - kind: "memory", - maxObjectBytes: 64, - }); - - await storage.putObject({ - body: new TextEncoder().encode("stream me"), - key: "tenant-1/stream.txt", - }); - - const stream = await storage.getObjectStream("tenant-1/stream.txt"); - const missing = await storage.getObjectStream("tenant-1/missing.txt"); - - expect(missing).toBeNull(); - await expect(readStream(stream)).resolves.toEqual(new TextEncoder().encode("stream me")); - }); - - it("copies memory metadata so callers cannot mutate retained object state", async () => { - const storage = createMemoryObjectStorageAdapter({ kind: "local", maxObjectBytes: 64 }); - const metadata = { sha256: "original" }; - - const putMetadata = await storage.putObject({ - body: new Uint8Array([1]), - key: "tenant-1/object.bin", - metadata, - }); - metadata.sha256 = "mutated"; - (putMetadata.metadata as Record).sha256 = "returned-mutation"; - - const firstHead = await storage.headObject("tenant-1/object.bin"); - expect(firstHead?.metadata).toEqual({ sha256: "original" }); - - if (firstHead) { - (firstHead.metadata as Record).sha256 = "head-mutation"; - } - - await expect(storage.headObject("tenant-1/object.bin")).resolves.toMatchObject({ - metadata: { sha256: "original" }, - }); - }); - - it("rejects memory writes that would exceed object count or total byte bounds", async () => { - const countBounded = createMemoryObjectStorageAdapter({ - kind: "local", - maxObjectBytes: 64, - maxObjects: 1, - }); - - await countBounded.putObject({ body: new Uint8Array([1]), key: "tenant-1/a.bin" }); - await expect( - countBounded.putObject({ body: new Uint8Array([1]), key: "tenant-1/b.bin" }), - ).rejects.toThrow("Object storage maxObjects=1 exceeded"); - - const byteBounded = createMemoryObjectStorageAdapter({ - kind: "local", - maxObjectBytes: 64, - maxTotalBytes: 2, - }); - - await byteBounded.putObject({ body: new Uint8Array([1, 2]), key: "tenant-1/a.bin" }); - await expect( - byteBounded.putObject({ body: new Uint8Array([3]), key: "tenant-1/b.bin" }), - ).rejects.toThrow("Object storage maxTotalBytes=2 exceeded"); - }); - - it("rejects unbounded list requests", async () => { - const storage = createMemoryObjectStorageAdapter({ kind: "local", maxObjectBytes: 64 }); - - await expect(storage.listObjects({ limit: 0, prefix: "tenant-1/" })).rejects.toThrow( - "Object list limit must be at least 1", - ); - }); - - it("rejects invalid capacity bounds", () => { - expect(() => createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 0 })).toThrow( - "Object storage maxObjectBytes must be at least 1", - ); - expect(() => - createMemoryObjectStorageAdapter({ kind: "memory", maxObjectBytes: 1, maxObjects: 0 }), - ).toThrow("Object storage maxObjects must be at least 1"); - expect(() => - createMemoryObjectStorageAdapter({ - kind: "memory", - maxObjectBytes: 1, - maxTotalBytes: 0, - }), - ).toThrow("Object storage maxTotalBytes must be at least 1"); - }); - - it("accounts for replacement bytes and preserves optional checksums", async () => { - const storage = createMemoryObjectStorageAdapter({ - kind: "memory", - maxObjectBytes: 4, - maxObjects: 1, - maxTotalBytes: 2, - }); - - await storage.putObject({ body: new Uint8Array([1, 2]), key: "object" }); - await expect( - storage.putObject({ - body: new Uint8Array([3]), - checksumSha256Base64: "checksum", - key: "object", - }), - ).resolves.toEqual({ - checksumSha256Base64: "checksum", - key: "object", - metadata: {}, - sizeBytes: 1, - }); - }); -}); - -describe("S3-compatible object storage adapter", () => { - it("presigns bounded single-part uploads without sending object bytes through the adapter", async () => { - const client = new FakeS3Client(); - const presigner = new FakeS3Presigner("https://objects.example/upload?signature=redacted"); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - maxDirectUploadBytes: 1_024, - now: () => 2_000_000, - presign: presigner.presign, - }); - - const upload = await storage.directUpload?.presignPutObject({ - checksumSha256Base64: "checksum-base64", - contentLength: 512, - contentType: "application/pdf", - expiresInSeconds: 60, - key: "namespaces/ns-1/spaces/space-1/uploads/session-1/source.pdf", - metadata: { upload_session_id: "session-1" }, - }); - - expect(upload).toEqual({ - expiresAt: 2_060_000, - headers: { - "content-type": "application/pdf", - "x-amz-checksum-sha256": "checksum-base64", - }, - method: "PUT", - url: "https://objects.example/upload?signature=redacted", - }); - expect(client.commands).toEqual([]); - expect(presigner.requests).toEqual([ - { - expiresInSeconds: 60, - input: { - Bucket: "knowledge-bucket", - ChecksumSHA256: "checksum-base64", - ContentLength: 512, - ContentType: "application/pdf", - Key: "namespaces/ns-1/spaces/space-1/uploads/session-1/source.pdf", - Metadata: { upload_session_id: "session-1" }, - }, - name: "PutObjectCommand", - }, - ]); - }); - - it("creates, presigns, completes, and aborts multipart uploads with fixed keys", async () => { - const client = new FakeS3Client({ - AbortMultipartUploadCommand: [{}], - CompleteMultipartUploadCommand: [{}], - CreateMultipartUploadCommand: [{ UploadId: "multipart-1" }], - }); - const presigner = new FakeS3Presigner("https://objects.example/part?signature=redacted"); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "r2", - now: () => 3_000_000, - presign: presigner.presign, - }); - const directUpload = storage.directUpload; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - const created = await directUpload.createMultipartUpload({ - contentType: "application/octet-stream", - key: "namespaces/ns-1/spaces/space-1/uploads/session-2/archive.bin", - metadata: { upload_session_id: "session-2" }, - }); - const part = await directUpload.presignMultipartPart({ - checksumSha256Base64: "part-checksum", - contentLength: 8 * 1024 * 1024, - expiresInSeconds: 45, - key: created.key, - partNumber: 7, - uploadId: created.uploadId, - }); - await directUpload.completeMultipartUpload({ - key: created.key, - parts: [{ checksumSha256Base64: "part-checksum", etag: '"etag-7"', partNumber: 7 }], - uploadId: created.uploadId, - }); - await directUpload.abortMultipartUpload({ - key: created.key, - uploadId: created.uploadId, - }); - - expect(created).toEqual({ key: created.key, uploadId: "multipart-1" }); - expect(part).toEqual({ - expiresAt: 3_045_000, - headers: { "x-amz-checksum-sha256": "part-checksum" }, - method: "PUT", - url: "https://objects.example/part?signature=redacted", - }); - expect(presigner.requests).toEqual([ - { - expiresInSeconds: 45, - input: { - Bucket: "knowledge-bucket", - ChecksumSHA256: "part-checksum", - ContentLength: 8 * 1024 * 1024, - Key: created.key, - PartNumber: 7, - UploadId: "multipart-1", - }, - name: "UploadPartCommand", - }, - ]); - expect(client.commands).toEqual([ - { - input: { - Bucket: "knowledge-bucket", - ChecksumAlgorithm: "SHA256", - ContentType: "application/octet-stream", - Key: created.key, - Metadata: { upload_session_id: "session-2" }, - }, - name: "CreateMultipartUploadCommand", - }, - { - input: { - Bucket: "knowledge-bucket", - Key: created.key, - MultipartUpload: { - Parts: [ - { - ChecksumSHA256: "part-checksum", - ETag: '"etag-7"', - PartNumber: 7, - }, - ], - }, - UploadId: "multipart-1", - }, - name: "CompleteMultipartUploadCommand", - }, - { - input: { Bucket: "knowledge-bucket", Key: created.key, UploadId: "multipart-1" }, - name: "AbortMultipartUploadCommand", - }, - ]); - }); - - it("streams the completed object to verify its full SHA-256 without buffering multipart data", async () => { - const body = new TextEncoder().encode("multipart-body"); - const checksumSha256Base64 = createHash("sha256").update(body).digest("base64"); - const streamedBody = async function* () { - yield body.slice(0, 5); - yield body.slice(5); - }; - const client = new FakeS3Client({ - GetObjectCommand: [ - { - Body: streamedBody(), - ContentLength: body.byteLength, - }, - { - Body: streamedBody(), - ContentLength: body.byteLength, - }, - ], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - const directUpload = storage.directUpload; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - await expect( - directUpload.verifyObjectSha256({ - checksumSha256Base64, - expectedSizeBytes: body.byteLength, - key: "namespaces/ns-1/spaces/space-1/uploads/session-2/source", - }), - ).resolves.toBe(true); - await expect( - directUpload.verifyObjectSha256({ - checksumSha256Base64: "wrong-checksum", - expectedSizeBytes: body.byteLength, - key: "namespaces/ns-1/spaces/space-1/uploads/session-2/source", - }), - ).resolves.toBe(false); - expect(client.commands).toEqual([ - { - input: { - Bucket: "knowledge-bucket", - Key: "namespaces/ns-1/spaces/space-1/uploads/session-2/source", - }, - name: "GetObjectCommand", - }, - { - input: { - Bucket: "knowledge-bucket", - Key: "namespaces/ns-1/spaces/space-1/uploads/session-2/source", - }, - name: "GetObjectCommand", - }, - ]); - }); - - it("best-effort cancels and releases verification bodies on metadata size mismatch", async () => { - let cancelled = 0; - const body = new ReadableStream({ - cancel() { - cancelled += 1; - throw new Error("verification cleanup failed"); - }, - start() {}, - }); - const client = new FakeS3Client({ - GetObjectCommand: [{ Body: body, ContentLength: 9 }], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect( - storage.directUpload?.verifyObjectSha256({ - checksumSha256Base64: "unused-checksum", - expectedSizeBytes: 10, - key: "namespaces/ns-1/spaces/space-1/uploads/session-3/source", - }), - ).resolves.toBe(false); - expect(cancelled).toBe(1); - expect(body.locked).toBe(false); - }); - - it("does not mask a streamed size mismatch when verification cleanup fails", async () => { - let cancelled = 0; - const body = new ReadableStream({ - cancel() { - cancelled += 1; - throw new Error("verification cleanup failed"); - }, - start(controller) { - controller.enqueue(new Uint8Array([1, 2])); - }, - }); - const client = new FakeS3Client({ GetObjectCommand: [{ Body: body }] }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect( - storage.directUpload?.verifyObjectSha256({ - checksumSha256Base64: "unused-checksum", - expectedSizeBytes: 1, - key: "namespaces/ns-1/spaces/space-1/uploads/session-4/source", - }), - ).resolves.toBe(false); - expect(cancelled).toBe(1); - expect(body.locked).toBe(false); - }); - - it("rejects unsafe direct-upload bounds before signing or sending a command", async () => { - const client = new FakeS3Client({ CreateMultipartUploadCommand: [{}] }); - const presigner = new FakeS3Presigner("https://objects.example/upload"); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - maxDirectUploadBytes: 10, - presign: presigner.presign, - }); - const directUpload = storage.directUpload; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - await expect( - directUpload.presignPutObject({ - contentLength: 11, - expiresInSeconds: 60, - key: "fixed-key", - }), - ).rejects.toThrow("exceeds maxDirectUploadBytes=10"); - await expect( - directUpload.presignMultipartPart({ - contentLength: 1, - expiresInSeconds: 0, - key: "fixed-key", - partNumber: 0, - uploadId: "upload-1", - }), - ).rejects.toThrow("expiresInSeconds must be between 1 and 900"); - await expect(directUpload.createMultipartUpload({ key: "fixed-key" })).rejects.toThrow( - "did not return an upload id", - ); - expect(presigner.requests).toEqual([]); - }); - - it("installs an abort-only multipart lifecycle rule without replacing bucket retention", async () => { - const retainedRule = { - Expiration: { Days: 365 }, - Filter: { Prefix: "archive/" }, - ID: "archive-retention", - Status: "Enabled", - }; - const client = new FakeS3Client({ - GetBucketLifecycleConfigurationCommand: [{ Rules: [retainedRule] }], - PutBucketLifecycleConfigurationCommand: [{}], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - const directUpload = storage.directUpload; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - await directUpload.ensureIncompleteMultipartUploadLifecycle({ daysAfterInitiation: 2 }); - - expect(client.commands).toEqual([ - { - input: { Bucket: "knowledge-bucket" }, - name: "GetBucketLifecycleConfigurationCommand", - }, - { - input: { - Bucket: "knowledge-bucket", - LifecycleConfiguration: { - Rules: [ - retainedRule, - { - AbortIncompleteMultipartUpload: { DaysAfterInitiation: 2 }, - Filter: { Prefix: "" }, - ID: "knowledge-fs-incomplete-multipart", - Status: "Enabled", - }, - ], - }, - }, - name: "PutBucketLifecycleConfigurationCommand", - }, - ]); - }); - - it("puts objects through the S3 API and returns object metadata", async () => { - const client = new FakeS3Client(); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - maxObjectBytes: 64, - }); - const body = new Uint8Array([1, 2, 3]); - - const metadata = await storage.putObject({ - body, - contentType: "application/octet-stream", - key: "tenant-1/documents/object.bin", - metadata: { sha256: "abc" }, - }); - - expect(metadata).toEqual({ - contentType: "application/octet-stream", - key: "tenant-1/documents/object.bin", - metadata: { sha256: "abc" }, - sizeBytes: 3, - }); - expect(client.commands).toEqual([ - { - input: { - Body: body, - Bucket: "knowledge-bucket", - ContentType: "application/octet-stream", - Key: "tenant-1/documents/object.bin", - Metadata: { sha256: "abc" }, - }, - name: "PutObjectCommand", - }, - ]); - }); - - it("rejects oversized objects before sending an S3 command", async () => { - const client = new FakeS3Client(); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - maxObjectBytes: 2, - }); - - await expect( - storage.putObject({ - body: new Uint8Array([1, 2, 3]), - key: "tenant-1/too-large.bin", - }), - ).rejects.toThrow("Object tenant-1/too-large.bin exceeds maxObjectBytes=2"); - expect(client.commands).toEqual([]); - }); - - it("gets object bytes as copies so callers cannot mutate retained state", async () => { - const storedBody = new Uint8Array([1, 2, 3]); - const client = new FakeS3Client({ - GetObjectCommand: [{ Body: storedBody }], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - const first = await storage.getObject("tenant-1/documents/object.bin"); - - expect(first).toEqual(new Uint8Array([1, 2, 3])); - - if (first) { - first[0] = 9; - } - - expect(storedBody).toEqual(new Uint8Array([1, 2, 3])); - expect(client.commands[0]).toEqual({ - input: { - Bucket: "knowledge-bucket", - Key: "tenant-1/documents/object.bin", - }, - name: "GetObjectCommand", - }); - }); - - it("reads S3 body variants without leaking mutable buffers", async () => { - const client = new FakeS3Client({ - GetObjectCommand: [ - { - Body: "plain text", - }, - { - Body: { - async *[Symbol.asyncIterator]() { - yield new Uint8Array([1]); - yield "2"; - }, - }, - }, - { - Body: { - transformToByteArray: async () => new Uint8Array([3, 4]), - }, - }, - {}, - ], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect(storage.getObject("tenant-1/text.txt")).resolves.toEqual( - new TextEncoder().encode("plain text"), - ); - await expect(storage.getObject("tenant-1/stream.bin")).resolves.toEqual( - new Uint8Array([1, 50]), - ); - await expect(storage.getObject("tenant-1/blob.bin")).resolves.toEqual(new Uint8Array([3, 4])); - await expect(storage.getObject("tenant-1/empty.bin")).resolves.toEqual(new Uint8Array()); - }); - - it("can return an S3 object stream for streaming consumers", async () => { - const client = new FakeS3Client({ - GetObjectCommand: [ - { - Body: { - async *[Symbol.asyncIterator]() { - yield new Uint8Array([1, 2]); - yield new Uint8Array([3]); - }, - }, - }, - ], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - maxObjectBytes: 64, - }); - - const stream = await storage.getObjectStream("tenant-1/stream.bin"); - - await expect(readStream(stream)).resolves.toEqual(new Uint8Array([1, 2, 3])); - }); - - it("closes an async-iterable S3 body exactly once after a normal stream read", async () => { - let nextCalls = 0; - let returnCalls = 0; - const iterator: AsyncIterator = { - next: async () => { - nextCalls += 1; - return nextCalls === 1 - ? { done: false, value: new Uint8Array([1, 2]) } - : { done: true, value: undefined }; - }, - return: async () => { - returnCalls += 1; - return { done: true, value: undefined }; - }, - }; - const body: AsyncIterable = { - [Symbol.asyncIterator]: () => iterator, - }; - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ GetObjectCommand: [{ Body: body }] }), - kind: "s3-compatible", - maxObjectBytes: 64, - }); - - await expect(readStream(await storage.getObjectStream("tenant-1/stream.bin"))).resolves.toEqual( - new Uint8Array([1, 2]), - ); - expect(returnCalls).toBe(1); - }); - - it("closes an async-iterable S3 body without masking a stream size error", async () => { - let returnCalls = 0; - const body: AsyncIterable = { - [Symbol.asyncIterator]: () => ({ - next: async () => ({ done: false, value: new Uint8Array([1, 2, 3]) }), - return: async () => { - returnCalls += 1; - throw new Error("iterator cleanup failed"); - }, - }), - }; - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ GetObjectCommand: [{ Body: body }] }), - kind: "s3-compatible", - maxObjectBytes: 2, - }); - - await expect( - readStream(await storage.getObjectStream("tenant-1/too-large-stream.bin")), - ).rejects.toThrow("Object tenant-1/too-large-stream.bin exceeds maxObjectBytes=2"); - expect(returnCalls).toBe(1); - }); - - it("releases a readable S3 body reader after a normal stream read", async () => { - const body = new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array([1, 2])); - controller.close(); - }, - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ GetObjectCommand: [{ Body: body }] }), - kind: "s3-compatible", - maxObjectBytes: 64, - }); - - await expect(readStream(await storage.getObjectStream("tenant-1/stream.bin"))).resolves.toEqual( - new Uint8Array([1, 2]), - ); - expect(body.locked).toBe(false); - }); - - it("cancels and releases a readable S3 body without masking a stream size error", async () => { - const cancelReasons: unknown[] = []; - const body = new ReadableStream({ - cancel(reason) { - cancelReasons.push(reason); - throw new Error("reader cleanup failed"); - }, - start(controller) { - controller.enqueue(new Uint8Array([1, 2, 3])); - }, - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ GetObjectCommand: [{ Body: body }] }), - kind: "s3-compatible", - maxObjectBytes: 2, - }); - - await expect( - readStream(await storage.getObjectStream("tenant-1/too-large-stream.bin")), - ).rejects.toThrow("Object tenant-1/too-large-stream.bin exceeds maxObjectBytes=2"); - expect(cancelReasons).toHaveLength(1); - expect(cancelReasons[0]).toMatchObject({ - message: "Object tenant-1/too-large-stream.bin exceeds maxObjectBytes=2", - }); - expect(body.locked).toBe(false); - }); - - it("rejects externally oversized S3 reads before retaining unbounded bytes", async () => { - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ - GetObjectCommand: [ - { - Body: new Uint8Array([1]), - ContentLength: 3, - }, - { - Body: { - async *[Symbol.asyncIterator]() { - yield new Uint8Array([1, 2]); - yield new Uint8Array([3]); - }, - }, - }, - ], - }), - kind: "s3-compatible", - maxObjectBytes: 2, - }); - - await expect(storage.getObject("tenant-1/too-large-by-head.bin")).rejects.toThrow( - "Object tenant-1/too-large-by-head.bin exceeds maxObjectBytes=2", - ); - await expect(storage.getObject("tenant-1/too-large-stream.bin")).rejects.toThrow( - "Object tenant-1/too-large-stream.bin exceeds maxObjectBytes=2", - ); - }); - - it("best-effort cancels an oversized readable body without masking metadata failure", async () => { - const cancelReasons: unknown[] = []; - const body = new ReadableStream({ - cancel(reason) { - cancelReasons.push(reason); - throw new Error("readable cleanup failed"); - }, - start() {}, - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ - GetObjectCommand: [{ Body: body, ContentLength: 3 }], - }), - kind: "s3-compatible", - maxObjectBytes: 2, - }); - - await expect(storage.getObject("tenant-1/too-large-readable.bin")).rejects.toThrow( - "Object tenant-1/too-large-readable.bin exceeds maxObjectBytes=2", - ); - expect(cancelReasons).toHaveLength(1); - expect(cancelReasons[0]).toMatchObject({ - message: "Object tenant-1/too-large-readable.bin exceeds maxObjectBytes=2", - }); - }); - - it("best-effort returns an oversized async body without reading or masking metadata failure", async () => { - let iteratorCalls = 0; - let nextCalls = 0; - let returnCalls = 0; - const body: AsyncIterable = { - [Symbol.asyncIterator]() { - iteratorCalls += 1; - return { - next: async () => { - nextCalls += 1; - return { done: false, value: new Uint8Array([1]) }; - }, - return: async () => { - returnCalls += 1; - throw new Error("iterator cleanup failed"); - }, - }; - }, - }; - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ - GetObjectCommand: [{ Body: body, ContentLength: 3 }], - }), - kind: "s3-compatible", - maxObjectBytes: 2, - }); - - await expect(storage.getObjectStream("tenant-1/too-large-iterable.bin")).rejects.toThrow( - "Object tenant-1/too-large-iterable.bin exceeds maxObjectBytes=2", - ); - expect(iteratorCalls).toBe(1); - expect(nextCalls).toBe(0); - expect(returnCalls).toBe(1); - }); - - it("returns null when getObject or headObject receives a missing-key S3 error", async () => { - const client = new FakeS3Client({ - GetObjectCommand: [s3Error("NoSuchKey", 404)], - HeadObjectCommand: [s3Error("NotFound", 404)], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "r2", - }); - - await expect(storage.getObject("tenant-1/missing.txt")).resolves.toBeNull(); - await expect(storage.headObject("tenant-1/missing.txt")).resolves.toBeNull(); - }); - - it("rethrows non-missing S3 read errors and unsupported bodies", async () => { - const client = new FakeS3Client({ - GetObjectCommand: [{ Body: 123 }, new Error("s3 unavailable")], - HeadObjectCommand: [new Error("s3 unavailable")], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect(storage.getObject("tenant-1/bad-body.bin")).rejects.toThrow( - "Unsupported S3 object body type", - ); - await expect(storage.getObject("tenant-1/error.bin")).rejects.toThrow("s3 unavailable"); - await expect(storage.headObject("tenant-1/error.bin")).rejects.toThrow("s3 unavailable"); - }); - - it("heads and deletes objects through S3", async () => { - const client = new FakeS3Client({ - HeadObjectCommand: [ - { - ChecksumSHA256: "checksum-base64", - ContentLength: 42, - ContentType: "text/plain", - Metadata: { sha256: "abc" }, - }, - ], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect(storage.headObject("tenant-1/documents/object.txt")).resolves.toEqual({ - checksumSha256Base64: "checksum-base64", - contentType: "text/plain", - key: "tenant-1/documents/object.txt", - metadata: { sha256: "abc" }, - sizeBytes: 42, - }); - - await storage.deleteObject("tenant-1/documents/object.txt"); - - expect(client.commands.at(-1)).toEqual({ - input: { - Bucket: "knowledge-bucket", - Key: "tenant-1/documents/object.txt", - }, - name: "DeleteObjectCommand", - }); - expect(client.commands[0]).toEqual({ - input: { - Bucket: "knowledge-bucket", - ChecksumMode: "ENABLED", - Key: "tenant-1/documents/object.txt", - }, - name: "HeadObjectCommand", - }); - }); - - it("maps sparse S3 head and list responses to safe object metadata defaults", async () => { - const client = new FakeS3Client({ - HeadObjectCommand: [{ Metadata: { sha256: "abc", ignored: 123 } }], - ListObjectsV2Command: [{ Contents: [{ Size: 7 }, { Key: "tenant-1/a.txt" }] }, {}], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect(storage.headObject("tenant-1/object.txt")).resolves.toEqual({ - key: "tenant-1/object.txt", - metadata: { sha256: "abc" }, - sizeBytes: 0, - }); - await expect(storage.listObjects({ limit: 10, prefix: "tenant-1/" })).resolves.toEqual({ - objects: [{ key: "tenant-1/a.txt", metadata: {}, sizeBytes: 0 }], - }); - await expect(storage.listObjects({ limit: 10, prefix: "tenant-2/" })).resolves.toEqual({ - objects: [], - }); - }); - - it("lists objects with explicit limits and S3 continuation cursors", async () => { - const client = new FakeS3Client({ - ListObjectsV2Command: [ - { - Contents: [ - { Key: "tenant-1/a.txt", Size: 1 }, - { Key: "tenant-1/b.txt", Size: 2 }, - ], - NextContinuationToken: "token-2", - }, - ], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect( - storage.listObjects({ - cursor: "token-1", - limit: 2, - prefix: "tenant-1/", - }), - ).resolves.toEqual({ - nextCursor: "token-2", - objects: [ - { key: "tenant-1/a.txt", metadata: {}, sizeBytes: 1 }, - { key: "tenant-1/b.txt", metadata: {}, sizeBytes: 2 }, - ], - }); - expect(client.commands[0]).toEqual({ - input: { - Bucket: "knowledge-bucket", - ContinuationToken: "token-1", - MaxKeys: 2, - Prefix: "tenant-1/", - }, - name: "ListObjectsV2Command", - }); - }); - - it("rejects unbounded S3 list requests", async () => { - const client = new FakeS3Client(); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect(storage.listObjects({ limit: 0, prefix: "tenant-1/" })).rejects.toThrow( - "Object list limit must be at least 1", - ); - expect(client.commands).toEqual([]); - }); - - it("reports S3 bucket health from HeadBucket", async () => { - const healthy = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ HeadBucketCommand: [{}] }), - kind: "s3-compatible", - }); - const unhealthy = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client({ HeadBucketCommand: [new Error("offline")] }), - kind: "s3-compatible", - }); - - await expect(healthy.health()).resolves.toBe(true); - await expect(unhealthy.health()).resolves.toBe(false); - }); - - it("puts sparse S3 objects without optional content fields", async () => { - const client = new FakeS3Client(); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect( - storage.putObject({ - body: new Uint8Array([1]), - checksumSha256Base64: "checksum", - key: "tenant-1/sparse.bin", - }), - ).resolves.toEqual({ - checksumSha256Base64: "checksum", - key: "tenant-1/sparse.bin", - metadata: {}, - sizeBytes: 1, - }); - expect(client.commands).toEqual([ - { - input: { - Body: new Uint8Array([1]), - Bucket: "knowledge-bucket", - ChecksumSHA256: "checksum", - Key: "tenant-1/sparse.bin", - Metadata: {}, - }, - name: "PutObjectCommand", - }, - ]); - }); - - it("streams empty, byte, and string S3 bodies and handles missing stream objects", async () => { - const client = new FakeS3Client({ - GetObjectCommand: [ - {}, - { Body: new Uint8Array([1, 2]) }, - { Body: "text" }, - { Body: 123 }, - s3Error("OtherS3Error", 404), - ], - }); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - - await expect(readStream(await storage.getObjectStream("empty"))).resolves.toEqual( - new Uint8Array(), - ); - await expect(readStream(await storage.getObjectStream("bytes"))).resolves.toEqual( - new Uint8Array([1, 2]), - ); - await expect(readStream(await storage.getObjectStream("string"))).resolves.toEqual( - new TextEncoder().encode("text"), - ); - await expect(storage.getObjectStream("unsupported")).rejects.toThrow( - "Unsupported S3 object body type", - ); - await expect(storage.getObjectStream("missing")).resolves.toBeNull(); - }); - - it("omits optional fields from sparse direct-upload commands", async () => { - const client = new FakeS3Client({ CompleteMultipartUploadCommand: [{}] }); - const presigner = new FakeS3Presigner("https://objects.example/upload"); - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - presign: presigner.presign, - }); - const directUpload = storage.directUpload; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - await directUpload.completeMultipartUpload({ - key: "fixed-key", - parts: [{ etag: "etag-1", partNumber: 1 }], - uploadId: "upload-1", - }); - await directUpload.presignMultipartPart({ - contentLength: 1, - expiresInSeconds: 60, - key: "fixed-key", - partNumber: 1, - uploadId: "upload-1", - }); - await directUpload.presignPutObject({ - contentLength: 1, - expiresInSeconds: 60, - key: "fixed-key", - }); - - expect(presigner.requests).toHaveLength(2); - expect(client.commands[0]).toMatchObject({ - input: { - MultipartUpload: { Parts: [{ ETag: "etag-1", PartNumber: 1 }] }, - }, - name: "CompleteMultipartUploadCommand", - }); - }); - - it("rejects invalid direct-upload identities, parts, lifecycle bounds, and metadata", async () => { - expect(() => - createS3ObjectStorageAdapter({ - bucket: " ", - client: new FakeS3Client(), - kind: "s3-compatible", - }), - ).toThrow("Object storage direct upload bucket is required"); - expect(() => - createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client(), - kind: "s3-compatible", - maxDirectUploadBytes: 0, - }), - ).toThrow("maxDirectUploadBytes must be a positive safe integer"); - - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client(), - kind: "s3-compatible", - presign: new FakeS3Presigner("https://objects.example/upload").presign, - }); - const directUpload = storage.directUpload; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - await expect( - directUpload.ensureIncompleteMultipartUploadLifecycle({ daysAfterInitiation: 0 }), - ).rejects.toThrow("daysAfterInitiation must be between 1 and 365"); - await expect( - directUpload.presignMultipartPart({ - contentLength: 1, - expiresInSeconds: 60, - key: "fixed-key", - partNumber: 0, - uploadId: "upload-1", - }), - ).rejects.toThrow("partNumber must be between 1 and 10000"); - await expect( - directUpload.completeMultipartUpload({ key: "fixed-key", parts: [], uploadId: "upload-1" }), - ).rejects.toThrow("requires 1-10000 parts"); - await expect( - directUpload.completeMultipartUpload({ - key: "fixed-key", - parts: [ - { etag: "etag-1", partNumber: 1 }, - { etag: "etag-2", partNumber: 1 }, - ], - uploadId: "upload-1", - }), - ).rejects.toThrow("parts must be strictly increasing"); - await expect(directUpload.createMultipartUpload({ key: "invalid\0key" })).rejects.toThrow( - "direct upload key is invalid", - ); - await expect( - directUpload.presignPutObject({ - contentLength: 1, - expiresInSeconds: 60, - key: "fixed-key", - metadata: { "invalid key": "value" }, - }), - ).rejects.toThrow("direct upload metadata is invalid"); - }); - - it("rejects invalid clocks and presigner URLs", async () => { - const input = { contentLength: 1, expiresInSeconds: 60, key: "fixed-key" }; - const invalidClock = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client(), - kind: "s3-compatible", - now: () => -1, - presign: new FakeS3Presigner("https://objects.example/upload").presign, - }); - const malformedUrl = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client(), - kind: "s3-compatible", - presign: new FakeS3Presigner("not-a-url").presign, - }); - const unsafeUrl = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: new FakeS3Client(), - kind: "s3-compatible", - presign: new FakeS3Presigner("ftp://user:password@objects.example/upload").presign, - }); - - await expect(invalidClock.directUpload?.presignPutObject(input)).rejects.toThrow( - "clock must return a non-negative integer", - ); - await expect(malformedUrl.directUpload?.presignPutObject(input)).rejects.toThrow( - "presigner returned an invalid URL", - ); - await expect(unsafeUrl.directUpload?.presignPutObject(input)).rejects.toThrow( - "presigner returned an invalid URL", - ); - }); - - it("ignores missing multipart aborts and rethrows other abort failures", async () => { - const noSuchUpload = new Error("missing"); - Object.defineProperty(noSuchUpload, "name", { value: "NoSuchUpload" }); - const failures: unknown[] = [ - noSuchUpload, - s3Error("OtherS3Error", 404), - {}, - new Error("abort failed"), - ]; - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client: { - async send() { - throw failures.shift(); - }, - }, - kind: "s3-compatible", - }); - const directUpload = storage.directUpload; - const input = { key: "fixed-key", uploadId: "upload-1" }; - - if (!directUpload) throw new Error("Expected S3 direct upload support"); - await expect(directUpload.abortMultipartUpload(input)).resolves.toBeUndefined(); - await expect(directUpload.abortMultipartUpload(input)).resolves.toBeUndefined(); - await expect(directUpload.abortMultipartUpload(input)).rejects.toEqual({}); - await expect(directUpload.abortMultipartUpload(input)).rejects.toThrow("abort failed"); - }); - - it("recovers from missing lifecycle configuration and replaces its own rule", async () => { - const missing = new Error("missing lifecycle"); - Object.defineProperty(missing, "name", { value: "NoSuchLifecycleConfiguration" }); - const missingClient = new FakeS3Client({ - GetBucketLifecycleConfigurationCommand: [missing], - PutBucketLifecycleConfigurationCommand: [{}], - }); - const replacementClient = new FakeS3Client({ - GetBucketLifecycleConfigurationCommand: [ - { - Rules: [ - null, - { - AbortIncompleteMultipartUpload: { DaysAfterInitiation: 7 }, - ID: "knowledge-fs-incomplete-multipart", - Status: "Enabled", - }, - ], - }, - ], - PutBucketLifecycleConfigurationCommand: [{}], - }); - const malformedClient = new FakeS3Client({ - GetBucketLifecycleConfigurationCommand: [{ Rules: "invalid" }], - PutBucketLifecycleConfigurationCommand: [{}], - }); - - for (const client of [missingClient, replacementClient, malformedClient]) { - const storage = createS3ObjectStorageAdapter({ - bucket: "knowledge-bucket", - client, - kind: "s3-compatible", - }); - await storage.directUpload?.ensureIncompleteMultipartUploadLifecycle({ - daysAfterInitiation: 3, - }); - } - - expect(replacementClient.commands[1]).toMatchObject({ - input: { - LifecycleConfiguration: { - Rules: [ - { - AbortIncompleteMultipartUpload: { DaysAfterInitiation: 3 }, - ID: "knowledge-fs-incomplete-multipart", - }, - ], - }, - }, - }); - }); -}); - -type FakeS3Response = Error | unknown; - -class FakeS3Client { - readonly commands: { input: unknown; name: string }[] = []; - private readonly responses: Record; - - constructor(responses: Record = {}) { - this.responses = responses; - } - - async send(command: { - readonly input: unknown; - readonly constructor: { readonly name: string }; - }) { - const name = command.constructor.name; - this.commands.push({ input: command.input, name }); - - const response = this.responses[name]?.shift() ?? {}; - - if (response instanceof Error) { - throw response; - } - - return response; - } -} - -class FakeS3Presigner { - readonly requests: { expiresInSeconds: number; input: unknown; name: string }[] = []; - readonly presign: (input: { - readonly command: { readonly input: unknown; readonly constructor: { readonly name: string } }; - readonly expiresInSeconds: number; - }) => Promise; - - constructor(url: string) { - this.presign = async ({ command, expiresInSeconds }) => { - this.requests.push({ - expiresInSeconds, - input: command.input, - name: command.constructor.name, - }); - return url; - }; - } -} - -function s3Error(name: string, statusCode: number): Error { - const error = new Error(name) as Error & { - readonly $metadata: { readonly httpStatusCode: number }; - readonly name: string; - }; - - Object.defineProperty(error, "name", { value: name }); - Object.defineProperty(error, "$metadata", { - value: { httpStatusCode: statusCode }, - }); - - return error; -} - -async function readStream(stream: ReadableStream | null): Promise { - if (!stream) { - throw new Error("Expected stream"); - } - - const reader = stream.getReader(); - const chunks: Uint8Array[] = []; - let totalBytes = 0; - - while (true) { - const result = await reader.read(); - - if (result.done) { - break; - } - - chunks.push(result.value); - totalBytes += result.value.byteLength; - } - - const bytes = new Uint8Array(totalBytes); - let offset = 0; - - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - - return bytes; -} diff --git a/knowledge-fs/packages/adapters/src/object-storage.ts b/knowledge-fs/packages/adapters/src/object-storage.ts deleted file mode 100644 index 945cfb48f15..00000000000 --- a/knowledge-fs/packages/adapters/src/object-storage.ts +++ /dev/null @@ -1,1122 +0,0 @@ -import { createHash } from "node:crypto"; -import { - AbortMultipartUploadCommand, - CompleteMultipartUploadCommand, - CreateMultipartUploadCommand, - DeleteObjectCommand, - GetBucketLifecycleConfigurationCommand, - GetObjectCommand, - HeadBucketCommand, - HeadObjectCommand, - type LifecycleRule, - ListObjectsV2Command, - PutBucketLifecycleConfigurationCommand, - PutObjectCommand, - S3Client, - UploadPartCommand, -} from "@aws-sdk/client-s3"; -import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; -import type { - CompleteMultipartObjectUploadInput, - CreateMultipartObjectUploadInput, - ListObjectsInput, - ListObjectsResult, - ObjectMetadata, - ObjectStorageAdapter, - ObjectStorageDirectUploadAdapter, - PresignMultipartObjectPartInput, - PresignPutObjectInput, - PresignedObjectUpload, - PutObjectInput, -} from "@knowledge/core"; - -export interface MemoryObjectStorageOptions { - readonly kind: ObjectStorageAdapter["kind"]; - readonly maxObjectBytes: number; - readonly maxObjects?: number; - readonly maxTotalBytes?: number; -} - -export interface S3ObjectStorageOptions { - readonly bucket: string; - readonly client?: S3ObjectStorageClient; - readonly kind: Extract; - readonly maxDirectUploadBytes?: number; - readonly maxObjectBytes?: number; - readonly now?: () => number; - readonly presign?: S3ObjectStoragePresigner; -} - -export interface S3ObjectStorageClient { - send(command: object): Promise; -} - -export interface S3ObjectStoragePresignInput { - readonly client: S3ObjectStorageClient; - readonly command: { - readonly input: unknown; - readonly constructor: { readonly name: string }; - }; - readonly expiresInSeconds: number; -} - -export type S3ObjectStoragePresigner = (input: S3ObjectStoragePresignInput) => Promise; - -interface StoredObject { - readonly body: Uint8Array; - readonly metadata: ObjectMetadata; -} - -const defaultMaxObjectBytes = 64 * 1024 * 1024; -const defaultMaxDirectUploadBytes = 5 * 1024 * 1024 * 1024 * 1024; -const maxPresignTtlSeconds = 15 * 60; -const incompleteMultipartLifecycleRuleId = "knowledge-fs-incomplete-multipart"; - -export function createMemoryObjectStorageAdapter({ - kind, - maxObjectBytes, - maxObjects = 1_000, - maxTotalBytes = maxObjectBytes * maxObjects, -}: MemoryObjectStorageOptions): ObjectStorageAdapter { - if (maxObjectBytes < 1) { - throw new Error("Object storage maxObjectBytes must be at least 1"); - } - - if (maxObjects < 1) { - throw new Error("Object storage maxObjects must be at least 1"); - } - - if (maxTotalBytes < 1) { - throw new Error("Object storage maxTotalBytes must be at least 1"); - } - - const objects = new Map(); - let totalBytes = 0; - - return { - kind, - deleteObject: async (key) => { - const stored = objects.get(key); - - if (stored) { - totalBytes -= stored.body.byteLength; - objects.delete(key); - } - }, - getObject: async (key) => { - const stored = objects.get(key); - - return stored ? copyBytes(stored.body) : null; - }, - getObjectStream: async (key) => { - const stored = objects.get(key); - - return stored ? bytesToStream(copyBytes(stored.body)) : null; - }, - health: async () => true, - headObject: async (key) => { - const stored = objects.get(key); - - return stored ? cloneObjectMetadata(stored.metadata) : null; - }, - listObjects: async (input) => listObjects(objects, input), - putObject: async (input) => { - if (input.body.byteLength > maxObjectBytes) { - throw new Error(`Object ${input.key} exceeds maxObjectBytes=${maxObjectBytes}`); - } - - const body = copyBytes(input.body); - const existing = objects.get(input.key); - const nextObjectCount = objects.size + (existing ? 0 : 1); - const nextTotalBytes = totalBytes - (existing?.body.byteLength ?? 0) + body.byteLength; - - if (nextObjectCount > maxObjects) { - throw new Error(`Object storage maxObjects=${maxObjects} exceeded`); - } - - if (nextTotalBytes > maxTotalBytes) { - throw new Error(`Object storage maxTotalBytes=${maxTotalBytes} exceeded`); - } - - const metadata = { - ...(input.checksumSha256Base64 ? { checksumSha256Base64: input.checksumSha256Base64 } : {}), - key: input.key, - metadata: cloneMetadata(input.metadata), - sizeBytes: body.byteLength, - ...(input.contentType ? { contentType: input.contentType } : {}), - }; - - objects.set(input.key, { - body, - metadata, - }); - totalBytes = nextTotalBytes; - - return cloneObjectMetadata(metadata); - }, - }; -} - -export function createS3ObjectStorageAdapter({ - bucket, - client = new S3Client({}), - kind, - maxDirectUploadBytes = defaultMaxDirectUploadBytes, - maxObjectBytes = defaultMaxObjectBytes, - now = Date.now, - presign = defaultS3ObjectStoragePresigner, -}: S3ObjectStorageOptions): ObjectStorageAdapter { - const normalizedBucket = requiredDirectUploadString(bucket, "bucket"); - positiveSafeInteger(maxDirectUploadBytes, "maxDirectUploadBytes"); - const directUpload = createS3DirectUploadAdapter({ - bucket: normalizedBucket, - client, - maxDirectUploadBytes, - now, - presign, - }); - - return { - kind, - directUpload, - deleteObject: async (key) => { - await client.send( - new DeleteObjectCommand({ - Bucket: normalizedBucket, - Key: key, - }), - ); - }, - getObject: async (key) => { - try { - const result = await client.send( - new GetObjectCommand({ - Bucket: normalizedBucket, - Key: key, - }), - ); - const body = getRecordField(result, "Body"); - const contentLength = readNumber(result, "ContentLength"); - - await assertObjectMetadataSize(body, key, contentLength, maxObjectBytes); - - if (body === undefined || body === null) { - return new Uint8Array(); - } - - return copyBytes(await bodyToBytes(body, key, maxObjectBytes)); - } catch (error) { - if (isMissingObjectError(error)) { - return null; - } - - throw error; - } - }, - getObjectStream: async (key) => { - try { - const result = await client.send( - new GetObjectCommand({ - Bucket: normalizedBucket, - Key: key, - }), - ); - const body = getRecordField(result, "Body"); - const contentLength = readNumber(result, "ContentLength"); - - await assertObjectMetadataSize(body, key, contentLength, maxObjectBytes); - - return bodyToStream(body, key, maxObjectBytes); - } catch (error) { - if (isMissingObjectError(error)) { - return null; - } - - throw error; - } - }, - health: async () => { - try { - await client.send(new HeadBucketCommand({ Bucket: normalizedBucket })); - return true; - } catch { - return false; - } - }, - headObject: async (key) => { - try { - const result = await client.send( - new HeadObjectCommand({ - Bucket: normalizedBucket, - ChecksumMode: "ENABLED", - Key: key, - }), - ); - - const checksumSha256Base64 = readString(result, "ChecksumSHA256"); - const contentType = readString(result, "ContentType"); - - return { - ...(checksumSha256Base64 ? { checksumSha256Base64 } : {}), - key, - metadata: readMetadata(result), - sizeBytes: readNumber(result, "ContentLength") ?? 0, - ...(contentType ? { contentType } : {}), - }; - } catch (error) { - if (isMissingObjectError(error)) { - return null; - } - - throw error; - } - }, - listObjects: async (input) => { - if (input.limit < 1) { - throw new Error("Object list limit must be at least 1"); - } - - const result = await client.send( - new ListObjectsV2Command({ - Bucket: normalizedBucket, - ...(input.cursor ? { ContinuationToken: input.cursor } : {}), - MaxKeys: input.limit, - Prefix: input.prefix, - }), - ); - - const objects = readContents(result).map((object) => ({ - key: object.key, - metadata: {}, - sizeBytes: object.sizeBytes, - })); - const nextCursor = readString(result, "NextContinuationToken"); - - return { - objects, - ...(nextCursor ? { nextCursor } : {}), - }; - }, - putObject: async (input) => { - if (input.body.byteLength > maxObjectBytes) { - throw new Error(`Object ${input.key} exceeds maxObjectBytes=${maxObjectBytes}`); - } - - await client.send( - new PutObjectCommand({ - Body: input.body, - Bucket: normalizedBucket, - ...(input.checksumSha256Base64 ? { ChecksumSHA256: input.checksumSha256Base64 } : {}), - ...(input.contentType ? { ContentType: input.contentType } : {}), - Key: input.key, - Metadata: input.metadata ?? {}, - }), - ); - - return { - ...(input.checksumSha256Base64 ? { checksumSha256Base64: input.checksumSha256Base64 } : {}), - key: input.key, - metadata: cloneMetadata(input.metadata), - sizeBytes: input.body.byteLength, - ...(input.contentType ? { contentType: input.contentType } : {}), - }; - }, - }; -} - -function createS3DirectUploadAdapter({ - bucket, - client, - maxDirectUploadBytes, - now, - presign, -}: { - readonly bucket: string; - readonly client: S3ObjectStorageClient; - readonly maxDirectUploadBytes: number; - readonly now: () => number; - readonly presign: S3ObjectStoragePresigner; -}): ObjectStorageDirectUploadAdapter { - return { - abortMultipartUpload: async (rawInput) => { - const input = normalizeMultipartIdentity(rawInput); - try { - await client.send( - new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: input.key, - UploadId: input.uploadId, - }), - ); - } catch (error) { - if (!isMissingMultipartUploadError(error)) throw error; - } - }, - completeMultipartUpload: async (rawInput) => { - const input = normalizeCompleteMultipartInput(rawInput); - await client.send( - new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: input.key, - MultipartUpload: { - Parts: input.parts.map((part) => ({ - ...(part.checksumSha256Base64 ? { ChecksumSHA256: part.checksumSha256Base64 } : {}), - ETag: part.etag, - PartNumber: part.partNumber, - })), - }, - UploadId: input.uploadId, - }), - ); - }, - createMultipartUpload: async (rawInput) => { - const input = normalizeCreateMultipartInput(rawInput); - const result = await client.send( - new CreateMultipartUploadCommand({ - Bucket: bucket, - ChecksumAlgorithm: "SHA256", - ...(input.contentType ? { ContentType: input.contentType } : {}), - Key: input.key, - Metadata: input.metadata, - }), - ); - const uploadId = readString(result, "UploadId"); - if (!uploadId) { - throw new Error("Object storage create multipart did not return an upload id"); - } - return { key: input.key, uploadId }; - }, - ensureIncompleteMultipartUploadLifecycle: async ({ daysAfterInitiation }) => { - if ( - !Number.isSafeInteger(daysAfterInitiation) || - daysAfterInitiation < 1 || - daysAfterInitiation > 365 - ) { - throw new Error( - "Object storage incomplete multipart lifecycle daysAfterInitiation must be between 1 and 365", - ); - } - let rules: LifecycleRule[] = []; - try { - const current = await client.send( - new GetBucketLifecycleConfigurationCommand({ Bucket: bucket }), - ); - rules = lifecycleRules(current).filter( - (rule) => rule.ID !== incompleteMultipartLifecycleRuleId, - ); - } catch (error) { - if (!isMissingLifecycleConfigurationError(error)) throw error; - } - rules.push({ - AbortIncompleteMultipartUpload: { DaysAfterInitiation: daysAfterInitiation }, - Filter: { Prefix: "" }, - ID: incompleteMultipartLifecycleRuleId, - Status: "Enabled", - }); - await client.send( - new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { Rules: rules }, - }), - ); - }, - presignMultipartPart: async (rawInput) => { - const input = normalizePresignMultipartPartInput(rawInput, maxDirectUploadBytes); - const command = new UploadPartCommand({ - Bucket: bucket, - ...(input.checksumSha256Base64 ? { ChecksumSHA256: input.checksumSha256Base64 } : {}), - ContentLength: input.contentLength, - Key: input.key, - PartNumber: input.partNumber, - UploadId: input.uploadId, - }); - return presignedUploadResponse({ - command, - expiresInSeconds: input.expiresInSeconds, - headers: input.checksumSha256Base64 - ? { "x-amz-checksum-sha256": input.checksumSha256Base64 } - : {}, - client, - now, - presign, - }); - }, - presignPutObject: async (rawInput) => { - const input = normalizePresignPutInput(rawInput, maxDirectUploadBytes); - const command = new PutObjectCommand({ - Bucket: bucket, - ...(input.checksumSha256Base64 ? { ChecksumSHA256: input.checksumSha256Base64 } : {}), - ContentLength: input.contentLength, - ...(input.contentType ? { ContentType: input.contentType } : {}), - Key: input.key, - Metadata: input.metadata, - }); - return presignedUploadResponse({ - command, - expiresInSeconds: input.expiresInSeconds, - headers: { - ...(input.contentType ? { "content-type": input.contentType } : {}), - ...(input.checksumSha256Base64 - ? { "x-amz-checksum-sha256": input.checksumSha256Base64 } - : {}), - }, - client, - now, - presign, - }); - }, - verifyObjectSha256: async (rawInput) => { - const key = directUploadKey(rawInput.key); - const checksumSha256Base64 = requiredDirectUploadString( - rawInput.checksumSha256Base64, - "checksum", - ); - const expectedSizeBytes = directUploadContentLength( - rawInput.expectedSizeBytes, - maxDirectUploadBytes, - ); - let result: unknown; - try { - result = await client.send( - new GetObjectCommand({ - Bucket: bucket, - Key: key, - }), - ); - } catch (error) { - if (isMissingObjectError(error)) return false; - throw error; - } - const stream = bodyToStream(getRecordField(result, "Body"), key, maxDirectUploadBytes); - const reader = stream.getReader(); - const contentLength = readNumber(result, "ContentLength"); - if (contentLength !== undefined && contentLength !== expectedSizeBytes) { - await reader.cancel().catch(() => undefined); - reader.releaseLock(); - return false; - } - - const hash = createHash("sha256"); - let sizeBytes = 0; - try { - while (true) { - const chunk = await reader.read(); - if (chunk.done) break; - sizeBytes += chunk.value.byteLength; - if (sizeBytes > expectedSizeBytes) { - await reader.cancel().catch(() => undefined); - return false; - } - hash.update(chunk.value); - } - } finally { - reader.releaseLock(); - } - return sizeBytes === expectedSizeBytes && hash.digest("base64") === checksumSha256Base64; - }, - }; -} - -async function defaultS3ObjectStoragePresigner({ - client, - command, - expiresInSeconds, -}: S3ObjectStoragePresignInput): Promise { - return getSignedUrl(client as S3Client, command as PutObjectCommand, { - expiresIn: expiresInSeconds, - }); -} - -async function presignedUploadResponse(input: { - readonly client: S3ObjectStorageClient; - readonly command: S3ObjectStoragePresignInput["command"]; - readonly expiresInSeconds: number; - readonly headers: Readonly>; - readonly now: () => number; - readonly presign: S3ObjectStoragePresigner; -}): Promise { - const timestamp = input.now(); - if (!Number.isSafeInteger(timestamp) || timestamp < 0) { - throw new Error("Object storage direct upload clock must return a non-negative integer"); - } - const url = await input.presign({ - client: input.client, - command: input.command, - expiresInSeconds: input.expiresInSeconds, - }); - validatePresignedUrl(url); - return { - expiresAt: timestamp + input.expiresInSeconds * 1_000, - headers: { ...input.headers }, - method: "PUT", - url, - }; -} - -function normalizePresignPutInput( - input: PresignPutObjectInput, - maxDirectUploadBytes: number, -): PresignPutObjectInput { - const contentLength = directUploadContentLength(input.contentLength, maxDirectUploadBytes); - const expiresInSeconds = directUploadExpiration(input.expiresInSeconds); - return { - ...(input.checksumSha256Base64 - ? { checksumSha256Base64: requiredDirectUploadString(input.checksumSha256Base64, "checksum") } - : {}), - contentLength, - ...(input.contentType - ? { contentType: requiredDirectUploadString(input.contentType, "contentType") } - : {}), - expiresInSeconds, - key: directUploadKey(input.key), - metadata: normalizeDirectUploadMetadata(input.metadata), - }; -} - -function normalizePresignMultipartPartInput( - input: PresignMultipartObjectPartInput, - maxDirectUploadBytes: number, -): PresignMultipartObjectPartInput { - const expiresInSeconds = directUploadExpiration(input.expiresInSeconds); - const contentLength = directUploadContentLength(input.contentLength, maxDirectUploadBytes); - if ( - !Number.isSafeInteger(input.partNumber) || - input.partNumber < 1 || - input.partNumber > 10_000 - ) { - throw new Error("Object storage direct upload partNumber must be between 1 and 10000"); - } - return { - ...(input.checksumSha256Base64 - ? { checksumSha256Base64: requiredDirectUploadString(input.checksumSha256Base64, "checksum") } - : {}), - contentLength, - expiresInSeconds, - key: directUploadKey(input.key), - partNumber: input.partNumber, - uploadId: requiredDirectUploadString(input.uploadId, "uploadId"), - }; -} - -function normalizeCreateMultipartInput( - input: CreateMultipartObjectUploadInput, -): CreateMultipartObjectUploadInput { - return { - ...(input.contentType - ? { contentType: requiredDirectUploadString(input.contentType, "contentType") } - : {}), - key: directUploadKey(input.key), - metadata: normalizeDirectUploadMetadata(input.metadata), - }; -} - -function normalizeCompleteMultipartInput( - input: CompleteMultipartObjectUploadInput, -): CompleteMultipartObjectUploadInput { - const identity = normalizeMultipartIdentity(input); - if (input.parts.length < 1 || input.parts.length > 10_000) { - throw new Error("Object storage multipart completion requires 1-10000 parts"); - } - let previousPartNumber = 0; - const parts = input.parts.map((part) => { - if ( - !Number.isSafeInteger(part.partNumber) || - part.partNumber < 1 || - part.partNumber > 10_000 || - part.partNumber <= previousPartNumber - ) { - throw new Error("Object storage multipart completion parts must be strictly increasing"); - } - previousPartNumber = part.partNumber; - return { - ...(part.checksumSha256Base64 - ? { - checksumSha256Base64: requiredDirectUploadString( - part.checksumSha256Base64, - "part checksum", - ), - } - : {}), - etag: requiredDirectUploadString(part.etag, "part etag"), - partNumber: part.partNumber, - }; - }); - return { - ...identity, - parts, - }; -} - -function normalizeMultipartIdentity(input: { - readonly key: string; - readonly uploadId: string; -}): { readonly key: string; readonly uploadId: string } { - return { - key: directUploadKey(input.key), - uploadId: requiredDirectUploadString(input.uploadId, "uploadId"), - }; -} - -function directUploadContentLength(value: number, maxDirectUploadBytes: number): number { - positiveSafeInteger(value, "contentLength"); - if (value > maxDirectUploadBytes) { - throw new Error(`Object direct upload exceeds maxDirectUploadBytes=${maxDirectUploadBytes}`); - } - return value; -} - -function directUploadExpiration(value: number): number { - if (!Number.isSafeInteger(value) || value < 1 || value > maxPresignTtlSeconds) { - throw new Error( - `Object storage direct upload expiresInSeconds must be between 1 and ${maxPresignTtlSeconds}`, - ); - } - return value; -} - -function directUploadKey(value: string): string { - const key = requiredDirectUploadString(value, "key"); - if (new TextEncoder().encode(key).byteLength > 1_024 || key.includes("\0")) { - throw new Error("Object storage direct upload key is invalid"); - } - return key; -} - -function normalizeDirectUploadMetadata( - input: Readonly> | undefined, -): Readonly> { - const metadata: Record = {}; - for (const [rawKey, rawValue] of Object.entries(input ?? {})) { - const key = requiredDirectUploadString(rawKey, "metadata key").toLowerCase(); - const value = requiredDirectUploadString(rawValue, `metadata ${key}`); - if (!/^[a-z0-9_-]+$/.test(key) || key.length > 64 || value.length > 1_024) { - throw new Error("Object storage direct upload metadata is invalid"); - } - metadata[key] = value; - } - return metadata; -} - -function requiredDirectUploadString(value: string, field: string): string { - const normalized = value.trim(); - if (!normalized) { - throw new Error(`Object storage direct upload ${field} is required`); - } - return normalized; -} - -function positiveSafeInteger(value: number, field: string): void { - if (!Number.isSafeInteger(value) || value < 1) { - throw new Error(`Object storage direct upload ${field} must be a positive safe integer`); - } -} - -function validatePresignedUrl(value: string): void { - let parsed: URL; - try { - parsed = new URL(value); - } catch { - throw new Error("Object storage presigner returned an invalid URL"); - } - if ( - (parsed.protocol !== "https:" && parsed.protocol !== "http:") || - parsed.username || - parsed.password - ) { - throw new Error("Object storage presigner returned an invalid URL"); - } -} - -function isMissingMultipartUploadError(error: unknown): boolean { - if (!(error instanceof Error)) return false; - if (error.name === "NoSuchUpload") return true; - const metadata = getRecordField(error, "$metadata"); - return readNumber(metadata, "httpStatusCode") === 404; -} - -function isMissingLifecycleConfigurationError(error: unknown): boolean { - const record = asRecord(error); - return ( - record?.name === "NoSuchLifecycleConfiguration" || - record?.Code === "NoSuchLifecycleConfiguration" - ); -} - -function lifecycleRules(value: unknown): LifecycleRule[] { - const rules = getRecordField(value, "Rules"); - return Array.isArray(rules) - ? rules.filter((rule): rule is LifecycleRule => asRecord(rule) !== undefined) - : []; -} - -function listObjects( - objects: ReadonlyMap, - { cursor, limit, prefix }: ListObjectsInput, -): ListObjectsResult { - if (limit < 1) { - throw new Error("Object list limit must be at least 1"); - } - - const keys = [...objects.keys()] - .filter((key) => key.startsWith(prefix)) - .filter((key) => (cursor ? key > cursor : true)) - .sort() - .slice(0, limit + 1); - const pageKeys = keys.slice(0, limit); - const pageObjects = pageKeys - .map((key) => objects.get(key)?.metadata) - .filter(isObjectMetadata) - .map(cloneObjectMetadata); - - const nextCursor = keys.length > limit ? pageKeys.at(-1) : undefined; - - return { - objects: pageObjects, - ...(nextCursor ? { nextCursor } : {}), - }; -} - -function copyBytes(bytes: Uint8Array): Uint8Array { - return new Uint8Array(bytes); -} - -function bytesToStream(bytes: Uint8Array): ReadableStream { - return new ReadableStream({ - start(controller) { - controller.enqueue(bytes); - controller.close(); - }, - }); -} - -function cloneMetadata( - metadata: Readonly> | undefined, -): Readonly> { - return { ...(metadata ?? {}) }; -} - -function cloneObjectMetadata(metadata: ObjectMetadata): ObjectMetadata { - return { - ...(metadata.checksumSha256Base64 - ? { checksumSha256Base64: metadata.checksumSha256Base64 } - : {}), - key: metadata.key, - metadata: cloneMetadata(metadata.metadata), - sizeBytes: metadata.sizeBytes, - ...(metadata.contentType ? { contentType: metadata.contentType } : {}), - }; -} - -function isObjectMetadata(value: ObjectMetadata | undefined): value is ObjectMetadata { - return Boolean(value); -} - -async function bodyToBytes( - body: unknown, - key: string, - maxObjectBytes: number, -): Promise { - if (body instanceof Uint8Array) { - assertObjectSize(key, body.byteLength, maxObjectBytes); - return body; - } - - if (typeof body === "string") { - const bytes = new TextEncoder().encode(body); - assertObjectSize(key, bytes.byteLength, maxObjectBytes); - - return bytes; - } - - const transformToByteArray = getFunctionField(body, "transformToByteArray"); - - if (transformToByteArray) { - const bytes = await transformToByteArray.call(body); - assertObjectSize(key, bytes.byteLength, maxObjectBytes); - - return bytes; - } - - if (isAsyncIterable(body)) { - const chunks: Uint8Array[] = []; - let totalBytes = 0; - - for await (const chunk of body) { - const bytes = chunk instanceof Uint8Array ? chunk : new TextEncoder().encode(String(chunk)); - totalBytes += bytes.byteLength; - assertObjectSize(key, totalBytes, maxObjectBytes); - chunks.push(bytes); - } - - return concatenateBytes(chunks); - } - - throw new Error("Unsupported S3 object body type"); -} - -function bodyToStream( - body: unknown, - key: string, - maxObjectBytes: number, -): ReadableStream { - if (body === undefined || body === null) { - return bytesToStream(new Uint8Array()); - } - - if (body instanceof ReadableStream) { - return limitReadableStream(body, key, maxObjectBytes); - } - - if (body instanceof Uint8Array) { - assertObjectSize(key, body.byteLength, maxObjectBytes); - return bytesToStream(copyBytes(body)); - } - - if (typeof body === "string") { - const bytes = new TextEncoder().encode(body); - assertObjectSize(key, bytes.byteLength, maxObjectBytes); - return bytesToStream(bytes); - } - - if (isAsyncIterable(body)) { - return asyncIterableToReadableStream(body, key, maxObjectBytes); - } - - throw new Error("Unsupported S3 object body type"); -} - -function asyncIterableToReadableStream( - iterable: AsyncIterable, - key: string, - maxObjectBytes: number, -): ReadableStream { - const iterator = iterable[Symbol.asyncIterator](); - let iteratorClosed = false; - let totalBytes = 0; - - const closeIterator = async (): Promise => { - if (iteratorClosed) return; - iteratorClosed = true; - await iterator.return?.(); - }; - - return new ReadableStream({ - async pull(controller) { - try { - const next = await iterator.next(); - - if (next.done) { - await closeIterator(); - controller.close(); - return; - } - - const chunk = - next.value instanceof Uint8Array - ? copyBytes(next.value) - : new TextEncoder().encode(String(next.value)); - totalBytes += chunk.byteLength; - assertObjectSize(key, totalBytes, maxObjectBytes); - controller.enqueue(chunk); - } catch (error) { - await closeIterator().catch(() => undefined); - throw error; - } - }, - async cancel() { - await closeIterator(); - }, - }); -} - -function limitReadableStream( - stream: ReadableStream, - key: string, - maxObjectBytes: number, -): ReadableStream { - const reader = stream.getReader(); - let readerClosed = false; - let totalBytes = 0; - - const releaseReader = (): void => { - if (readerClosed) return; - readerClosed = true; - reader.releaseLock(); - }; - - const cancelReader = async (reason?: unknown): Promise => { - if (readerClosed) return; - readerClosed = true; - try { - await reader.cancel(reason); - } finally { - reader.releaseLock(); - } - }; - - return new ReadableStream({ - async pull(controller) { - try { - const result = await reader.read(); - - if (result.done) { - releaseReader(); - controller.close(); - return; - } - - totalBytes += result.value.byteLength; - assertObjectSize(key, totalBytes, maxObjectBytes); - controller.enqueue(copyBytes(result.value)); - } catch (error) { - await cancelReader(error).catch(() => undefined); - throw error; - } - }, - async cancel(reason) { - await cancelReader(reason); - }, - }); -} - -function assertObjectSize(key: string, sizeBytes: number, maxObjectBytes: number): void { - if (sizeBytes > maxObjectBytes) { - throw new Error(`Object ${key} exceeds maxObjectBytes=${maxObjectBytes}`); - } -} - -async function assertObjectMetadataSize( - body: unknown, - key: string, - sizeBytes: number | undefined, - maxObjectBytes: number, -): Promise { - if (sizeBytes === undefined) return; - try { - assertObjectSize(key, sizeBytes, maxObjectBytes); - } catch (error) { - await closeObjectBodyBestEffort(body, error); - throw error; - } -} - -async function closeObjectBodyBestEffort(body: unknown, reason: unknown): Promise { - try { - if (body instanceof ReadableStream) { - await body.cancel(reason); - return; - } - if (isAsyncIterable(body)) { - await body[Symbol.asyncIterator]().return?.(); - } - } catch { - return; - } -} - -function concatenateBytes(chunks: readonly Uint8Array[]): Uint8Array { - const totalBytes = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0); - const bytes = new Uint8Array(totalBytes); - let offset = 0; - - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.byteLength; - } - - return bytes; -} - -function isAsyncIterable(value: unknown): value is AsyncIterable { - return typeof value === "object" && value !== null && Symbol.asyncIterator in value; -} - -function isMissingObjectError(error: unknown): boolean { - const record = asRecord(error); - const metadata = asRecord(record?.$metadata); - - return ( - record?.name === "NoSuchKey" || - record?.name === "NotFound" || - record?.Code === "NoSuchKey" || - metadata?.httpStatusCode === 404 - ); -} - -function readContents( - value: unknown, -): readonly { readonly key: string; readonly sizeBytes: number }[] { - const contents = getRecordField(value, "Contents"); - - if (!Array.isArray(contents)) { - return []; - } - - return contents - .map((item) => { - const key = readString(item, "Key"); - - if (!key) { - return null; - } - - return { - key, - sizeBytes: readNumber(item, "Size") ?? 0, - }; - }) - .filter(isListedObject); -} - -function readMetadata(value: unknown): Readonly> { - const metadata = getRecordField(value, "Metadata"); - - if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) { - return {}; - } - - return Object.fromEntries( - Object.entries(metadata).filter( - (entry): entry is [string, string] => typeof entry[1] === "string", - ), - ); -} - -function readNumber(value: unknown, key: string): number | undefined { - const field = getRecordField(value, key); - - return typeof field === "number" ? field : undefined; -} - -function readString(value: unknown, key: string): string | undefined { - const field = getRecordField(value, key); - - return typeof field === "string" ? field : undefined; -} - -function getFunctionField( - value: unknown, - key: string, -): ((this: unknown) => Promise) | undefined { - const field = getRecordField(value, key); - - return typeof field === "function" - ? (field as (this: unknown) => Promise) - : undefined; -} - -function getRecordField(value: unknown, key: string): unknown { - return asRecord(value)?.[key]; -} - -function asRecord(value: unknown): Record | undefined { - return typeof value === "object" && value !== null - ? (value as Record) - : undefined; -} - -function isListedObject( - value: { readonly key: string; readonly sizeBytes: number } | null, -): value is { readonly key: string; readonly sizeBytes: number } { - return value !== null; -} diff --git a/knowledge-fs/packages/api/src/core-resource-response-schemas.ts b/knowledge-fs/packages/api/src/core-resource-response-schemas.ts index b0dcce41074..83ee51edd17 100644 --- a/knowledge-fs/packages/api/src/core-resource-response-schemas.ts +++ b/knowledge-fs/packages/api/src/core-resource-response-schemas.ts @@ -406,7 +406,7 @@ export const KnowledgeSpaceStatusResponseSchema = z manifestVersion: z.number().int().positive(), metadataDialect: z.enum(["portable", "postgres", "tidb"]), objectKeyPrefix: z.string(), - storageProvider: z.enum(["memory-dev", "r2", "s3-compatible"]), + storageProvider: z.enum(["dify", "memory-dev", "r2", "s3-compatible"]), }), parser: z.object({ kind: z.enum(["native-html", "native-markdown", "native-structured", "unstructured"]), @@ -414,8 +414,8 @@ export const KnowledgeSpaceStatusResponseSchema = z }), storage: z.object({ healthy: z.boolean(), - objectStorageKind: z.enum(["r2", "s3-compatible", "local", "memory"]), - provider: z.enum(["memory-dev", "r2", "s3-compatible"]), + objectStorageKind: z.enum(["dify", "local", "memory"]), + provider: z.enum(["dify", "memory-dev", "r2", "s3-compatible"]), }), tenantId: z.string(), }) diff --git a/knowledge-fs/packages/api/src/document-compilation-worker.ts b/knowledge-fs/packages/api/src/document-compilation-worker.ts index 86a2c003f52..9316b8eacab 100644 --- a/knowledge-fs/packages/api/src/document-compilation-worker.ts +++ b/knowledge-fs/packages/api/src/document-compilation-worker.ts @@ -71,7 +71,7 @@ export interface DocumentCompilationWorkerOptions { readonly indexOverrides?: DocumentCompilationIndexOverrideResolver | undefined; /** * Durable runners own retry/terminal transitions and must keep transient failures out of the - * asset and legacy job records. The default preserves the existing standalone worker contract. + * asset and legacy job records. The default preserves the existing legacy worker contract. */ readonly failureManagement?: "caller" | "worker" | undefined; readonly generateKnowledgePathId?: (() => string) | undefined; diff --git a/knowledge-fs/packages/api/src/gateway-document-write.test.ts b/knowledge-fs/packages/api/src/gateway-document-write.test.ts index 3baa090141f..c2bad28fb1b 100644 --- a/knowledge-fs/packages/api/src/gateway-document-write.test.ts +++ b/knowledge-fs/packages/api/src/gateway-document-write.test.ts @@ -2448,17 +2448,23 @@ describe("document write gateway integration", () => { }, ); expect(quotaBulkResponse.status).toBe(202); - await expect(quotaBulkResponse.json()).resolves.toMatchObject({ + const quotaBulkPayload = (await quotaBulkResponse.json()) as { + items: readonly { + asset?: { readonly objectKey?: string }; + readonly reason?: string; + readonly status?: string; + }[]; + }; + expect(quotaBulkPayload).toMatchObject({ accepted: 1, excluded: 1, items: [{ status: "accepted" }, { reason: "quota_exceeded", status: "excluded" }], }); + const acceptedObjectKey = quotaBulkPayload.items[0]?.asset?.objectKey; + expect(acceptedObjectKey).toEqual(expect.any(String)); await expect( - quotaBulkAdapter.objectStorage.listObjects({ - limit: 10, - prefix: "tenant-1/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/", - }), - ).resolves.toMatchObject({ objects: [expect.objectContaining({ key: expect.any(String) })] }); + quotaBulkAdapter.objectStorage.headObject(acceptedObjectKey ?? ""), + ).resolves.toMatchObject({ key: acceptedObjectKey }); const manifestQuotaAdapter = createNodePlatformAdapter({ env: {} }); const manifestQuotaManifests = createInMemoryKnowledgeSpaceManifestRepository({ @@ -2500,6 +2506,13 @@ describe("document write gateway integration", () => { }, tenantId: "tenant-1", }); + const manifestQuotaObjectPrefix = + "tenant-1/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/"; + const objectsBeforeManifestQuotaRejection = + await manifestQuotaAdapter.objectStorage.listObjects({ + limit: 10, + prefix: manifestQuotaObjectPrefix, + }); const manifestQuotaForm = new FormData(); manifestQuotaForm.append("file", new File([new Uint8Array(3)], "too-large.md")); const manifestQuotaResponse = await manifestQuotaApp.request( @@ -2517,9 +2530,9 @@ describe("document write gateway integration", () => { await expect( manifestQuotaAdapter.objectStorage.listObjects({ limit: 10, - prefix: "tenant-1/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42/documents/", + prefix: manifestQuotaObjectPrefix, }), - ).resolves.toEqual({ objects: [] }); + ).resolves.toEqual(objectsBeforeManifestQuotaRejection); const noDurableJobsApp = createKnowledgeGateway({ adapter: createNodePlatformAdapter({ env: {} }), diff --git a/knowledge-fs/packages/api/src/knowledge-space-control-plane-diagnostics.test.ts b/knowledge-fs/packages/api/src/knowledge-space-control-plane-diagnostics.test.ts index 8bfc26884f2..7d2665ada27 100644 --- a/knowledge-fs/packages/api/src/knowledge-space-control-plane-diagnostics.test.ts +++ b/knowledge-fs/packages/api/src/knowledge-space-control-plane-diagnostics.test.ts @@ -384,7 +384,7 @@ describe("KnowledgeSpace control-plane diagnostics", () => { consistencyClass: "path-consistent", manifestVersion: 1, objectKeyPrefix: `tenant-1/spaces/${SPACE_ID}`, - storageProvider: "memory-dev", + storageProvider: "dify", }, parser: { kind: "native-markdown", @@ -392,8 +392,8 @@ describe("KnowledgeSpace control-plane diagnostics", () => { }, storage: { healthy: true, - objectStorageKind: "memory", - provider: "memory-dev", + objectStorageKind: "dify", + provider: "dify", }, tenantId: "tenant-1", }); @@ -677,7 +677,7 @@ describe("KnowledgeSpace control-plane diagnostics", () => { await expect(response.json()).resolves.toMatchObject({ storage: { healthy: false, - objectStorageKind: "memory", + objectStorageKind: "dify", }, }); }); diff --git a/knowledge-fs/packages/api/src/knowledge-space-manifest-bootstrap.test.ts b/knowledge-fs/packages/api/src/knowledge-space-manifest-bootstrap.test.ts index dd57e861c41..d96fa57248a 100644 --- a/knowledge-fs/packages/api/src/knowledge-space-manifest-bootstrap.test.ts +++ b/knowledge-fs/packages/api/src/knowledge-space-manifest-bootstrap.test.ts @@ -95,7 +95,7 @@ describe("KnowledgeSpace manifest bootstrap", () => { id: "018f0d60-7a49-7cc2-9c1b-5b36f18f7b10", manifestVersion: 1, objectKeyPrefix: "tenant-1/spaces/018f0d60-7a49-7cc2-9c1b-5b36f18f2c42", - storageProvider: "memory-dev", + storageProvider: "dify", }); }); diff --git a/knowledge-fs/packages/api/src/upload-session.test.ts b/knowledge-fs/packages/api/src/upload-session.test.ts index 5dccce04ba2..2c529ff99a7 100644 --- a/knowledge-fs/packages/api/src/upload-session.test.ts +++ b/knowledge-fs/packages/api/src/upload-session.test.ts @@ -845,7 +845,7 @@ function fakeStorage(input: { readonly deleteObject: ReturnType; } { return { - kind: input.direct ? "s3-compatible" : "memory", + kind: "memory", ...(input.direct ? { directUpload: input.direct } : {}), deleteObject: vi.fn(async () => undefined), getObject: vi.fn(async () => null), diff --git a/knowledge-fs/packages/api/vitest.config.ts b/knowledge-fs/packages/api/vitest.config.ts index a7bc5c1e51e..9b771d70b88 100644 --- a/knowledge-fs/packages/api/vitest.config.ts +++ b/knowledge-fs/packages/api/vitest.config.ts @@ -1,7 +1,10 @@ +import { resolve } from "node:path"; + import { defineConfig } from "vitest/config"; export default defineConfig({ test: { + setupFiles: [resolve(import.meta.dirname, "../../test/setup-dify-object-storage.ts")], coverage: { exclude: ["src/**/*.test.ts"], include: ["src/**/*.ts"], diff --git a/knowledge-fs/packages/core/src/models.test.ts b/knowledge-fs/packages/core/src/models.test.ts index 6b6b2800608..78637a62c1a 100644 --- a/knowledge-fs/packages/core/src/models.test.ts +++ b/knowledge-fs/packages/core/src/models.test.ts @@ -688,7 +688,7 @@ describe("core domain models", () => { failedCommitRetentionDays: 14, traceRetentionDays: 30, }, - storageProvider: "memory-dev", + storageProvider: "dify", tenantId: "tenant-1", updatedAt, }); diff --git a/knowledge-fs/packages/core/src/models.ts b/knowledge-fs/packages/core/src/models.ts index 8525cfa3b0e..7946f82abd6 100644 --- a/knowledge-fs/packages/core/src/models.ts +++ b/knowledge-fs/packages/core/src/models.ts @@ -51,7 +51,14 @@ export const KnowledgeSpaceSchema = z.object({ }); export type KnowledgeSpace = z.infer; -export const KnowledgeSpaceStorageProviderSchema = z.enum(["memory-dev", "r2", "s3-compatible"]); +// Non-Dify values remain readable only for legacy manifests during coexistence. New manifests +// always use `dify`, and the production adapter has no direct provider implementation. +export const KnowledgeSpaceStorageProviderSchema = z.enum([ + "dify", + "memory-dev", + "r2", + "s3-compatible", +]); export type KnowledgeSpaceStorageProvider = z.infer; export const KnowledgeSpaceMetadataDialectSchema = z.enum(["portable", "postgres", "tidb"]); @@ -736,7 +743,7 @@ export function createDefaultKnowledgeSpaceManifest( traceRetentionDays: 30, }, ...(input.retrievalProfile ? { retrievalProfile: input.retrievalProfile } : {}), - storageProvider: "memory-dev", + storageProvider: "dify", tenantId: input.tenantId, updatedAt: input.updatedAt, }); diff --git a/knowledge-fs/packages/core/src/platform-adapter.test.ts b/knowledge-fs/packages/core/src/platform-adapter.test.ts index 674d0ff381b..affcb2173a2 100644 --- a/knowledge-fs/packages/core/src/platform-adapter.test.ts +++ b/knowledge-fs/packages/core/src/platform-adapter.test.ts @@ -59,7 +59,7 @@ describe("collectPlatformHealth", () => { callback({ execute: async () => ({ rows: [], rowsAffected: 0 }) }), }, objectStorage: { - kind: "s3-compatible", + kind: "memory", deleteObject: async () => undefined, getObject: async () => null, getObjectStream: async () => null, @@ -267,7 +267,7 @@ async function createHealthyPlatformAdapter(): Promise { callback({ execute: async () => ({ rows: [], rowsAffected: 0 }) }), }, objectStorage: { - kind: "s3-compatible", + kind: "memory", deleteObject: async () => undefined, getObject: async () => null, getObjectStream: async () => null, diff --git a/knowledge-fs/packages/core/src/platform-adapter.ts b/knowledge-fs/packages/core/src/platform-adapter.ts index fab595bd3df..53f39a0892c 100644 --- a/knowledge-fs/packages/core/src/platform-adapter.ts +++ b/knowledge-fs/packages/core/src/platform-adapter.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -export const RuntimeTargetSchema = z.enum(["cloudflare-workers", "node-docker"]); +export const RuntimeTargetSchema = z.enum(["node-docker"]); export type RuntimeTarget = z.infer; export const HealthStatusSchema = z.object({ @@ -242,7 +242,7 @@ export interface ObjectStorageDirectUploadAdapter { } export interface ObjectStorageAdapter { - readonly kind: "r2" | "s3-compatible" | "local" | "memory"; + readonly kind: "dify" | "local" | "memory"; close?(): Promise; deleteObject(key: string): Promise; readonly directUpload?: ObjectStorageDirectUploadAdapter; diff --git a/knowledge-fs/packages/plugin-daemon-client/package.json b/knowledge-fs/packages/plugin-daemon-client/package.json deleted file mode 100644 index 12a7ff5c22d..00000000000 --- a/knowledge-fs/packages/plugin-daemon-client/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "@knowledge/plugin-daemon-client", - "private": true, - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "build": "tsc --noEmit", - "test": "vitest run", - "test:coverage": "vitest run --coverage", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "zod": "^3.24.1" - }, - "devDependencies": { - "@types/node": "^22.10.2", - "typescript": "^5.7.2", - "vitest": "^2.1.8" - } -} diff --git a/knowledge-fs/packages/plugin-daemon-client/src/index.test.ts b/knowledge-fs/packages/plugin-daemon-client/src/index.test.ts deleted file mode 100644 index 78a46c415c6..00000000000 --- a/knowledge-fs/packages/plugin-daemon-client/src/index.test.ts +++ /dev/null @@ -1,468 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { - type PluginDaemonClient, - type PluginDaemonDatasourceInput, - PluginDaemonError, - createPluginDaemonClient, -} from "./index"; - -function sseStreamResponse(chunks: readonly string[], init: ResponseInit = {}): Response { - const encoder = new TextEncoder(); - const stream = new ReadableStream({ - start(controller) { - for (const chunk of chunks) { - controller.enqueue(encoder.encode(chunk)); - } - controller.close(); - }, - }); - - return new Response(stream, { - headers: { "content-type": "text/event-stream" }, - status: 200, - ...init, - }); -} - -function envelopeChunk(envelope: Record): string { - return `data: ${JSON.stringify(envelope)}\n\n`; -} - -async function collect( - client: PluginDaemonClient, - input: PluginDaemonDatasourceInput = BASE_INPUT, -): Promise { - const values: unknown[] = []; - for await (const value of client.dispatchDatasourceStream(input)) { - values.push(value); - } - return values; -} - -const OPTIONS = { - apiKey: "plugin-api-key", - baseUrl: "http://plugin-daemon:5002/", -} as const; - -const BASE_INPUT = { - data: { credentials: {}, provider: "firecrawl" }, - method: "get_website_crawl", - pluginId: "langgenius/firecrawl_datasource", - tenantId: "tenant-abc", -} as const satisfies PluginDaemonDatasourceInput; - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("plugin-daemon datasource client", () => { - it("uses the datasource-only path, headers, body, and streams every envelope", async () => { - const calls: { init: RequestInit; url: string }[] = []; - const fetchImpl = vi.fn(async (url: string, init: RequestInit) => { - calls.push({ init, url }); - return sseStreamResponse([ - envelopeChunk({ code: 0, data: { source_url: "https://a" }, message: "" }), - envelopeChunk({ code: 0, data: { source_url: "https://b" }, message: "" }), - ]); - }) as unknown as typeof fetch; - const client = createPluginDaemonClient({ ...OPTIONS, fetch: fetchImpl }); - - await expect(collect(client, { ...BASE_INPUT, userId: "user-1" })).resolves.toEqual([ - { source_url: "https://a" }, - { source_url: "https://b" }, - ]); - expect(calls[0]?.url).toBe( - "http://plugin-daemon:5002/plugin/tenant-abc/dispatch/datasource/get_website_crawl", - ); - const headers = calls[0]?.init.headers as Record; - expect(headers["x-api-key"]).toBe("plugin-api-key"); - expect(headers["x-plugin-id"]).toBe("langgenius/firecrawl_datasource"); - expect(JSON.parse(String(calls[0]?.init.body))).toEqual({ - data: BASE_INPUT.data, - user_id: "user-1", - }); - expect(calls[0]?.url).not.toContain("/invoke"); - }); - - it("accepts bare envelope lines", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse(['{"code":0,"message":"","data":{"value":42}}\n']), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).resolves.toEqual([{ value: 42 }]); - }); - - it("buffers an envelope split across chunks", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse(['data: {"code":0,"message":"",', '"data":{"value":42}}\n\n']), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).resolves.toEqual([{ value: 42 }]); - }); - - it("parses a final event without a trailing newline", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse(['data: {"code":0,"message":"","data":{"tail":1}}']), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).resolves.toEqual([{ tail: 1 }]); - }); - - it.each([null, undefined])("rejects an empty success payload: %s", async (data) => { - const envelope = data === undefined ? { code: 0, message: "" } : { code: 0, data, message: "" }; - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse([envelopeChunk(envelope)]), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).rejects.toMatchObject({ - code: "plugin_daemon_response_invalid", - name: "PluginDaemonError", - }); - }); - - it("unwraps nested daemon errors and redacts datasource credentials", async () => { - const secret = "datasource-secret/?="; - const shared = { token: secret }; - const message = JSON.stringify({ - error_type: "PluginInvokeError", - message: JSON.stringify({ - error_type: `CredentialsValidateFailedError:${secret}`, - message: `credential ${encodeURIComponent(secret)} is invalid`, - }), - }); - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse([envelopeChunk({ code: -500, data: null, message })]), - ) as unknown as typeof fetch, - }); - const error = await collect(client, { - ...BASE_INPUT, - data: { credentials: { repeated: shared, token: secret, values: [shared] } }, - }).catch((cause: unknown) => cause); - - expect(error).toBeInstanceOf(PluginDaemonError); - expect(error).toMatchObject({ - code: "plugin_daemon_invoke", - daemonCode: -500, - errorType: "CredentialsValidateFailedError:[REDACTED]", - }); - expect((error as Error).message).toContain("[REDACTED]"); - expect(JSON.stringify(error)).not.toContain(secret); - expect((error as Error).message).not.toContain(encodeURIComponent(secret)); - }); - - it("uses the daemon code when an error has no message", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse([envelopeChunk({ code: 7, data: null })]), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).rejects.toMatchObject({ - code: "plugin_daemon_invoke", - daemonCode: 7, - message: "Plugin daemon error code 7", - }); - }); - - it("passes through a plain daemon error message", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse([envelopeChunk({ code: 7, data: null, message: "plain failure text" })]), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).rejects.toMatchObject({ - daemonCode: 7, - message: "plain failure text", - }); - }); - - it("enforces a deadline when fetch ignores AbortSignal", async () => { - vi.useFakeTimers(); - let observedSignal: AbortSignal | null = null; - const client = createPluginDaemonClient({ - ...OPTIONS, - dispatchRequestTimeoutMs: 25, - fetch: vi.fn((_url: string, init: RequestInit) => { - observedSignal = init.signal as AbortSignal; - return new Promise(() => undefined); - }) as unknown as typeof fetch, - }); - const request = collect(client); - const expectation = expect(request).rejects.toMatchObject({ code: "plugin_daemon_timeout" }); - - await vi.advanceTimersByTimeAsync(25); - await expectation; - expect((observedSignal as AbortSignal | null)?.aborted).toBe(true); - }); - - it("enforces a deadline when the response reader ignores abort", async () => { - vi.useFakeTimers(); - const response = new Response( - new ReadableStream({ - start() { - // Intentionally never enqueue or close. - }, - }), - { status: 200 }, - ); - const client = createPluginDaemonClient({ - ...OPTIONS, - dispatchRequestTimeoutMs: 25, - fetch: vi.fn(async () => response) as unknown as typeof fetch, - }); - const next = client.dispatchDatasourceStream(BASE_INPUT).next(); - const expectation = expect(next).rejects.toMatchObject({ code: "plugin_daemon_timeout" }); - - await vi.advanceTimersByTimeAsync(25); - await expectation; - }); - - it("classifies caller abort even when fetch ignores AbortSignal", async () => { - const controller = new AbortController(); - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(() => new Promise(() => undefined)) as unknown as typeof fetch, - }); - const request = collect(client, { ...BASE_INPUT, signal: controller.signal }); - - controller.abort(); - await expect(request).rejects.toMatchObject({ code: "plugin_daemon_aborted" }); - }); - - it("rejects a pre-aborted request without I/O", async () => { - const controller = new AbortController(); - controller.abort(); - const fetchImpl = vi.fn(async () => sseStreamResponse([])) as unknown as typeof fetch; - const client = createPluginDaemonClient({ ...OPTIONS, fetch: fetchImpl }); - - await expect( - collect(client, { ...BASE_INPUT, signal: controller.signal }), - ).rejects.toMatchObject({ code: "plugin_daemon_aborted" }); - expect(fetchImpl).not.toHaveBeenCalled(); - }); - - it("normalizes transport errors without echoing credentials", async () => { - const secret = "transport-secret-must-not-leak"; - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => { - throw new Error(`adapter echoed ${secret}`); - }) as unknown as typeof fetch, - }); - const error = await collect(client, { - ...BASE_INPUT, - data: { credentials: { api_key: secret } }, - }).catch((cause: unknown) => cause); - - expect(error).toMatchObject({ - code: "plugin_daemon_request_failed", - message: "Plugin daemon request failed", - }); - expect((error as Error).message).not.toContain(secret); - }); - - it.each([ - { code: "plugin_daemon_rate_limited", status: 429 }, - { code: "plugin_daemon_request_failed", status: 503 }, - ])("maps HTTP $status to $code", async ({ code, status }) => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => new Response("", { status })) as unknown as typeof fetch, - }); - - await expect(collect(client)).rejects.toMatchObject({ code, status }); - }); - - it("retries retryable responses and cancels their bodies", async () => { - let attempt = 0; - let cancelled = false; - const retryBody = new ReadableStream({ - cancel() { - cancelled = true; - }, - }); - const sleep = vi.fn(async () => undefined); - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => { - attempt += 1; - return attempt === 1 - ? new Response(retryBody, { status: 503 }) - : sseStreamResponse([envelopeChunk({ code: 0, data: { ok: true }, message: "" })]); - }) as unknown as typeof fetch, - maxRetries: 1, - retryDelayMs: 1, - sleep, - }); - - await expect(collect(client)).resolves.toEqual([{ ok: true }]); - expect(attempt).toBe(2); - expect(cancelled).toBe(true); - expect(sleep).toHaveBeenCalledWith(1); - }); - - it("supports a zero retry delay with the default sleeper", async () => { - let attempt = 0; - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => { - attempt += 1; - return attempt === 1 - ? new Response("", { status: 408 }) - : sseStreamResponse([envelopeChunk({ code: 0, data: { ok: true }, message: "" })]); - }) as unknown as typeof fetch, - maxRetries: 1, - retryDelayMs: 0, - }); - - await expect(collect(client)).resolves.toEqual([{ ok: true }]); - }); - - it("enforces the streaming response byte cap", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse([envelopeChunk({ code: 0, data: { big: "x".repeat(64) }, message: "" })]), - ) as unknown as typeof fetch, - maxResponseBytes: 8, - }); - - await expect(collect(client)).rejects.toMatchObject({ - code: "plugin_daemon_response_invalid", - }); - }); - - it("reads bodyless responses and enforces declared and actual byte limits", async () => { - const bodyless = (body: string, contentLength?: string): Response => - ({ - body: null, - headers: new Headers(contentLength ? { "content-length": contentLength } : {}), - ok: true, - status: 200, - text: async () => body, - }) as unknown as Response; - const success = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - bodyless(envelopeChunk({ code: 0, data: { ok: true }, message: "" })), - ) as unknown as typeof fetch, - }); - await expect(collect(success)).resolves.toEqual([{ ok: true }]); - - const declared = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => bodyless("{}", "1024")) as unknown as typeof fetch, - maxResponseBytes: 16, - }); - await expect(collect(declared)).rejects.toThrow("maxResponseBytes=16"); - - const actual = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => bodyless("x".repeat(32))) as unknown as typeof fetch, - maxResponseBytes: 16, - }); - await expect(collect(actual)).rejects.toThrow("maxResponseBytes=16"); - }); - - it("ignores keep-alive and blank lines", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => - sseStreamResponse(["\n\n", "data: \n", " \n"]), - ) as unknown as typeof fetch, - }); - - await expect(collect(client)).resolves.toEqual([]); - }); - - it.each([ - { body: "data: not-json\n", label: "invalid JSON" }, - { body: envelopeChunk({ data: { unexpected: true } }), label: "invalid envelope" }, - ])("rejects $label", async ({ body }) => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => sseStreamResponse([body])) as unknown as typeof fetch, - }); - - await expect(collect(client)).rejects.toMatchObject({ - code: "plugin_daemon_response_invalid", - }); - }); - - it.each([ - { label: "array root", value: [] }, - { label: "non-finite number", value: { score: Number.NaN } }, - { label: "non-JSON object", value: { createdAt: new Date() } }, - ])("rejects $label dispatch data before I/O", async ({ value }) => { - const fetchImpl = vi.fn(async () => sseStreamResponse([])) as unknown as typeof fetch; - const client = createPluginDaemonClient({ ...OPTIONS, fetch: fetchImpl }); - - await expect( - collect(client, { ...BASE_INPUT, data: value as Record }), - ).rejects.toMatchObject({ code: "plugin_daemon_input" }); - expect(fetchImpl).not.toHaveBeenCalled(); - }); - - it("rejects circular and over-deep dispatch data", async () => { - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => sseStreamResponse([])) as unknown as typeof fetch, - }); - const circular: Record = {}; - circular.self = circular; - await expect(collect(client, { ...BASE_INPUT, data: circular })).rejects.toThrow( - "circular references", - ); - - const root: Record = {}; - let cursor = root; - for (let index = 0; index < 26; index += 1) { - const next: Record = {}; - cursor.next = next; - cursor = next; - } - await expect(collect(client, { ...BASE_INPUT, data: root })).rejects.toThrow( - "complexity limits", - ); - }); - - it("validates constructor bounds and dispatch identifiers", async () => { - expect(() => createPluginDaemonClient({ apiKey: "k", baseUrl: " " })).toThrow("baseUrl"); - expect(() => createPluginDaemonClient({ apiKey: " ", baseUrl: "http://x" })).toThrow("apiKey"); - expect(() => createPluginDaemonClient({ ...OPTIONS, maxResponseBytes: 0 })).toThrow( - "maxResponseBytes", - ); - expect(() => createPluginDaemonClient({ ...OPTIONS, maxRetries: -1 })).toThrow("maxRetries"); - expect(() => createPluginDaemonClient({ ...OPTIONS, dispatchRequestTimeoutMs: 0 })).toThrow( - "dispatchRequestTimeoutMs", - ); - expect(() => - createPluginDaemonClient({ ...OPTIONS, dispatchRequestTimeoutMs: 600_001 }), - ).toThrow("dispatchRequestTimeoutMs"); - - const client = createPluginDaemonClient({ - ...OPTIONS, - fetch: vi.fn(async () => sseStreamResponse([])) as unknown as typeof fetch, - }); - await expect(collect(client, { ...BASE_INPUT, tenantId: " " })).rejects.toThrow("tenantId"); - await expect(collect(client, { ...BASE_INPUT, pluginId: " " })).rejects.toThrow("pluginId"); - }); -}); diff --git a/knowledge-fs/packages/plugin-daemon-client/src/index.ts b/knowledge-fs/packages/plugin-daemon-client/src/index.ts deleted file mode 100644 index 994cd648594..00000000000 --- a/knowledge-fs/packages/plugin-daemon-client/src/index.ts +++ /dev/null @@ -1,724 +0,0 @@ -import { z } from "zod"; - -/** - * Transport client for plugin-daemon datasource dispatch only. - * - * Mirrors the contract Dify uses (api/core/plugin/impl/base.py + datasource.py): - * POST {baseUrl}/plugin/{tenant_id}/dispatch/datasource/{method} - * headers: X-Api-Key, X-Plugin-ID, Content-Type: application/json - * body: { user_id?, data: {...} } - * response: one JSON envelope `{"code":0,"message":"","data":{...}}` per non-empty line, - * with an optional `data:` SSE prefix (dify strips it when present); code != 0 is an - * error whose message may be a nested JSON {error_type, message} (PluginInvokeError - * wraps the real error), and a success envelope must carry non-empty data. - * - * Model invocation is deliberately absent. KnowledgeFS calls Dify's inner model API, and Dify's - * ModelManager resolves the tenant model instance and its plugin-daemon credentials. - */ - -export type PluginDaemonErrorCode = - | "plugin_daemon_aborted" - | "plugin_daemon_input" - | "plugin_daemon_invoke" - | "plugin_daemon_rate_limited" - | "plugin_daemon_request_failed" - | "plugin_daemon_response_invalid" - | "plugin_daemon_timeout"; - -export class PluginDaemonError extends Error { - readonly code: PluginDaemonErrorCode; - readonly daemonCode?: number; - readonly errorType?: string; - readonly status?: number; - - constructor( - message: string, - { - cause, - code, - daemonCode, - errorType, - status, - }: { - readonly cause?: unknown; - readonly code: PluginDaemonErrorCode; - readonly daemonCode?: number | undefined; - readonly errorType?: string | undefined; - readonly status?: number | undefined; - }, - ) { - super(message, cause === undefined ? undefined : { cause }); - this.name = "PluginDaemonError"; - this.code = code; - if (daemonCode !== undefined) { - this.daemonCode = daemonCode; - } - if (errorType !== undefined) { - this.errorType = errorType; - } - if (status !== undefined) { - this.status = status; - } - } -} - -export interface PluginDaemonClientOptions { - readonly apiKey: string; - readonly baseUrl: string; - /** Hard deadline for datasource dispatch, including streaming response iteration. */ - readonly dispatchRequestTimeoutMs?: number | undefined; - readonly fetch?: typeof fetch | undefined; - readonly maxResponseBytes?: number | undefined; - readonly maxRetries?: number | undefined; - readonly retryDelayMs?: number | undefined; - readonly sleep?: ((ms: number) => Promise) | undefined; -} - -/** - * Datasource dispatch methods (dify api/core/plugin/impl/datasource.py). Note the wire path is - * `dispatch/datasource/{method}` WITHOUT the `/invoke` suffix that model ops use. - */ -export type PluginDaemonDatasourceMethod = - | "get_online_document_page_content" - | "get_online_document_pages" - | "get_website_crawl" - | "online_drive_browse_files" - | "online_drive_download_file" - | "validate_credentials"; - -export interface PluginDaemonDatasourceInput { - readonly data: Record; - readonly method: PluginDaemonDatasourceMethod; - readonly pluginId: string; - readonly signal?: AbortSignal | undefined; - readonly tenantId: string; - readonly userId?: string | undefined; -} - -export interface PluginDaemonDatasourceClient { - /** Stream every envelope `data` payload from a datasource method dispatch. */ - dispatchDatasourceStream(input: PluginDaemonDatasourceInput): AsyncGenerator; -} - -export type PluginDaemonClient = PluginDaemonDatasourceClient; - -interface PluginDaemonRuntime { - readonly apiKey: string; - readonly baseUrl: string; - readonly dispatchRequestTimeoutMs: number; - readonly fetchImpl: typeof fetch; - readonly maxResponseBytes: number; - readonly maxRetries: number; - readonly retryDelayMs: number; - readonly sleep: (ms: number) => Promise; -} - -const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; -const DEFAULT_DISPATCH_REQUEST_TIMEOUT_MS = 60_000; -const MAX_REQUEST_TIMEOUT_MS = 10 * 60_000; -const DEFAULT_MAX_RETRIES = 0; -const DEFAULT_RETRY_DELAY_MS = 100; -const MAX_JSON_DEPTH = 24; -const MAX_JSON_NODES = 16_384; - -const PluginDaemonEnvelopeSchema = z.object({ - code: z.number(), - data: z.unknown().optional(), - message: z.string().optional(), -}); - -export function createPluginDaemonClient(options: PluginDaemonClientOptions): PluginDaemonClient { - const baseUrl = options.baseUrl.trim(); - - if (!baseUrl) { - throw new PluginDaemonError("Plugin daemon baseUrl is required", { - code: "plugin_daemon_input", - }); - } - - if (!options.apiKey.trim()) { - throw new PluginDaemonError("Plugin daemon apiKey is required", { - code: "plugin_daemon_input", - }); - } - - const maxResponseBytes = options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES; - const dispatchRequestTimeoutMs = - options.dispatchRequestTimeoutMs ?? DEFAULT_DISPATCH_REQUEST_TIMEOUT_MS; - const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES; - const retryDelayMs = options.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS; - - if (!Number.isInteger(maxResponseBytes) || maxResponseBytes < 1) { - throw new PluginDaemonError("Plugin daemon maxResponseBytes must be at least 1", { - code: "plugin_daemon_input", - }); - } - - validateRequestTimeout(dispatchRequestTimeoutMs, "dispatchRequestTimeoutMs"); - - if (!Number.isInteger(maxRetries) || maxRetries < 0) { - throw new PluginDaemonError("Plugin daemon maxRetries must be at least 0", { - code: "plugin_daemon_input", - }); - } - - const runtime: PluginDaemonRuntime = { - apiKey: options.apiKey, - baseUrl: baseUrl.replace(/\/+$/u, ""), - dispatchRequestTimeoutMs, - fetchImpl: options.fetch ?? fetch, - maxResponseBytes, - maxRetries, - retryDelayMs, - sleep: options.sleep ?? sleepMs, - }; - - async function* streamPath( - path: string, - input: { - readonly data: Record; - readonly pluginId: string; - readonly signal?: AbortSignal | undefined; - readonly userId?: string | undefined; - }, - ): AsyncGenerator { - const url = `${runtime.baseUrl}${path}`; - assertJsonRecord(input.data, "dispatch data"); - const redactions = dispatchCredentialRedactions(input.data); - const init: RequestInit = { - body: JSON.stringify({ - ...(input.userId ? { user_id: input.userId } : {}), - data: input.data, - }), - headers: { - accept: "text/event-stream", - "content-type": "application/json", - "x-api-key": runtime.apiKey, - "x-plugin-id": input.pluginId.trim(), - }, - method: "POST", - ...(input.signal ? { signal: input.signal } : {}), - }; - - const response = await fetchWithRetries(runtime, url, init); - - if (!response.ok) { - throw pluginDaemonRequestError(response.status); - } - - for await (const event of readSseEvents(response, runtime.maxResponseBytes)) { - yield unwrapEnvelope(event.data, redactions); - } - } - - function dispatchDatasource(input: PluginDaemonDatasourceInput): AsyncGenerator { - validateDispatchInput(input); - - // Datasource dispatch has NO `/invoke` suffix (dify contract). - return withDispatchDeadline(runtime, input.signal, (signal) => - streamPath( - `/plugin/${encodeURIComponent(input.tenantId.trim())}/dispatch/datasource/${input.method}`, - { - data: input.data, - pluginId: input.pluginId, - signal, - ...(input.userId ? { userId: input.userId } : {}), - }, - ), - ); - } - - return { - dispatchDatasourceStream: (input) => dispatchDatasource(input), - }; -} - -function assertJsonRecord(value: unknown, name: string): asserts value is Record { - if (!isPlainRecord(value)) { - throw new PluginDaemonError(`Plugin daemon ${name} must be a JSON object`, { - code: "plugin_daemon_input", - }); - } - - assertJsonValue(value, name); -} - -function assertJsonValue(value: unknown, name: string): void { - let nodes = 0; - const ancestors = new Set(); - - const visit = (current: unknown, depth: number): void => { - nodes += 1; - - if (nodes > MAX_JSON_NODES || depth > MAX_JSON_DEPTH) { - throw new PluginDaemonError(`Plugin daemon ${name} exceeds JSON complexity limits`, { - code: "plugin_daemon_input", - }); - } - - if (current === null || typeof current === "string" || typeof current === "boolean") { - return; - } - - if (typeof current === "number") { - if (Number.isFinite(current)) { - return; - } - - throw new PluginDaemonError(`Plugin daemon ${name} contains a non-finite number`, { - code: "plugin_daemon_input", - }); - } - - if (typeof current !== "object" || (!Array.isArray(current) && !isPlainRecord(current))) { - throw new PluginDaemonError(`Plugin daemon ${name} must contain only JSON values`, { - code: "plugin_daemon_input", - }); - } - - if (ancestors.has(current)) { - throw new PluginDaemonError(`Plugin daemon ${name} must not contain circular references`, { - code: "plugin_daemon_input", - }); - } - - ancestors.add(current); - - if (Array.isArray(current)) { - for (const item of current) { - visit(item, depth + 1); - } - } else { - for (const item of Object.values(current)) { - visit(item, depth + 1); - } - } - - ancestors.delete(current); - }; - - visit(value, 0); -} - -function isPlainRecord(value: unknown): value is Record { - if (value === null || typeof value !== "object" || Array.isArray(value)) { - return false; - } - - const prototype = Object.getPrototypeOf(value); - return prototype === Object.prototype || prototype === null; -} - -function collectCredentialRedactions(credentials: Readonly>): string[] { - const values = new Set(); - const visited = new WeakSet(); - let nodes = 0; - - const visit = (value: unknown, depth: number): void => { - nodes += 1; - if (nodes > MAX_JSON_NODES || depth > MAX_JSON_DEPTH) { - return; - } - if (typeof value === "string" && value.length > 0) { - values.add(value); - const encoded = encodeURIComponent(value); - if (encoded !== value) { - values.add(encoded); - } - return; - } - - if (Array.isArray(value)) { - if (visited.has(value)) return; - visited.add(value); - for (const item of value) { - visit(item, depth + 1); - } - return; - } - - if (isPlainRecord(value)) { - if (visited.has(value)) return; - visited.add(value); - for (const item of Object.values(value)) { - visit(item, depth + 1); - } - } - }; - - visit(credentials, 0); - return [...values].sort((left, right) => right.length - left.length); -} - -function dispatchCredentialRedactions(data: Readonly>): string[] { - return isPlainRecord(data.credentials) ? collectCredentialRedactions(data.credentials) : []; -} - -function redactSensitiveText(value: string, redactions: readonly string[]): string { - let redacted = value; - - for (const secret of redactions) { - redacted = redacted.split(secret).join("[REDACTED]"); - } - - return redacted; -} - -const DISPATCH_TIMEOUT = Symbol("plugin-daemon-dispatch-timeout"); -const DISPATCH_ABORTED = Symbol("plugin-daemon-dispatch-aborted"); - -type DispatchDeadlineReason = typeof DISPATCH_ABORTED | typeof DISPATCH_TIMEOUT; - -/** - * Applies one hard deadline to the full async-generator lifetime. Every pending `next()` races the - * same deadline, so a fetch implementation or response reader that ignores AbortSignal cannot keep - * datasource callers pending forever. Iterator cleanup is deliberately fire-and-forget: - * awaiting a non-cooperative iterator's `return()` would reintroduce the hang this fence prevents. - */ -async function* withDispatchDeadline( - runtime: PluginDaemonRuntime, - externalSignal: AbortSignal | undefined, - operation: (signal: AbortSignal) => AsyncGenerator, -): AsyncGenerator { - if (externalSignal?.aborted) { - throw dispatchDeadlineError(DISPATCH_ABORTED); - } - - const controller = new AbortController(); - let deadlineReason: DispatchDeadlineReason | undefined; - let resolveDeadline: ((reason: DispatchDeadlineReason) => void) | undefined; - const deadline = new Promise((resolve) => { - resolveDeadline = resolve; - }); - const settleDeadline = (reason: DispatchDeadlineReason): void => { - if (deadlineReason !== undefined) return; - deadlineReason = reason; - resolveDeadline?.(reason); - controller.abort(); - }; - const onExternalAbort = (): void => settleDeadline(DISPATCH_ABORTED); - externalSignal?.addEventListener("abort", onExternalAbort, { once: true }); - // Close the check/add race if the caller aborted between the initial guard and listener setup. - if (externalSignal?.aborted) { - onExternalAbort(); - } - const timeout = setTimeout( - () => settleDeadline(DISPATCH_TIMEOUT), - runtime.dispatchRequestTimeoutMs, - ); - const iterator = operation(controller.signal)[Symbol.asyncIterator](); - - try { - while (true) { - const outcome: - | { readonly result: IteratorResult; readonly type: "next" } - | { readonly reason: DispatchDeadlineReason; readonly type: "deadline" } = - await Promise.race([ - // Register the deadline first. settleDeadline resolves it before aborting the transport, - // so an abort-aware fetch cannot win the race with an implementation-specific error. - deadline.then((reason) => ({ reason, type: "deadline" }) as const), - iterator.next().then((result) => ({ result, type: "next" }) as const), - ]); - - if (outcome.type === "deadline") { - throw dispatchDeadlineError(outcome.reason); - } - if (outcome.result.done) { - return; - } - yield outcome.result.value; - } - } catch (cause) { - if (deadlineReason !== undefined) { - throw dispatchDeadlineError(deadlineReason); - } - if (cause instanceof PluginDaemonError) { - throw cause; - } - throw new PluginDaemonError("Plugin daemon request failed", { - code: "plugin_daemon_request_failed", - }); - } finally { - clearTimeout(timeout); - externalSignal?.removeEventListener("abort", onExternalAbort); - if (!controller.signal.aborted) { - controller.abort(); - } - const cleanup = iterator.return?.(undefined); - if (cleanup) { - void cleanup.catch(() => undefined); - } - } -} - -function dispatchDeadlineError(reason: DispatchDeadlineReason): PluginDaemonError { - return reason === DISPATCH_TIMEOUT - ? new PluginDaemonError("Plugin daemon request timed out", { - code: "plugin_daemon_timeout", - }) - : new PluginDaemonError("Plugin daemon request was aborted", { - code: "plugin_daemon_aborted", - }); -} - -function validateRequestTimeout(value: number, name: string): void { - if (!Number.isInteger(value) || value < 1 || value > MAX_REQUEST_TIMEOUT_MS) { - throw new PluginDaemonError( - `Plugin daemon ${name} must be between 1 and ${MAX_REQUEST_TIMEOUT_MS}`, - { code: "plugin_daemon_input" }, - ); - } -} - -function validateDispatchInput(input: { - readonly pluginId: string; - readonly tenantId: string; -}): void { - if (!input.tenantId.trim()) { - throw new PluginDaemonError("Plugin daemon dispatch requires a tenantId", { - code: "plugin_daemon_input", - }); - } - - if (!input.pluginId.trim()) { - throw new PluginDaemonError("Plugin daemon dispatch requires a pluginId", { - code: "plugin_daemon_input", - }); - } -} - -function unwrapEnvelope(raw: string, redactions: readonly string[] = []): unknown { - let parsed: unknown; - - try { - parsed = JSON.parse(raw); - } catch (cause) { - throw new PluginDaemonError("Plugin daemon returned invalid JSON", { - cause, - code: "plugin_daemon_response_invalid", - }); - } - - const envelope = PluginDaemonEnvelopeSchema.safeParse(parsed); - - if (!envelope.success) { - throw new PluginDaemonError("Plugin daemon returned an invalid envelope", { - cause: envelope.error, - code: "plugin_daemon_response_invalid", - }); - } - - if (envelope.data.code !== 0) { - const unwrapped = unwrapDaemonError(envelope.data.message ?? ""); - const safeMessage = redactSensitiveText(unwrapped.message, redactions); - const safeErrorType = unwrapped.errorType - ? redactSensitiveText(unwrapped.errorType, redactions) - : undefined; - - throw new PluginDaemonError(safeMessage || `Plugin daemon error code ${envelope.data.code}`, { - code: "plugin_daemon_invoke", - daemonCode: envelope.data.code, - ...(safeErrorType ? { errorType: safeErrorType } : {}), - }); - } - - // Mirrors dify base.py: a success envelope with empty `data` is an error. - if (envelope.data.data === undefined || envelope.data.data === null) { - throw new PluginDaemonError("Plugin daemon returned an empty data payload", { - code: "plugin_daemon_response_invalid", - }); - } - - return envelope.data.data; -} - -function unwrapDaemonError(message: string): { errorType?: string; message: string } { - const parsed = tryParseJson(message); - - if (parsed && typeof parsed === "object") { - const record = parsed as Record; - const errorType = record.error_type; - const innerMessage = record.message; - - if (typeof errorType === "string") { - // plugin-daemon nests the real error inside PluginInvokeError. - if (errorType === "PluginInvokeError" && typeof innerMessage === "string") { - return unwrapDaemonError(innerMessage); - } - - return { - errorType, - message: typeof innerMessage === "string" ? innerMessage : message, - }; - } - } - - return { message }; -} - -function tryParseJson(value: string): unknown { - try { - return JSON.parse(value); - } catch { - return null; - } -} - -function pluginDaemonRequestError(status: number): PluginDaemonError { - const message = `Plugin daemon request failed with status ${status}`; - - if (status === 429) { - return new PluginDaemonError(message, { code: "plugin_daemon_rate_limited", status }); - } - - return new PluginDaemonError(message, { code: "plugin_daemon_request_failed", status }); -} - -async function fetchWithRetries( - runtime: PluginDaemonRuntime, - input: string, - init: RequestInit, -): Promise { - for (let attempt = 0; ; attempt += 1) { - const response = await runtime.fetchImpl(input, init); - - if (!isRetryableStatus(response.status) || attempt >= runtime.maxRetries) { - return response; - } - - await response.body?.cancel().catch(() => undefined); - await runtime.sleep(runtime.retryDelayMs); - } -} - -function isRetryableStatus(status: number): boolean { - return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500; -} - -async function sleepMs(ms: number): Promise { - if (ms === 0) { - return; - } - - await new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} - -interface SseEvent { - readonly data: string; -} - -async function* readSseEvents(response: Response, maxBytes: number): AsyncGenerator { - if (!response.body) { - for (const event of parseSseEvents(await readBoundedText(response, maxBytes))) { - yield event; - } - - return; - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - let bytes = 0; - let completed = false; - - try { - while (true) { - const { done, value } = await reader.read(); - - if (done) { - completed = true; - break; - } - - bytes += value.byteLength; - - if (bytes > maxBytes) { - await reader.cancel().catch(() => undefined); - completed = true; - throw responseTooLarge(maxBytes); - } - - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split("\n"); - buffer = lines.pop() ?? ""; - - for (const rawLine of lines) { - const event = parseSseLine(rawLine); - if (event) { - yield event; - } - } - } - - buffer += decoder.decode(); - if (buffer) { - const event = parseSseLine(buffer); - if (event) { - yield event; - } - } - } finally { - if (!completed) { - await reader.cancel().catch(() => undefined); - } - - reader.releaseLock(); - } -} - -function parseSseEvents(text: string): SseEvent[] { - const events: SseEvent[] = []; - - for (const line of text.split(/\r?\n/u)) { - const event = parseSseLine(line); - if (event) { - events.push(event); - } - } - - return events; -} - -/** - * Mirrors dify base.py `_stream_request` line handling exactly: every non-empty line is one - * event, with an optional `data:` prefix stripped. The daemon emits one complete JSON envelope - * per line; there is no multi-line `data:` accumulation in the reference client. - */ -function parseSseLine(rawLine: string): SseEvent | null { - let line = rawLine.trim(); - - if (line.startsWith("data:")) { - line = line.slice(5).trim(); - } - - if (!line) { - return null; - } - - return { data: line }; -} - -async function readBoundedText(response: Response, maxBytes: number): Promise { - const declared = Number(response.headers.get("content-length")); - - if (Number.isFinite(declared) && declared > maxBytes) { - throw responseTooLarge(maxBytes); - } - - const text = await response.text(); - - if (new TextEncoder().encode(text).byteLength > maxBytes) { - throw responseTooLarge(maxBytes); - } - - return text; -} - -function responseTooLarge(maxBytes: number): PluginDaemonError { - return new PluginDaemonError(`Plugin daemon response exceeds maxResponseBytes=${maxBytes}`, { - code: "plugin_daemon_response_invalid", - }); -} diff --git a/knowledge-fs/packages/plugin-daemon-client/tsconfig.json b/knowledge-fs/packages/plugin-daemon-client/tsconfig.json deleted file mode 100644 index 9e25e6ece9a..00000000000 --- a/knowledge-fs/packages/plugin-daemon-client/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": ["src/**/*.ts"] -} diff --git a/knowledge-fs/packages/plugin-daemon-client/vitest.config.ts b/knowledge-fs/packages/plugin-daemon-client/vitest.config.ts deleted file mode 100644 index 7f126472859..00000000000 --- a/knowledge-fs/packages/plugin-daemon-client/vitest.config.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - coverage: { - exclude: ["src/**/*.test.ts"], - include: ["src/**/*.ts"], - provider: "v8", - reporter: ["text", "json-summary"], - thresholds: { - branches: 90, - functions: 90, - lines: 90, - statements: 90, - }, - }, - }, -}); diff --git a/knowledge-fs/pnpm-lock.yaml b/knowledge-fs/pnpm-lock.yaml index b99f52c884a..f482b231786 100644 --- a/knowledge-fs/pnpm-lock.yaml +++ b/knowledge-fs/pnpm-lock.yaml @@ -97,9 +97,6 @@ importers: '@knowledge/parsers': specifier: workspace:* version: link:../../packages/parsers - '@knowledge/plugin-daemon-client': - specifier: workspace:* - version: link:../../packages/plugin-daemon-client hono: specifier: ^4.12.25 version: 4.12.31 @@ -122,12 +119,6 @@ importers: packages/adapters: dependencies: - '@aws-sdk/client-s3': - specifier: ^3.1045.0 - version: 3.1045.0 - '@aws-sdk/s3-request-presigner': - specifier: 3.1045.0 - version: 3.1045.0 '@knowledge/core': specifier: workspace:* version: link:../core @@ -363,22 +354,6 @@ importers: specifier: ^2.1.8 version: 2.1.9(@types/node@22.19.18) - packages/plugin-daemon-client: - dependencies: - zod: - specifier: ^3.24.1 - version: 3.25.76 - devDependencies: - '@types/node': - specifier: ^22.10.2 - version: 22.19.18 - typescript: - specifier: ^5.7.2 - version: 5.9.3 - vitest: - specifier: ^2.1.8 - version: 2.1.9(@types/node@22.19.18) - packages: '@ampproject/remapping@2.3.0': @@ -390,193 +365,6 @@ packages: peerDependencies: zod: ^3.20.2 - '@aws-crypto/crc32@5.2.0': - resolution: {integrity: sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==} - engines: {node: '>=16.0.0'} - - '@aws-crypto/crc32c@5.2.0': - resolution: {integrity: sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==} - - '@aws-crypto/sha1-browser@5.2.0': - resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} - - '@aws-crypto/sha256-browser@5.2.0': - resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} - - '@aws-crypto/sha256-js@5.2.0': - resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} - engines: {node: '>=16.0.0'} - - '@aws-crypto/supports-web-crypto@5.2.0': - resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} - - '@aws-crypto/util@5.2.0': - resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} - - '@aws-sdk/client-s3@3.1045.0': - resolution: {integrity: sha512-fsuO3Y6t+3Ro9Bsg41DKj4Sfy53CGSrhnMldNplWmG8Tx0UbYk+YDa4RD1hVlJpERw4JBmPkl0+J9qlxMh1pcA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/core@3.974.8': - resolution: {integrity: sha512-njR2qoG6ZuB0kvAS2FyICsFZJ6gmCcf2X/7JcD14sUvGDm26wiZ5BrA6LOiUxKFEF+IVe7kdroxyE00YlkiYsw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/core@3.975.3': - resolution: {integrity: sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/crc64-nvme@3.972.7': - resolution: {integrity: sha512-QUagVVBbC8gODCF6e1aV0mE2TXWB9Opz4k8EJFdNrujUVQm5R4AjJa1mpOqzwOuROBzqJU9zawzig7M96L8Ejg==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-env@3.972.34': - resolution: {integrity: sha512-XT0jtf8Fw9JE6ppsQeoNnZRiG+jqRixMT1v1ZR17G60UvVdsQmTG8nbEyHuEPfMxDXEhfdARaM/XiEhca4lGHQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-http@3.972.36': - resolution: {integrity: sha512-DPoGWfy7J7RKxvbf5kOKIGQkD2ek3dbKgzKIGrnLuvZBz5myU+Im/H6pmc14QcnFbqHMqxvtWSgRDSJW3qXLQg==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-ini@3.972.38': - resolution: {integrity: sha512-oDzUBu2MGJFgoar05sPMCwSrhw44ASyccrHzj66vO69OZqi7I6hZZxXfuPLC8OCzW7C+sU+bI73XHij41yekgQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-login@3.972.38': - resolution: {integrity: sha512-g1NosS8qe4OF++G2UFCM5ovSkgipC7YYor5KCWatG0UoMSO5YFj9C8muePlyVmOBV/WTI16Jo3/s1NUo/o1Bww==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-node@3.972.39': - resolution: {integrity: sha512-HEswDQyxUtadoZ/bJsPPENHg7R0Lzym5LuMksJeHvqhCOpP+rtkDLKI4/ZChH4w3cf5kG8n6bZuI8PzajoiqMg==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-process@3.972.34': - resolution: {integrity: sha512-T3IFs4EVmVi1dVN5RciFnklCANSzvrQd/VuHY9ThHSQmYkTogjcGkoJEr+oNUPQZnso52183088NqysMPji1/Q==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-sso@3.972.38': - resolution: {integrity: sha512-5ZxG+t0+3Q3QPh8KEjX6syskhgNf7I0MN7oGioTf6Lm1NTjfP7sIcYGNsthXC2qR8vcD3edNZwCr2ovfSSWuRA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/credential-provider-web-identity@3.972.38': - resolution: {integrity: sha512-lYHFF30DGI20jZcYX8cm6Ns0V7f1dDN6g/MBDLTyD/5iw+bXs3yBr2iAiHDkx4RFU5JgsnZvCHYKiRVPRdmOgw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-bucket-endpoint@3.972.10': - resolution: {integrity: sha512-Vbc2frZH7wXlMNd+ZZSXUEs/l1Sv8Jj4zUnIfwrYF5lwaLdXHZ9xx4U3rjUcaye3HRhFVc+E5DbBxpRAbB16BA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-expect-continue@3.972.10': - resolution: {integrity: sha512-2Yn0f1Qiq/DjxYR3wfI3LokXnjOhFM7Ssn4LTdFDIxRMCE6I32MAsVnhPX1cUZsuVA9tiZtwwhlSLAtFGxAZlQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-flexible-checksums@3.974.16': - resolution: {integrity: sha512-6ru8doI0/XzszqLIPXf0E/V7HhAw1Pu94010XCKYtBUfD0LxF0BuOzrUf8OQGR6j2o6wgKTHUniOmndQycHwCA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-host-header@3.972.10': - resolution: {integrity: sha512-IJSsIMeVQ8MMCPbuh1AbltkFhLBLXn7aejzfX5YKT/VLDHn++Dcz8886tXckE+wQssyPUhaXrJhdakO2VilRhg==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-location-constraint@3.972.10': - resolution: {integrity: sha512-rI3NZvJcEvjoD0+0PI0iUAwlPw2IlSlhyvgBK/3WkKJQE/YiKFedd9dMN2lVacdNxPNhxL/jzQaKQdrGtQagjQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-logger@3.972.10': - resolution: {integrity: sha512-OOuGvvz1Dm20SjZo5oEBePFqxt5nf8AwkNDSyUHvD9/bfNASmstcYxFAHUowy4n6Io7mWUZ04JURZwSBvyQanQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-recursion-detection@3.972.11': - resolution: {integrity: sha512-+zz6f79Kj9V5qFK2P+D8Ehjnw4AhphAlCAsPjUqEcInA9umtSSKMrHbSagEeOIsDNuvVrH98bjRHcyQukTrhaQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-sdk-s3@3.972.37': - resolution: {integrity: sha512-Km7M+i8DrLArVzrid1gfxeGhYHBd3uxvE77g0s5a52zPSVosxzQBnJ0gwWb6NIp/DOk8gsBMhi7V+cpJG0ndTA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-ssec@3.972.10': - resolution: {integrity: sha512-Gli9A0u8EVVb+5bFDGS/QbSVg28w/wpEidg1ggVcSj65BDTdGR6punsOcVjqdiu1i42WHWo51MCvARPIIz9juw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/middleware-user-agent@3.972.38': - resolution: {integrity: sha512-iz+B29TXcAZsJpwB+AwG/TTGA5l/VnmMZ2UxtiySOZjI6gCdmviXPwdgzcmuazMy16rXoPY4mYCGe7zdNKfx5A==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/nested-clients@3.997.6': - resolution: {integrity: sha512-WBDnqatJl+kGObpfmfSxqnXeYTu3Me8wx8WCtvoxX3pfWrrTv8I4WTMSSs7PZqcRcVh8WeUKMgGFjMG+52SR1w==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/region-config-resolver@3.972.13': - resolution: {integrity: sha512-CvJ2ZIjK/jVD/lbOpowBVElJyC1YxLTIJ13yM0AEo0t2v7swOzGjSA6lJGH+DwZXQhcjUjoYwc8bVYCX5MDr1A==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/s3-request-presigner@3.1045.0': - resolution: {integrity: sha512-VDRF8GIuUPX+K4DUYrvcODj/h54LOmdJ7DhpLQ0wrYrdxzIiJEpi0n9jZ1bbjT2UxhwTbOorse5EGo+gnOK2aA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/signature-v4-multi-region@3.996.25': - resolution: {integrity: sha512-+CMIt3e1VzlklAECmG+DtP1sV8iKq25FuA0OKpnJ4KA0kxUtd7CgClY7/RU6VzJBQwbN4EJ9Ue6plvqx1qGadw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/signature-v4-multi-region@3.996.41': - resolution: {integrity: sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/token-providers@3.1041.0': - resolution: {integrity: sha512-Th7kPI6YPtvJUcdznooXJMy+9rQWjmEF81LxaJssngBzuysK4a/x+l8kjm1zb7nYsUPbndnBdUnwng/3PLvtGw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/types@3.973.8': - resolution: {integrity: sha512-gjlAdtHMbtR9X5iIhVUvbVcy55KnznpC6bkDUWW9z915bi0ckdUr5cjf16Kp6xq0bP5HBD2xzgbL9F9Quv5vUw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/types@3.974.2': - resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/util-arn-parser@3.972.3': - resolution: {integrity: sha512-HzSD8PMFrvgi2Kserxuff5VitNq2sgf3w9qxmskKDiDTThWfVteJxuCS9JXiPIPtmCrp+7N9asfIaVhBFORllA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/util-endpoints@3.996.8': - resolution: {integrity: sha512-oOZHcRDihk5iEe5V25NVWg45b3qEA8OpHWVdU/XQh8Zj4heVPAJqWvMphQnU7LkufmUo10EpvFPZuQMiFLJK3g==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/util-format-url@3.972.35': - resolution: {integrity: sha512-PIqDWSr2N3akz0K17KAo1Nusac2JpKG9scY8u2XUVH7UBg0G9AO9xOM4iSdn+4Ab7AETZr/tcVNfyfVR+ot/fw==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/util-locate-window@3.965.5': - resolution: {integrity: sha512-WhlJNNINQB+9qtLtZJcpQdgZw3SCDCpXdUJP7cToGwHbCWCnRckGlc6Bx/OhWwIYFNAn+FIydY8SZ0QmVu3xTQ==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/util-user-agent-browser@3.972.10': - resolution: {integrity: sha512-FAzqXvfEssGdSIz8ejatan0bOdx1qefBWKF/gWmVBXIP1HkS7v/wjjaqrAGGKvyihrXTXW00/2/1nTJtxpXz7g==} - - '@aws-sdk/util-user-agent-node@3.973.24': - resolution: {integrity: sha512-ZWwlkjcIp7cEL8ZfTpTAPNkwx25p7xol0xlKoWVVf22+nsjwmLcHYtTPjIV1cSpmB/b6DaK4cb1fSkvCXHgRdw==} - engines: {node: '>=20.0.0'} - peerDependencies: - aws-crt: '>=1.0.0' - peerDependenciesMeta: - aws-crt: - optional: true - - '@aws-sdk/xml-builder@3.972.22': - resolution: {integrity: sha512-PMYKKtJd70IsSG0yHrdAbxBr+ZWBKLvzFZfD3/urxgf6hXVMzuU5M+3MJ5G67RpOmLBu1fAUN65SbWuKUCOlAA==} - engines: {node: '>=20.0.0'} - - '@aws-sdk/xml-builder@3.972.36': - resolution: {integrity: sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==} - engines: {node: '>=20.0.0'} - - '@aws/lambda-invoke-store@0.2.4': - resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} - engines: {node: '>=18.0.0'} - - '@aws/lambda-invoke-store@0.3.0': - resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} - engines: {node: '>=18.0.0'} - '@babel/helper-string-parser@7.27.1': resolution: {integrity: sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==} engines: {node: '>=6.9.0'} @@ -1519,230 +1307,6 @@ packages: cpu: [x64] os: [win32] - '@smithy/chunked-blob-reader-native@4.2.3': - resolution: {integrity: sha512-jA5k5Udn7Y5717L86h4EIv06wIr3xn8GM1qHRi/Nf31annXcXHJjBKvgztnbn2TxH3xWrPBfgwHsOwZf0UmQWw==} - engines: {node: '>=18.0.0'} - - '@smithy/chunked-blob-reader@5.2.2': - resolution: {integrity: sha512-St+kVicSyayWQca+I1rGitaOEH6uKgE8IUWoYnnEX26SWdWQcL6LvMSD19Lg+vYHKdT9B2Zuu7rd3i6Wnyb/iw==} - engines: {node: '>=18.0.0'} - - '@smithy/config-resolver@4.4.17': - resolution: {integrity: sha512-TzDZcAnhTyAHbXVxWZo7/tEcrIeFq20IBk8So3OLOetWpR8EwY/yEqBMBFaJMeyEiREDq4NfEl+qO3OAUD+vbQ==} - engines: {node: '>=18.0.0'} - - '@smithy/core@3.23.17': - resolution: {integrity: sha512-x7BlLbUFL8NWCGjMF9C+1N5cVCxcPa7g6Tv9B4A2luWx3be3oU8hQ96wIwxe/s7OhIzvoJH73HAUSg5JXVlEtQ==} - engines: {node: '>=18.0.0'} - - '@smithy/core@3.29.6': - resolution: {integrity: sha512-TO3w25cdGWBeYqKNDaqH3v4O3jjMPpKwf39YlG5X5xhqWfpOWJbi5gQi1lrllukuwohdhY0TPB8jBEv6UC50Vg==} - engines: {node: '>=18.0.0'} - - '@smithy/credential-provider-imds@4.2.14': - resolution: {integrity: sha512-Au28zBN48ZAoXdooGUHemuVBrkE+Ie6RPmGNIAJsFqj33Vhb6xAgRifUydZ2aY+M+KaMAETAlKk5NC5h1G7wpg==} - engines: {node: '>=18.0.0'} - - '@smithy/eventstream-codec@4.2.14': - resolution: {integrity: sha512-erZq0nOIpzfeZdCyzZjdJb4nVSKLUmSkaQUVkRGQTXs30gyUGeKnrYEg+Xe1W5gE3aReS7IgsvANwVPxSzY6Pw==} - engines: {node: '>=18.0.0'} - - '@smithy/eventstream-serde-browser@4.2.14': - resolution: {integrity: sha512-8IelTCtTctWRbb+0Dcy+C0aICh1qa0qWXqgjcXDmMuCvPJRnv26hiDZoAau2ILOniki65mCPKqOQs/BaWvO4CQ==} - engines: {node: '>=18.0.0'} - - '@smithy/eventstream-serde-config-resolver@4.3.14': - resolution: {integrity: sha512-sqHiHpYRYo3FJlaIxD1J8PhbcmJAm7IuM16mVnwSkCToD7g00IBZzKuiLNMGmftULmEUX6/UAz8/NN5uMP8bVA==} - engines: {node: '>=18.0.0'} - - '@smithy/eventstream-serde-node@4.2.14': - resolution: {integrity: sha512-Ht/8BuGlKfFTy0H3+8eEu0vdpwGztCnaLLXtpXNdQqiR7Hj4vFScU3T436vRAjATglOIPjJXronY+1WxxNLSiw==} - engines: {node: '>=18.0.0'} - - '@smithy/eventstream-serde-universal@4.2.14': - resolution: {integrity: sha512-lWyt4T2XQZUZgK3tQ3Wn0w3XBvZsK/vjTuJl6bXbnGZBHH0ZUSONTYiK9TgjTTzU54xQr3DRFwpjmhp0oLm3gg==} - engines: {node: '>=18.0.0'} - - '@smithy/fetch-http-handler@5.3.17': - resolution: {integrity: sha512-bXOvQzaSm6MnmLaWA1elgfQcAtN4UP3vXqV97bHuoOrHQOJiLT3ds6o9eo5bqd0TJfRFpzdGnDQdW3FACiAVdw==} - engines: {node: '>=18.0.0'} - - '@smithy/hash-blob-browser@4.2.15': - resolution: {integrity: sha512-0PJ4Al3fg2nM4qKrAIxyNcApgqHAXcBkN8FeizOz69z0rb26uZ6lMESYtxegaTlXB5Hj84JfwMPavMrwDMjucA==} - engines: {node: '>=18.0.0'} - - '@smithy/hash-node@4.2.14': - resolution: {integrity: sha512-8ZBDY2DD4wr+GGjTpPtiglEsqr0lUP+KHqgZcWczFf6qeZ/YRjMIOoQWVQlmwu7EtxKTd8YXD8lblmYcpBIA1g==} - engines: {node: '>=18.0.0'} - - '@smithy/hash-stream-node@4.2.14': - resolution: {integrity: sha512-tw4GANWkZPb6+BdD4Fgucqzey2+r73Z/GRo9zklsCdwrnxxumUV83ZIaBDdudV4Ylazw3EPTiJZhpX42105ruQ==} - engines: {node: '>=18.0.0'} - - '@smithy/invalid-dependency@4.2.14': - resolution: {integrity: sha512-c21qJiTSb25xvvOp+H2TNZzPCngrvl5vIPqPB8zQ/DmJF4QWXO19x1dWfMJZ6wZuuWUPPm0gV8C0cU3+ifcWuw==} - engines: {node: '>=18.0.0'} - - '@smithy/is-array-buffer@2.2.0': - resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} - engines: {node: '>=14.0.0'} - - '@smithy/is-array-buffer@4.2.2': - resolution: {integrity: sha512-n6rQ4N8Jj4YTQO3YFrlgZuwKodf4zUFs7EJIWH86pSCWBaAtAGBFfCM7Wx6D2bBJ2xqFNxGBSrUWswT3M0VJow==} - engines: {node: '>=18.0.0'} - - '@smithy/md5-js@4.2.14': - resolution: {integrity: sha512-V2v0vx+h0iUSNG1Alt+GNBMSLGCrl9iVsdd+Ap67HPM9PN479x12V8LkuMoKImNZxn3MXeuyUjls+/7ZACZghA==} - engines: {node: '>=18.0.0'} - - '@smithy/middleware-content-length@4.2.14': - resolution: {integrity: sha512-xhHq7fX4/3lv5NHxLUk3OeEvl0xZ+Ek3qIbWaCL4f9JwgDZEclPBElljaZCAItdGPQl/kSM4LPMOpy1MYgprpw==} - engines: {node: '>=18.0.0'} - - '@smithy/middleware-endpoint@4.4.32': - resolution: {integrity: sha512-ZZkgyjnJppiZbIm6Qbx92pbXYi1uzenIvGhBSCDlc7NwuAkiqSgS75j1czAD25ZLs2FjMjYy1q7gyRVWG6JA0Q==} - engines: {node: '>=18.0.0'} - - '@smithy/middleware-retry@4.5.7': - resolution: {integrity: sha512-bRt6ZImqVSeTk39Nm81K20ObIiAZ3WefY7G6+iz/0tZjs4dgRRjvRX2sgsH+zi6iDCRR/aQvQofLKxxz4rPBZg==} - engines: {node: '>=18.0.0'} - - '@smithy/middleware-serde@4.2.20': - resolution: {integrity: sha512-Lx9JMO9vArPtiChE3wbEZ5akMIDQpWQtlu90lhACQmNOXcGXRbaDywMHDzuDZ2OkZzP+9wQfZi3YJT9F67zTQQ==} - engines: {node: '>=18.0.0'} - - '@smithy/middleware-stack@4.2.14': - resolution: {integrity: sha512-2dvkUKLuFdKsCRmOE4Mn63co0Djtsm+JMh0bYZQupN1pJwMeE8FmQmRLLzzEMN0dnNi7CDCYYH8F0EVwWiPBeA==} - engines: {node: '>=18.0.0'} - - '@smithy/node-config-provider@4.3.14': - resolution: {integrity: sha512-S+gFjyo/weSVL0P1b9Ts8C/CwIfNCgUPikk3sl6QVsfE/uUuO+QsF+NsE/JkpvWqqyz1wg7HFdiaZuj5CoBMRg==} - engines: {node: '>=18.0.0'} - - '@smithy/node-http-handler@4.6.1': - resolution: {integrity: sha512-iB+orM4x3xrr57X3YaXazfKnntl0LHlZB1kcXSGzMV1Tt0+YwEjGlbjk/44qEGtBzXAz6yFDzkYTKSV6Pj2HUg==} - engines: {node: '>=18.0.0'} - - '@smithy/property-provider@4.2.14': - resolution: {integrity: sha512-WuM31CgfsnQ/10i7NYr0PyxqknD72Y5uMfUMVSniPjbEPceiTErb4eIqJQ+pdxNEAUEWrewrGjIRjVbVHsxZiQ==} - engines: {node: '>=18.0.0'} - - '@smithy/protocol-http@5.3.14': - resolution: {integrity: sha512-dN5F8kHx8RNU0r+pCwNmFZyz6ChjMkzShy/zup6MtkRmmix4vZzJdW+di7x//b1LiynIev88FM18ie+wwPcQtQ==} - engines: {node: '>=18.0.0'} - - '@smithy/querystring-builder@4.2.14': - resolution: {integrity: sha512-XYA5Z0IqTeF+5XDdh4BBmSA0HvbgVZIyv4cmOoUheDNR57K1HgBp9ukUMx3Cr3XpDHHpLBnexPE3LAtDsZkj2A==} - engines: {node: '>=18.0.0'} - - '@smithy/querystring-parser@4.2.14': - resolution: {integrity: sha512-hr+YyqBD23GVvRxGGrcc/oOeNlK3PzT5Fu4dzrDXxzS1LpFiuL2PQQqKPs87M79aW7ziMs+nvB3qdw77SqE7Lw==} - engines: {node: '>=18.0.0'} - - '@smithy/service-error-classification@4.3.1': - resolution: {integrity: sha512-aUQuDGh760ts/8MU+APjIZhlLPKhIIfqyzZaJikLEIMrdxFvxuLYD0WxWzaYWpmLbQlXDe9p7EWM3HsBe0K6Gw==} - engines: {node: '>=18.0.0'} - - '@smithy/shared-ini-file-loader@4.4.9': - resolution: {integrity: sha512-495/V2I15SHgedSJoDPD23JuSfKAp726ZI1V0wtjB07Wh7q/0tri/0e0DLefZCHgxZonrGKt/OCTpAtP1wE1kQ==} - engines: {node: '>=18.0.0'} - - '@smithy/signature-v4@5.3.14': - resolution: {integrity: sha512-1D9Y/nmlVjCeSivCbhZ7hgEpmHyY1h0GvpSZt3l0xcD9JjmjVC1CHOozS6+Gh+/ldMH8JuJ6cujObQqfayAVFA==} - engines: {node: '>=18.0.0'} - - '@smithy/signature-v4@5.6.7': - resolution: {integrity: sha512-32PmEsuZV9lz7SZk3gJcm+EfIAoIVu83AJyEzgALpwmSqLvuacdAu0fvCVNMbDbegyk1S0lHUDrMWIfR47Micw==} - engines: {node: '>=18.0.0'} - - '@smithy/smithy-client@4.12.13': - resolution: {integrity: sha512-y/Pcj1V9+qG98gyu1gvftHB7rDpdh+7kIBIggs55yGm3JdtBV8GT8IFF3a1qxZ79QnaJHX9GXzvBG6tAd+czJA==} - engines: {node: '>=18.0.0'} - - '@smithy/types@4.14.1': - resolution: {integrity: sha512-59b5HtSVrVR/eYNei3BUj3DCPKD/G7EtDDe7OEJE7i7FtQFugYo6MxbotS8mVJkLNVf8gYaAlEBwwtJ9HzhWSg==} - engines: {node: '>=18.0.0'} - - '@smithy/types@4.16.1': - resolution: {integrity: sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==} - engines: {node: '>=18.0.0'} - - '@smithy/url-parser@4.2.14': - resolution: {integrity: sha512-p06BiBigJ8bTA3MgnOfCtDUWnAMY0YfedO/GRpmc7p+wg3KW8vbXy1xwSu5ASy0wV7rRYtlfZOIKH4XqfhjSQQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-base64@4.3.2': - resolution: {integrity: sha512-XRH6b0H/5A3SgblmMa5ErXQ2XKhfbQB+Fm/oyLZ2O2kCUrwgg55bU0RekmzAhuwOjA9qdN5VU2BprOvGGUkOOQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-body-length-browser@4.2.2': - resolution: {integrity: sha512-JKCrLNOup3OOgmzeaKQwi4ZCTWlYR5H4Gm1r2uTMVBXoemo1UEghk5vtMi1xSu2ymgKVGW631e2fp9/R610ZjQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-body-length-node@4.2.3': - resolution: {integrity: sha512-ZkJGvqBzMHVHE7r/hcuCxlTY8pQr1kMtdsVPs7ex4mMU+EAbcXppfo5NmyxMYi2XU49eqaz56j2gsk4dHHPG/g==} - engines: {node: '>=18.0.0'} - - '@smithy/util-buffer-from@2.2.0': - resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} - engines: {node: '>=14.0.0'} - - '@smithy/util-buffer-from@4.2.2': - resolution: {integrity: sha512-FDXD7cvUoFWwN6vtQfEta540Y/YBe5JneK3SoZg9bThSoOAC/eGeYEua6RkBgKjGa/sz6Y+DuBZj3+YEY21y4Q==} - engines: {node: '>=18.0.0'} - - '@smithy/util-config-provider@4.2.2': - resolution: {integrity: sha512-dWU03V3XUprJwaUIFVv4iOnS1FC9HnMHDfUrlNDSh4315v0cWyaIErP8KiqGVbf5z+JupoVpNM7ZB3jFiTejvQ==} - engines: {node: '>=18.0.0'} - - '@smithy/util-defaults-mode-browser@4.3.49': - resolution: {integrity: sha512-a5bNrdiONYB/qE2BuKegvUMd/+ZDwdg4vsNuuSzYE8qs2EYAdK9CynL+Rzn29PbPiUqoz/cbpRbcLzD5lEevHw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-defaults-mode-node@4.2.54': - resolution: {integrity: sha512-g1cvrJvOnzeJgEdf7AE4luI7gp6L8weE0y9a9wQUSGtjb8QRHDbCJYuE4Sy0SD9N8RrnNPFsPltAz/OSoBR9Zw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-endpoints@3.4.2': - resolution: {integrity: sha512-a55Tr+3OKld4TTtnT+RhKOQHyPxm3j/xL4OR83WBUhLJaKDS9dnJ7arRMOp3t31dcLhApwG9bgvrRXBHlLdIkg==} - engines: {node: '>=18.0.0'} - - '@smithy/util-hex-encoding@4.2.2': - resolution: {integrity: sha512-Qcz3W5vuHK4sLQdyT93k/rfrUwdJ8/HZ+nMUOyGdpeGA1Wxt65zYwi3oEl9kOM+RswvYq90fzkNDahPS8K0OIg==} - engines: {node: '>=18.0.0'} - - '@smithy/util-middleware@4.2.14': - resolution: {integrity: sha512-1Su2vj9RYNDEv/V+2E+jXkkwGsgR7dc4sfHn9Z7ruzQHJIEni9zzw5CauvRXlFJfmgcqYP8fWa0dkh2Q2YaQyw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-retry@4.3.8': - resolution: {integrity: sha512-LUIxbTBi+OpvXpg91poGA6BdyoleMDLnfXjVDqyi2RvZmTveY5loE/FgYUBCR5LU2BThW2SoZRh8dTIIy38IPw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-stream@4.5.25': - resolution: {integrity: sha512-/PFpG4k8Ze8Ei+mMKj3oiPICYekthuzePZMgZbCqMiXIHHf4n2aZ4Ps0aSRShycFTGuj/J6XldmC0x0DwednIA==} - engines: {node: '>=18.0.0'} - - '@smithy/util-uri-escape@4.2.2': - resolution: {integrity: sha512-2kAStBlvq+lTXHyAZYfJRb/DfS3rsinLiwb+69SstC9Vb0s9vNWkRwpnj918Pfi85mzi42sOqdV72OLxWAISnw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-utf8@2.3.0': - resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} - engines: {node: '>=14.0.0'} - - '@smithy/util-utf8@4.2.2': - resolution: {integrity: sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw==} - engines: {node: '>=18.0.0'} - - '@smithy/util-waiter@4.3.0': - resolution: {integrity: sha512-JyjYmLAfS+pdxF92o4yLgEoy0zhayKTw73FU1aofLWwLcJw7iSqIY2exGmMTrl/lmZugP5p/zxdFSippJDfKWA==} - engines: {node: '>=18.0.0'} - - '@smithy/uuid@1.1.2': - resolution: {integrity: sha512-O/IEdcCUKkubz60tFbGA7ceITTAJsty+lBjNoorP4Z6XRqaFb/OjQjZODophEcuq68nKm6/0r+6/lLQ+XVpk8g==} - engines: {node: '>=18.0.0'} - '@swc/helpers@0.5.15': resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==} @@ -1883,9 +1447,6 @@ packages: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} - bowser@2.14.1: - resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} - brace-expansion@2.1.0: resolution: {integrity: sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==} @@ -2089,16 +1650,9 @@ packages: fast-uri@3.1.4: resolution: {integrity: sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==} - fast-xml-builder@1.1.9: - resolution: {integrity: sha512-jcyKVSEX13iseJqg7n/KWw+xnu/7fdrZ333Fac54KjHDIELVCfDDJXYIm6DTJ0Su4gSzrhqiK0DzY/wZbF40mw==} - fast-xml-builder@1.2.0: resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} - fast-xml-parser@5.7.2: - resolution: {integrity: sha512-P7oW7tLbYnhOLQk/Gv7cZgzgMPP/XN03K02/Jy6Y/NHzyIAIpxuZIM/YqAkfiXFPxA2CTm7NtCijK9EDu09u2w==} - hasBin: true - fast-xml-parser@5.8.0: resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true @@ -2766,499 +2320,6 @@ snapshots: openapi3-ts: 4.5.0 zod: 3.25.76 - '@aws-crypto/crc32@5.2.0': - dependencies: - '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 - tslib: 2.8.1 - - '@aws-crypto/crc32c@5.2.0': - dependencies: - '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 - tslib: 2.8.1 - - '@aws-crypto/sha1-browser@5.2.0': - dependencies: - '@aws-crypto/supports-web-crypto': 5.2.0 - '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-locate-window': 3.965.5 - '@smithy/util-utf8': 2.3.0 - tslib: 2.8.1 - - '@aws-crypto/sha256-browser@5.2.0': - dependencies: - '@aws-crypto/sha256-js': 5.2.0 - '@aws-crypto/supports-web-crypto': 5.2.0 - '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-locate-window': 3.965.5 - '@smithy/util-utf8': 2.3.0 - tslib: 2.8.1 - - '@aws-crypto/sha256-js@5.2.0': - dependencies: - '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.973.8 - tslib: 2.8.1 - - '@aws-crypto/supports-web-crypto@5.2.0': - dependencies: - tslib: 2.8.1 - - '@aws-crypto/util@5.2.0': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/util-utf8': 2.3.0 - tslib: 2.8.1 - - '@aws-sdk/client-s3@3.1045.0': - dependencies: - '@aws-crypto/sha1-browser': 5.2.0 - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/credential-provider-node': 3.972.39 - '@aws-sdk/middleware-bucket-endpoint': 3.972.10 - '@aws-sdk/middleware-expect-continue': 3.972.10 - '@aws-sdk/middleware-flexible-checksums': 3.974.16 - '@aws-sdk/middleware-host-header': 3.972.10 - '@aws-sdk/middleware-location-constraint': 3.972.10 - '@aws-sdk/middleware-logger': 3.972.10 - '@aws-sdk/middleware-recursion-detection': 3.972.11 - '@aws-sdk/middleware-sdk-s3': 3.972.37 - '@aws-sdk/middleware-ssec': 3.972.10 - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/region-config-resolver': 3.972.13 - '@aws-sdk/signature-v4-multi-region': 3.996.25 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@aws-sdk/util-user-agent-browser': 3.972.10 - '@aws-sdk/util-user-agent-node': 3.973.24 - '@smithy/config-resolver': 4.4.17 - '@smithy/core': 3.23.17 - '@smithy/eventstream-serde-browser': 4.2.14 - '@smithy/eventstream-serde-config-resolver': 4.3.14 - '@smithy/eventstream-serde-node': 4.2.14 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/hash-blob-browser': 4.2.15 - '@smithy/hash-node': 4.2.14 - '@smithy/hash-stream-node': 4.2.14 - '@smithy/invalid-dependency': 4.2.14 - '@smithy/md5-js': 4.2.14 - '@smithy/middleware-content-length': 4.2.14 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-retry': 4.5.7 - '@smithy/middleware-serde': 4.2.20 - '@smithy/middleware-stack': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.49 - '@smithy/util-defaults-mode-node': 4.2.54 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.8 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - '@smithy/util-waiter': 4.3.0 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/core@3.974.8': - dependencies: - '@aws-sdk/types': 3.973.8 - '@aws-sdk/xml-builder': 3.972.22 - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.8 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@aws-sdk/core@3.975.3': - dependencies: - '@aws-sdk/types': 3.974.2 - '@aws-sdk/xml-builder': 3.972.36 - '@aws/lambda-invoke-store': 0.3.0 - '@smithy/core': 3.29.6 - '@smithy/signature-v4': 5.6.7 - '@smithy/types': 4.16.1 - bowser: 2.14.1 - tslib: 2.8.1 - - '@aws-sdk/crc64-nvme@3.972.7': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/credential-provider-env@3.972.34': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/credential-provider-http@3.972.36': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/node-http-handler': 4.6.1 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-stream': 4.5.25 - tslib: 2.8.1 - - '@aws-sdk/credential-provider-ini@3.972.38': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/credential-provider-env': 3.972.34 - '@aws-sdk/credential-provider-http': 3.972.36 - '@aws-sdk/credential-provider-login': 3.972.38 - '@aws-sdk/credential-provider-process': 3.972.34 - '@aws-sdk/credential-provider-sso': 3.972.38 - '@aws-sdk/credential-provider-web-identity': 3.972.38 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/types': 3.973.8 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-login@3.972.38': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-node@3.972.39': - dependencies: - '@aws-sdk/credential-provider-env': 3.972.34 - '@aws-sdk/credential-provider-http': 3.972.36 - '@aws-sdk/credential-provider-ini': 3.972.38 - '@aws-sdk/credential-provider-process': 3.972.34 - '@aws-sdk/credential-provider-sso': 3.972.38 - '@aws-sdk/credential-provider-web-identity': 3.972.38 - '@aws-sdk/types': 3.973.8 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-process@3.972.34': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/credential-provider-sso@3.972.38': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/token-providers': 3.1041.0 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/credential-provider-web-identity@3.972.38': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/middleware-bucket-endpoint@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-arn-parser': 3.972.3 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - tslib: 2.8.1 - - '@aws-sdk/middleware-expect-continue@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-flexible-checksums@3.974.16': - dependencies: - '@aws-crypto/crc32': 5.2.0 - '@aws-crypto/crc32c': 5.2.0 - '@aws-crypto/util': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/crc64-nvme': 3.972.7 - '@aws-sdk/types': 3.973.8 - '@smithy/is-array-buffer': 4.2.2 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@aws-sdk/middleware-host-header@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-location-constraint@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-logger@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-recursion-detection@3.972.11': - dependencies: - '@aws-sdk/types': 3.973.8 - '@aws/lambda-invoke-store': 0.2.4 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-sdk-s3@3.972.37': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-arn-parser': 3.972.3 - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@aws-sdk/middleware-ssec@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/middleware-user-agent@3.972.38': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@smithy/core': 3.23.17 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-retry': 4.3.8 - tslib: 2.8.1 - - '@aws-sdk/nested-clients@3.997.6': - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.974.8 - '@aws-sdk/middleware-host-header': 3.972.10 - '@aws-sdk/middleware-logger': 3.972.10 - '@aws-sdk/middleware-recursion-detection': 3.972.11 - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/region-config-resolver': 3.972.13 - '@aws-sdk/signature-v4-multi-region': 3.996.25 - '@aws-sdk/types': 3.973.8 - '@aws-sdk/util-endpoints': 3.996.8 - '@aws-sdk/util-user-agent-browser': 3.972.10 - '@aws-sdk/util-user-agent-node': 3.973.24 - '@smithy/config-resolver': 4.4.17 - '@smithy/core': 3.23.17 - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/hash-node': 4.2.14 - '@smithy/invalid-dependency': 4.2.14 - '@smithy/middleware-content-length': 4.2.14 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-retry': 4.5.7 - '@smithy/middleware-serde': 4.2.20 - '@smithy/middleware-stack': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/node-http-handler': 4.6.1 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.49 - '@smithy/util-defaults-mode-node': 4.2.54 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.8 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/region-config-resolver@3.972.13': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/config-resolver': 4.4.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/s3-request-presigner@3.1045.0': - dependencies: - '@aws-sdk/signature-v4-multi-region': 3.996.41 - '@aws-sdk/types': 3.974.2 - '@aws-sdk/util-format-url': 3.972.35 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/protocol-http': 5.3.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/signature-v4-multi-region@3.996.25': - dependencies: - '@aws-sdk/middleware-sdk-s3': 3.972.37 - '@aws-sdk/types': 3.973.8 - '@smithy/protocol-http': 5.3.14 - '@smithy/signature-v4': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/signature-v4-multi-region@3.996.41': - dependencies: - '@aws-sdk/types': 3.974.2 - '@smithy/signature-v4': 5.6.7 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/token-providers@3.1041.0': - dependencies: - '@aws-sdk/core': 3.974.8 - '@aws-sdk/nested-clients': 3.997.6 - '@aws-sdk/types': 3.973.8 - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - - '@aws-sdk/types@3.973.8': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@aws-sdk/types@3.974.2': - dependencies: - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws-sdk/util-arn-parser@3.972.3': - dependencies: - tslib: 2.8.1 - - '@aws-sdk/util-endpoints@3.996.8': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-endpoints': 3.4.2 - tslib: 2.8.1 - - '@aws-sdk/util-format-url@3.972.35': - dependencies: - '@aws-sdk/core': 3.975.3 - tslib: 2.8.1 - - '@aws-sdk/util-locate-window@3.965.5': - dependencies: - tslib: 2.8.1 - - '@aws-sdk/util-user-agent-browser@3.972.10': - dependencies: - '@aws-sdk/types': 3.973.8 - '@smithy/types': 4.14.1 - bowser: 2.14.1 - tslib: 2.8.1 - - '@aws-sdk/util-user-agent-node@3.973.24': - dependencies: - '@aws-sdk/middleware-user-agent': 3.972.38 - '@aws-sdk/types': 3.973.8 - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - tslib: 2.8.1 - - '@aws-sdk/xml-builder@3.972.22': - dependencies: - '@nodable/entities': 2.1.0 - '@smithy/types': 4.14.1 - fast-xml-parser: 5.7.2 - tslib: 2.8.1 - - '@aws-sdk/xml-builder@3.972.36': - dependencies: - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@aws/lambda-invoke-store@0.2.4': {} - - '@aws/lambda-invoke-store@0.3.0': {} - '@babel/helper-string-parser@7.27.1': {} '@babel/helper-validator-identifier@7.28.5': {} @@ -3814,354 +2875,6 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.60.3': optional: true - '@smithy/chunked-blob-reader-native@4.2.3': - dependencies: - '@smithy/util-base64': 4.3.2 - tslib: 2.8.1 - - '@smithy/chunked-blob-reader@5.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/config-resolver@4.4.17': - dependencies: - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-config-provider': 4.2.2 - '@smithy/util-endpoints': 3.4.2 - '@smithy/util-middleware': 4.2.14 - tslib: 2.8.1 - - '@smithy/core@3.23.17': - dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-stream': 4.5.25 - '@smithy/util-utf8': 4.2.2 - '@smithy/uuid': 1.1.2 - tslib: 2.8.1 - - '@smithy/core@3.29.6': - dependencies: - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@smithy/credential-provider-imds@4.2.14': - dependencies: - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - tslib: 2.8.1 - - '@smithy/eventstream-codec@4.2.14': - dependencies: - '@aws-crypto/crc32': 5.2.0 - '@smithy/types': 4.14.1 - '@smithy/util-hex-encoding': 4.2.2 - tslib: 2.8.1 - - '@smithy/eventstream-serde-browser@4.2.14': - dependencies: - '@smithy/eventstream-serde-universal': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/eventstream-serde-config-resolver@4.3.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/eventstream-serde-node@4.2.14': - dependencies: - '@smithy/eventstream-serde-universal': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/eventstream-serde-universal@4.2.14': - dependencies: - '@smithy/eventstream-codec': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/fetch-http-handler@5.3.17': - dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/querystring-builder': 4.2.14 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 - tslib: 2.8.1 - - '@smithy/hash-blob-browser@4.2.15': - dependencies: - '@smithy/chunked-blob-reader': 5.2.2 - '@smithy/chunked-blob-reader-native': 4.2.3 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/hash-node@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-buffer-from': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/hash-stream-node@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/invalid-dependency@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/is-array-buffer@2.2.0': - dependencies: - tslib: 2.8.1 - - '@smithy/is-array-buffer@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/md5-js@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/middleware-content-length@4.2.14': - dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/middleware-endpoint@4.4.32': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/middleware-serde': 4.2.20 - '@smithy/node-config-provider': 4.3.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - '@smithy/url-parser': 4.2.14 - '@smithy/util-middleware': 4.2.14 - tslib: 2.8.1 - - '@smithy/middleware-retry@4.5.7': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/node-config-provider': 4.3.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/service-error-classification': 4.3.1 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-retry': 4.3.8 - '@smithy/uuid': 1.1.2 - tslib: 2.8.1 - - '@smithy/middleware-serde@4.2.20': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/middleware-stack@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/node-config-provider@4.3.14': - dependencies: - '@smithy/property-provider': 4.2.14 - '@smithy/shared-ini-file-loader': 4.4.9 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/node-http-handler@4.6.1': - dependencies: - '@smithy/protocol-http': 5.3.14 - '@smithy/querystring-builder': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/property-provider@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/protocol-http@5.3.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/querystring-builder@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - '@smithy/util-uri-escape': 4.2.2 - tslib: 2.8.1 - - '@smithy/querystring-parser@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/service-error-classification@4.3.1': - dependencies: - '@smithy/types': 4.14.1 - - '@smithy/shared-ini-file-loader@4.4.9': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/signature-v4@5.3.14': - dependencies: - '@smithy/is-array-buffer': 4.2.2 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-hex-encoding': 4.2.2 - '@smithy/util-middleware': 4.2.14 - '@smithy/util-uri-escape': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/signature-v4@5.6.7': - dependencies: - '@smithy/core': 3.29.6 - '@smithy/types': 4.16.1 - tslib: 2.8.1 - - '@smithy/smithy-client@4.12.13': - dependencies: - '@smithy/core': 3.23.17 - '@smithy/middleware-endpoint': 4.4.32 - '@smithy/middleware-stack': 4.2.14 - '@smithy/protocol-http': 5.3.14 - '@smithy/types': 4.14.1 - '@smithy/util-stream': 4.5.25 - tslib: 2.8.1 - - '@smithy/types@4.14.1': - dependencies: - tslib: 2.8.1 - - '@smithy/types@4.16.1': - dependencies: - tslib: 2.8.1 - - '@smithy/url-parser@4.2.14': - dependencies: - '@smithy/querystring-parser': 4.2.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-base64@4.3.2': - dependencies: - '@smithy/util-buffer-from': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/util-body-length-browser@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/util-body-length-node@4.2.3': - dependencies: - tslib: 2.8.1 - - '@smithy/util-buffer-from@2.2.0': - dependencies: - '@smithy/is-array-buffer': 2.2.0 - tslib: 2.8.1 - - '@smithy/util-buffer-from@4.2.2': - dependencies: - '@smithy/is-array-buffer': 4.2.2 - tslib: 2.8.1 - - '@smithy/util-config-provider@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/util-defaults-mode-browser@4.3.49': - dependencies: - '@smithy/property-provider': 4.2.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-defaults-mode-node@4.2.54': - dependencies: - '@smithy/config-resolver': 4.4.17 - '@smithy/credential-provider-imds': 4.2.14 - '@smithy/node-config-provider': 4.3.14 - '@smithy/property-provider': 4.2.14 - '@smithy/smithy-client': 4.12.13 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-endpoints@3.4.2': - dependencies: - '@smithy/node-config-provider': 4.3.14 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-hex-encoding@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/util-middleware@4.2.14': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-retry@4.3.8': - dependencies: - '@smithy/service-error-classification': 4.3.1 - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/util-stream@4.5.25': - dependencies: - '@smithy/fetch-http-handler': 5.3.17 - '@smithy/node-http-handler': 4.6.1 - '@smithy/types': 4.14.1 - '@smithy/util-base64': 4.3.2 - '@smithy/util-buffer-from': 4.2.2 - '@smithy/util-hex-encoding': 4.2.2 - '@smithy/util-utf8': 4.2.2 - tslib: 2.8.1 - - '@smithy/util-uri-escape@4.2.2': - dependencies: - tslib: 2.8.1 - - '@smithy/util-utf8@2.3.0': - dependencies: - '@smithy/util-buffer-from': 2.2.0 - tslib: 2.8.1 - - '@smithy/util-utf8@4.2.2': - dependencies: - '@smithy/util-buffer-from': 4.2.2 - tslib: 2.8.1 - - '@smithy/util-waiter@4.3.0': - dependencies: - '@smithy/types': 4.14.1 - tslib: 2.8.1 - - '@smithy/uuid@1.1.2': - dependencies: - tslib: 2.8.1 - '@swc/helpers@0.5.15': dependencies: tslib: 2.8.1 @@ -4312,8 +3025,6 @@ snapshots: transitivePeerDependencies: - supports-color - bowser@2.14.1: {} - brace-expansion@2.1.0: dependencies: balanced-match: 1.0.2 @@ -4577,22 +3288,11 @@ snapshots: fast-uri@3.1.4: {} - fast-xml-builder@1.1.9: - dependencies: - path-expression-matcher: 1.5.0 - fast-xml-builder@1.2.0: dependencies: path-expression-matcher: 1.5.0 xml-naming: 0.1.0 - fast-xml-parser@5.7.2: - dependencies: - '@nodable/entities': 2.1.0 - fast-xml-builder: 1.1.9 - path-expression-matcher: 1.5.0 - strnum: 2.3.0 - fast-xml-parser@5.8.0: dependencies: '@nodable/entities': 2.1.0 diff --git a/knowledge-fs/scripts/api-image-bundle-smoke.mjs b/knowledge-fs/scripts/api-image-bundle-smoke.mjs index dc267719aa4..76fe7b69af7 100644 --- a/knowledge-fs/scripts/api-image-bundle-smoke.mjs +++ b/knowledge-fs/scripts/api-image-bundle-smoke.mjs @@ -55,6 +55,8 @@ try { console.log( JSON.stringify({ compute: health.components.compute, + difyDependencyConnected: health.components.objectStorage, + healthOk: health.ok, imageTag, ok: true, port, @@ -98,7 +100,12 @@ async function waitForHealth(url) { const response = await fetch(url); const payload = await readBoundedJson(response); - if (response.status === 200 && payload.ok === true && payload.components?.compute === true) { + if ( + response.status === 200 && + payload.ok === false && + payload.components?.compute === true && + payload.components?.objectStorage === false + ) { return payload; } diff --git a/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs b/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs index d37614535bf..60d69c1db3f 100644 --- a/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs +++ b/knowledge-fs/scripts/api-image-bundle-smoke.test.mjs @@ -13,7 +13,10 @@ test("isolated API bundle smoke starts the container and checks compute health", assert.match(smokeScript, /127\.0\.0\.1::8787/); assert.match(smokeScript, /dockerPort/); assert.match(smokeScript, /\/health/); + assert.match(smokeScript, /payload\.ok === false/); assert.match(smokeScript, /components\?\.compute === true/); + assert.match(smokeScript, /components\?\.objectStorage === false/); + assert.match(smokeScript, /difyDependencyConnected/); assert.match(smokeScript, /productionConfigValidated: false/); assert.match(smokeScript, /scope: "isolated-bundle"/); assert.match(smokeScript, /dockerStop/); diff --git a/knowledge-fs/scripts/audit-backend-dependencies.mjs b/knowledge-fs/scripts/audit-backend-dependencies.mjs new file mode 100644 index 00000000000..6e1aff4983c --- /dev/null +++ b/knowledge-fs/scripts/audit-backend-dependencies.mjs @@ -0,0 +1,84 @@ +import { spawnSync } from "node:child_process"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const blockingSeverities = new Set(["high", "critical"]); +const adminWorkspacePrefix = "apps__admin"; + +export function collectBlockingBackendAdvisories(report) { + const advisories = + report && + typeof report === "object" && + report.advisories && + typeof report.advisories === "object" + ? Object.values(report.advisories) + : []; + + return advisories.flatMap((advisory) => { + if (!advisory || typeof advisory !== "object" || !blockingSeverities.has(advisory.severity)) { + return []; + } + + const findingsValid = + Array.isArray(advisory.findings) && + advisory.findings.length > 0 && + advisory.findings.every( + (finding) => + finding && + typeof finding === "object" && + Array.isArray(finding.paths) && + finding.paths.every((path) => typeof path === "string"), + ); + const paths = findingsValid + ? advisory.findings + .flatMap((finding) => finding.paths) + .filter( + (path) => path !== adminWorkspacePrefix && !path.startsWith(`${adminWorkspacePrefix}>`), + ) + : [""]; + if (paths.length === 0) return []; + + return [ + { + id: advisory.github_advisory_id ?? advisory.id, + module: advisory.module_name, + paths: [...new Set(paths)].sort(), + severity: advisory.severity, + title: advisory.title, + url: advisory.url, + }, + ]; + }); +} + +function runAudit() { + const audit = spawnSync("pnpm", ["audit", "--prod", "--json"], { + cwd: process.cwd(), + encoding: "utf8", + }); + if (audit.error) { + console.error(`Unable to run pnpm audit: ${audit.error.message}`); + return 2; + } + + let report; + try { + report = JSON.parse(audit.stdout); + } catch { + console.error(audit.stderr || audit.stdout || "pnpm audit returned invalid JSON"); + return 2; + } + + const blocking = collectBlockingBackendAdvisories(report); + if (blocking.length > 0) { + console.error(JSON.stringify({ advisories: blocking }, null, 2)); + return 1; + } + + console.log("No high or critical vulnerabilities found in backend production dependencies."); + return 0; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + process.exitCode = runAudit(); +} diff --git a/knowledge-fs/scripts/audit-backend-dependencies.test.mjs b/knowledge-fs/scripts/audit-backend-dependencies.test.mjs new file mode 100644 index 00000000000..cdc9acaaec8 --- /dev/null +++ b/knowledge-fs/scripts/audit-backend-dependencies.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { collectBlockingBackendAdvisories } from "./audit-backend-dependencies.mjs"; + +test("ignores Admin-only findings while retaining backend high and critical findings", () => { + const result = collectBlockingBackendAdvisories({ + advisories: { + admin: { + findings: [{ paths: ["apps__admin>next"] }], + severity: "high", + }, + backend: { + findings: [ + { + paths: [ + "apps__admin>shared-package", + "apps__api>shared-package", + "packages__api>shared-package", + ], + }, + ], + github_advisory_id: "GHSA-backend", + module_name: "shared-package", + severity: "critical", + title: "Backend issue", + url: "https://example.test/GHSA-backend", + }, + moderate: { + findings: [{ paths: ["apps__api>moderate-package"] }], + severity: "moderate", + }, + }, + }); + + assert.deepEqual(result, [ + { + id: "GHSA-backend", + module: "shared-package", + paths: ["apps__api>shared-package", "packages__api>shared-package"], + severity: "critical", + title: "Backend issue", + url: "https://example.test/GHSA-backend", + }, + ]); +}); + +test("fails closed for malformed blocking advisories", () => { + assert.deepEqual(collectBlockingBackendAdvisories(undefined), []); + assert.deepEqual(collectBlockingBackendAdvisories({ advisories: [] }), []); + assert.deepEqual( + collectBlockingBackendAdvisories({ + advisories: { + malformed: { + findings: "invalid", + github_advisory_id: "GHSA-malformed", + module_name: "unknown", + severity: "high", + title: "Malformed issue", + url: "https://example.test/GHSA-malformed", + }, + }, + }), + [ + { + id: "GHSA-malformed", + module: "unknown", + paths: [""], + severity: "high", + title: "Malformed issue", + url: "https://example.test/GHSA-malformed", + }, + ], + ); +}); diff --git a/knowledge-fs/scripts/compose-apps.test.mjs b/knowledge-fs/scripts/compose-apps.test.mjs index 0e571115e85..0a2c905c84e 100644 --- a/knowledge-fs/scripts/compose-apps.test.mjs +++ b/knowledge-fs/scripts/compose-apps.test.mjs @@ -36,6 +36,13 @@ function serviceBlock(source, serviceName) { return lines.slice(start, end).join("\n"); } +function envVariableNames(source) { + return source + .split("\n") + .filter((line) => /^[A-Z][A-Z0-9_]*=/.test(line)) + .map((line) => line.slice(0, line.indexOf("="))); +} + test("deployment Compose and Kubernetes artifacts are valid YAML", () => { for (const source of [compose, ...difyComposeFiles]) { const document = parse(source); @@ -87,31 +94,30 @@ test("app compose profile builds the API image with development-only static auth compose, /^ {6}KNOWLEDGE_DEV_AUTH_TOKEN: \$\{KNOWLEDGE_DEV_AUTH_TOKEN:-dev-token\}$/m, ); - assert.match( - compose, - /^ {6}KNOWLEDGE_EMBEDDING_PROVIDER: \$\{KNOWLEDGE_EMBEDDING_PROVIDER:-\}$/m, - ); - assert.match(compose, /^ {6}KNOWLEDGE_EMBEDDING_MODEL: \$\{KNOWLEDGE_EMBEDDING_MODEL:-\}$/m); - assert.match(compose, /^ {6}OPENAI_EMBEDDING_BASE_URL: \$\{OPENAI_EMBEDDING_BASE_URL:-\}$/m); + assert.doesNotMatch(compose, /^ {6}KNOWLEDGE_(?:EMBEDDING|RERANK|ANSWER)_/m); }); -test("app compose profile waits for durable middleware readiness before API startup", () => { +test("app compose profile waits for local database and parser readiness before API startup", () => { assert.match(compose, /^ {4}depends_on:$/m); assert.match(compose, /^ {6}postgres:$/m); assert.match(compose, /^ {8}condition: service_healthy$/m); - assert.match(compose, /^ {6}minio-bootstrap:$/m); - assert.match(compose, /^ {8}condition: service_completed_successfully$/m); assert.match(compose, /^ {6}unstructured:$/m); assert.match(compose, /^ {8}condition: service_started$/m); + assert.doesNotMatch(compose, /^ {2}minio(?:-bootstrap)?:$/m); }); -test("app compose profile uses service-local middleware URLs inside the API container", () => { +test("app compose profile uses local middleware and the required Dify dependency", () => { assert.match( compose, /^ {6}DATABASE_URL: postgresql:\/\/\$\{POSTGRES_USER:-knowledge_fs\}:\$\{POSTGRES_PASSWORD:-knowledge_fs\}@postgres:5432\/\$\{POSTGRES_DB:-knowledge_fs\}$/m, ); - assert.match(compose, /^ {6}MINIO_ENDPOINT: http:\/\/minio:9000$/m); assert.match(compose, /^ {6}UNSTRUCTURED_API_URL: http:\/\/unstructured:8000$/m); + assert.match( + compose, + /^ {6}DIFY_INNER_API_URL: \$\{DIFY_INNER_API_URL:-http:\/\/host\.docker\.internal:5001\}$/m, + ); + assert.match(compose, /^ {6}DIFY_INNER_API_KEY: \$\{DIFY_INNER_API_KEY:-\}$/m); + assert.doesNotMatch(compose, /^ {6}(?:MINIO|R2|OPENAI|ANTHROPIC|COHERE|GEMINI|VOYAGE)_/m); }); test("app compose profile builds Admin as a production image after API readiness", () => { @@ -131,10 +137,14 @@ test("app compose profile builds Admin as a production image after API readiness assert.doesNotMatch(compose, /^ {2}pnpm-store:$/m); }); -test("Dify compose keeps the integrated KnowledgeFS API internal and disabled by profile", () => { +test("Dify compose starts the integrated KnowledgeFS API by default and keeps it internal", () => { for (const difyCompose of difyComposeFiles) { const knowledgeFs = serviceBlock(difyCompose, "knowledge_fs"); - assert.match(knowledgeFs, /^ {4}profiles: \["knowledge-fs"\]$/m); + assert.doesNotMatch(knowledgeFs, /^ {4}profiles:/m); + assert.match( + knowledgeFs, + /^ {4}image: \$\{KNOWLEDGE_FS_API_IMAGE:-langgenius\/dify-knowledge-fs-api:deploy-konwledge\}$/m, + ); assert.match(knowledgeFs, /^ {6}context: \.\.\/knowledge-fs$/m); assert.match(knowledgeFs, /^ {6}dockerfile: apps\/api\/Dockerfile$/m); assert.match(knowledgeFs, /^ {4}expose:$/m); @@ -144,6 +154,11 @@ test("Dify compose keeps the integrated KnowledgeFS API internal and disabled by knowledgeFs, /^ {6}KNOWLEDGE_INTEGRATED_MODE_ENABLED: \$\{KNOWLEDGE_INTEGRATED_MODE_ENABLED:-true\}$/m, ); + assert.match( + knowledgeFs, + /^ {6}DIFY_INNER_API_URL: \$\{PLUGIN_DIFY_INNER_API_URL:-http:\/\/api:5001\}$/m, + ); + assert.match(knowledgeFs, /^ {6}DIFY_INNER_API_KEY: \$\{PLUGIN_DIFY_INNER_API_KEY:-.+\}$/m); assert.doesNotMatch(knowledgeFs, /^ {6}PLUGIN_DAEMON_(?:URL|KEY):/m); assert.doesNotMatch(knowledgeFs, /^ {6}plugin_daemon:$/m); assert.match(knowledgeFs, /http:\/\/127\.0\.0\.1:8787\/ready/); @@ -152,7 +167,21 @@ test("Dify compose keeps the integrated KnowledgeFS API internal and disabled by assert.match(difyApiEnv, /^KNOWLEDGE_FS_ENABLED=\$\{KNOWLEDGE_FS_ENABLED:-false\}$/m); }); -test("deployment examples expose every guarded KnowledgeFS rollout capability as disabled", () => { +test("KnowledgeFS deployment env contains only operator-owned runtime inputs", () => { + assert.deepEqual(envVariableNames(difyKnowledgeFsEnv), [ + "DATABASE_URL", + "KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME", + "KNOWLEDGE_FS_CAPABILITY_V2_ENABLED", + "KNOWLEDGE_FS_CAPABILITY_V2_PUBLIC_JWKS", + "UNSTRUCTURED_API_URL", + "UNSTRUCTURED_API_KEY", + ]); + assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME=on$/m); + assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false$/m); + assert.doesNotMatch(difyKnowledgeFsEnv, /^MINIO_/m); +}); + +test("deployment examples keep Dify KnowledgeFS rollout capabilities disabled", () => { for (const variable of [ "KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED", "KNOWLEDGE_FS_INTEGRATED_PROVISION_READY", @@ -160,17 +189,6 @@ test("deployment examples expose every guarded KnowledgeFS rollout capability as ]) { assert.match(difyApiEnv, new RegExp(`^${variable}=false$`, "m")); } - for (const variable of [ - "KNOWLEDGE_FS_CAPABILITY_V2_ENABLED", - "KNOWLEDGE_INTEGRATED_MODE_ENABLED", - "KNOWLEDGE_LEGACY_ACL_READ_ONLY", - "KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED", - ]) { - assert.match(difyKnowledgeFsEnv, new RegExp(`^${variable}=false$`, "m")); - } - assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_DIRECT_UPLOAD_ENABLED=off$/m); - assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_DIRECT_UPLOAD_ALLOWED_ORIGINS=$/m); - assert.match(difyKnowledgeFsEnv, /^KNOWLEDGE_DIRECT_STREAM_ENABLED=off$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_INTEGRATED_MODE_ENABLED: "false"$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_LEGACY_AUTHORIZATION_REMOVED: "false"$/m); assert.match(kubernetesBaseline, /^ {2}KNOWLEDGE_DIRECT_UPLOAD_ENABLED: "off"$/m); diff --git a/knowledge-fs/scripts/compose-middleware.test.mjs b/knowledge-fs/scripts/compose-middleware.test.mjs index 67f8fdb9d4d..89c7945763c 100644 --- a/knowledge-fs/scripts/compose-middleware.test.mjs +++ b/knowledge-fs/scripts/compose-middleware.test.mjs @@ -10,16 +10,15 @@ const compose = readFileSync( test("middleware compose file contains only local middleware services", () => { assert.match(compose, /^services:$/m); assert.match(compose, /^ {2}postgres:$/m); - assert.match(compose, /^ {2}minio:$/m); - assert.match(compose, /^ {2}minio-bootstrap:$/m); assert.match(compose, /^ {2}unstructured:$/m); + assert.doesNotMatch(compose, /^ {2}minio(?:-bootstrap)?:$/m); assert.doesNotMatch(compose, /^ {2}api:$/m); assert.doesNotMatch(compose, /^ {2}admin:$/m); }); -test("middleware compose keeps bounded local storage volumes", () => { +test("middleware compose keeps only the local database volume", () => { assert.match(compose, /^volumes:$/m); assert.match(compose, /^ {2}postgres-data:$/m); - assert.match(compose, /^ {2}minio-data:$/m); + assert.doesNotMatch(compose, /^ {2}minio-data:$/m); assert.doesNotMatch(compose, /^ {2}pnpm-store:$/m); }); diff --git a/knowledge-fs/scripts/dify-compose-config.mjs b/knowledge-fs/scripts/dify-compose-config.mjs index 8282f739e98..7fd67ddb669 100644 --- a/knowledge-fs/scripts/dify-compose-config.mjs +++ b/knowledge-fs/scripts/dify-compose-config.mjs @@ -40,8 +40,6 @@ function runDifyComposeConfig() { examplePath, "-f", composePath, - "--profile", - "knowledge-fs", "config", "--quiet", ], diff --git a/knowledge-fs/scripts/github-actions-workflow.test.mjs b/knowledge-fs/scripts/github-actions-workflow.test.mjs index 3627f6c969f..2849bc41e53 100644 --- a/knowledge-fs/scripts/github-actions-workflow.test.mjs +++ b/knowledge-fs/scripts/github-actions-workflow.test.mjs @@ -124,7 +124,7 @@ test("root workflow runs explicit local security gates", () => { assert.equal(packageJson.scripts["security:secrets"], "node scripts/secret-scan.mjs"); assert.equal( packageJson.scripts["security:dependencies"], - "pnpm audit --prod --audit-level high", + "node scripts/audit-backend-dependencies.mjs", ); assert.match(packageJson.scripts["ci:workflow:test"], /scripts\/secret-scan\.test\.mjs/); }); diff --git a/knowledge-fs/scripts/local-happy-path-smoke.mjs b/knowledge-fs/scripts/local-happy-path-smoke.mjs index cd62d9e227e..6940d91992d 100644 --- a/knowledge-fs/scripts/local-happy-path-smoke.mjs +++ b/knowledge-fs/scripts/local-happy-path-smoke.mjs @@ -289,13 +289,7 @@ function assertContains(value, expected, label) { } function assertDurableEnvironment() { - for (const name of [ - "DATABASE_URL", - "MINIO_ACCESS_KEY", - "MINIO_BUCKET", - "MINIO_ENDPOINT", - "MINIO_SECRET_KEY", - ]) { + for (const name of ["DATABASE_URL", "DIFY_INNER_API_KEY", "DIFY_INNER_API_URL"]) { if (!process.env[name]?.trim()) { throw new Error(`LOCAL_SMOKE_EXPECT_DURABLE requires ${name}`); } diff --git a/knowledge-fs/scripts/local-happy-path-smoke.test.mjs b/knowledge-fs/scripts/local-happy-path-smoke.test.mjs index edd41b66e3d..86a55f236c0 100644 --- a/knowledge-fs/scripts/local-happy-path-smoke.test.mjs +++ b/knowledge-fs/scripts/local-happy-path-smoke.test.mjs @@ -29,8 +29,9 @@ test("local happy-path smoke covers the core API endpoints without manual DB edi assert.match(smokeScript, /LOCAL_SMOKE_RUN_MIGRATIONS/); assert.match(smokeScript, /LOCAL_SMOKE_EXPECT_DURABLE/); assert.match(smokeScript, /DATABASE_URL/); - assert.match(smokeScript, /MINIO_ENDPOINT/); - assert.match(smokeScript, /MINIO_BUCKET/); + assert.match(smokeScript, /DIFY_INNER_API_URL/); + assert.match(smokeScript, /DIFY_INNER_API_KEY/); + assert.doesNotMatch(smokeScript, /MINIO_/); assert.match(smokeScript, /local:db:migrate/); assert.match(smokeScript, /\/knowledge-spaces\?limit=100/); assert.match(smokeScript, /\/knowledge-spaces/); diff --git a/knowledge-fs/test/setup-dify-object-storage.ts b/knowledge-fs/test/setup-dify-object-storage.ts new file mode 100644 index 00000000000..cd8fa8cf404 --- /dev/null +++ b/knowledge-fs/test/setup-dify-object-storage.ts @@ -0,0 +1,124 @@ +import { beforeEach } from "vitest"; + +interface StoredObject { + readonly body: Uint8Array; + readonly checksumSha256Base64?: string | undefined; + readonly contentType?: string | undefined; + readonly metadata: Readonly>; +} + +const objects = new Map(); +const metadataHeader = "X-Knowledge-FS-Metadata"; +const checksumHeader = "X-Knowledge-FS-Checksum-Sha256"; +const contentTypeHeader = "X-Knowledge-FS-Content-Type"; + +const difyObjectStorageFetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (!url.pathname.startsWith("/inner/api/knowledge-fs/storage/")) { + throw new Error(`Unexpected test fetch request: ${request.method} ${url.href}`); + } + + if (url.pathname === "/inner/api/knowledge-fs/storage/health") { + return Response.json({ ok: true }); + } + + if (url.pathname === "/inner/api/knowledge-fs/storage/objects") { + const prefix = url.searchParams.get("prefix") ?? ""; + const cursor = url.searchParams.get("cursor"); + const limit = Number.parseInt(url.searchParams.get("limit") ?? "100", 10); + const keys = [...objects.keys()] + .filter((key) => key.startsWith(prefix) && (!cursor || key > cursor)) + .sort(); + const selected = keys.slice(0, limit); + const hasMore = keys.length > selected.length; + return Response.json({ + ...(hasMore && selected.length > 0 ? { nextCursor: selected.at(-1) } : {}), + objects: selected.map((key) => metadata(key, requiredObject(key))), + }); + } + + const key = url.searchParams.get("key"); + if (!key) return Response.json({ error: "missing key" }, { status: 400 }); + + if (url.pathname === "/inner/api/knowledge-fs/storage/object/metadata") { + const stored = objects.get(key); + return stored + ? Response.json(metadata(key, stored)) + : Response.json({ error: "not found" }, { status: 404 }); + } + + if (url.pathname !== "/inner/api/knowledge-fs/storage/object") { + return Response.json({ error: "not found" }, { status: 404 }); + } + + if (request.method === "DELETE") { + objects.delete(key); + return new Response(null, { status: 204 }); + } + if (request.method === "GET") { + const stored = objects.get(key); + return stored + ? new Response(stored.body.slice(), { + headers: stored.contentType ? { "content-type": stored.contentType } : {}, + }) + : Response.json({ error: "not found" }, { status: 404 }); + } + if (request.method === "PUT") { + const body = new Uint8Array(await request.arrayBuffer()); + const stored: StoredObject = { + body, + metadata: decodeMetadata(request.headers.get(metadataHeader)), + ...(request.headers.get(checksumHeader) + ? { checksumSha256Base64: request.headers.get(checksumHeader) ?? undefined } + : {}), + ...(request.headers.get(contentTypeHeader) + ? { contentType: request.headers.get(contentTypeHeader) ?? undefined } + : {}), + }; + objects.set(key, stored); + return Response.json(metadata(key, stored)); + } + + return Response.json({ error: "method not allowed" }, { status: 405 }); +}; + +globalThis.fetch = difyObjectStorageFetch; + +beforeEach(() => { + objects.clear(); + globalThis.fetch = difyObjectStorageFetch; +}); + +function requiredObject(key: string): StoredObject { + const stored = objects.get(key); + if (!stored) throw new Error(`Missing test object ${key}`); + return stored; +} + +function metadata(key: string, stored: StoredObject) { + return { + ...(stored.checksumSha256Base64 + ? { checksumSha256Base64: stored.checksumSha256Base64 } + : {}), + ...(stored.contentType ? { contentType: stored.contentType } : {}), + key, + metadata: stored.metadata, + sizeBytes: stored.body.byteLength, + }; +} + +function decodeMetadata(value: string | null): Readonly> { + if (!value) return {}; + const decoded = Buffer.from(value, "base64url").toString("utf8"); + const parsed: unknown = JSON.parse(decoded); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Invalid test object metadata"); + } + return Object.fromEntries( + Object.entries(parsed).map(([key, item]) => { + if (typeof item !== "string") throw new Error("Invalid test object metadata"); + return [key, item]; + }), + ); +}