mirror of
https://github.com/langgenius/dify.git
synced 2026-07-24 13:08:34 +08:00
feat(dataset): expose New RAG KnowledgeFS contracts (#39314)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stephen Zhou
GitHub
parent
018fe7a9d3
commit
bd8f0104d6
@ -8,8 +8,9 @@ can be validated explicitly against the pinned KnowledgeFS contract during devel
|
||||
Console auth and contract-specific dataset RBAC run before forwarding. Request
|
||||
bodies are capped at 64 MiB, JSON and binary responses have separate bounds,
|
||||
SSE responses remain streaming with a bounded idle read timeout, and only safe
|
||||
response headers are exposed. Upstream 401 responses become 502 so they cannot
|
||||
trigger Dify browser-session recovery; resource-level 403 responses remain 403.
|
||||
response headers are exposed. Operation-specific upstream error mappings are
|
||||
applied before Console JSON error handling; the default maps 401 to 502 so it
|
||||
cannot trigger browser-session recovery and preserves resource-level 403.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@ -27,9 +28,11 @@ from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
GatewayTimeout,
|
||||
HTTPException,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
default_exceptions,
|
||||
)
|
||||
|
||||
from configs import dify_config
|
||||
@ -41,21 +44,28 @@ from controllers.console.wraps import (
|
||||
)
|
||||
from core.helper import ssrf_proxy
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from services.knowledge_fs_operations import KnowledgeFSMethod
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSAuthorization,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_authorized_knowledge_fs_request,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
type _KnowledgeFSRequestForwarder = Callable[
|
||||
[str | None, str | None, bytes | None, bytes | None],
|
||||
KnowledgeFSUpstreamResponse,
|
||||
]
|
||||
|
||||
_MAX_PROXY_BODY_BYTES = 64 * 1024 * 1024
|
||||
_RESPONSE_HEADER_ALLOWLIST = (
|
||||
"Cache-Control",
|
||||
@ -128,27 +138,25 @@ def _translate_proxy_error(exc: Exception, *, tenant_id: str) -> NoReturn:
|
||||
|
||||
|
||||
def _knowledge_fs_operation_access_required(
|
||||
view: Callable[[KnowledgeFSMethod, str], ResponseReturnValue],
|
||||
view: Callable[[KnowledgeFSAuthorization], ResponseReturnValue],
|
||||
) -> Callable[[KnowledgeFSMethod, str], ResponseReturnValue]:
|
||||
"""Authorize one declared operation before billing and request-body work."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(method: KnowledgeFSMethod, upstream_path: str) -> ResponseReturnValue:
|
||||
try:
|
||||
operation = get_knowledge_fs_operation(method, upstream_path)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
authorize_knowledge_fs_request(
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=current_user,
|
||||
tenant_id=tenant_id,
|
||||
operation=operation,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
except KnowledgeFSAccessDeniedError as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return view(method, upstream_path)
|
||||
return view(authorization)
|
||||
|
||||
return decorated
|
||||
|
||||
@ -190,21 +198,26 @@ def _proxy_response(
|
||||
"""Expose raw content, status, and allowlisted headers from KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
BadGateway: KnowledgeFS rejects the configured server credential.
|
||||
Forbidden: KnowledgeFS denies the account access to the requested resource.
|
||||
HTTPException: KnowledgeFS returns a status normalized by the operation contract.
|
||||
"""
|
||||
upstream = upstream_result.response
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
mapped_status = dict(upstream_result.operation.error_status_map).get(upstream.status_code)
|
||||
if mapped_status is not None:
|
||||
upstream.close()
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
raise BadGateway("KnowledgeFS authentication failed")
|
||||
if upstream.status_code == HTTPStatus.FORBIDDEN:
|
||||
upstream.close()
|
||||
raise Forbidden()
|
||||
description = "KnowledgeFS upstream request failed"
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
description = "KnowledgeFS authentication failed"
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
exception_type = default_exceptions.get(mapped_status)
|
||||
if exception_type is None:
|
||||
exception = HTTPException(description)
|
||||
exception.code = mapped_status
|
||||
raise exception
|
||||
raise exception_type(description)
|
||||
|
||||
allowed_header_names = dict.fromkeys(
|
||||
name.lower() for name in (*_RESPONSE_HEADER_ALLOWLIST, *contract_response_headers)
|
||||
@ -237,26 +250,21 @@ def _proxy_response(
|
||||
return Response(content, status=upstream.status_code, headers=headers)
|
||||
|
||||
|
||||
def _proxy_request(method: KnowledgeFSMethod, upstream_path: str) -> Response:
|
||||
"""Forward the current raw request and return its filtered upstream response.
|
||||
|
||||
The call performs one outbound KnowledgeFS request. Integration failures are
|
||||
converted to Console HTTP exceptions for the outer JSON error adapter.
|
||||
"""
|
||||
def _proxy_current_request(
|
||||
*,
|
||||
method: KnowledgeFSMethod,
|
||||
tenant_id: str,
|
||||
forward: _KnowledgeFSRequestForwarder,
|
||||
) -> Response:
|
||||
"""Forward the current raw request through one preconfigured service entry."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
proxy_result = proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=request.headers.get("Accept"),
|
||||
content_type=request.content_type,
|
||||
query=request.query_string or None,
|
||||
body=_request_body() if method != "GET" else None,
|
||||
request_headers=request.headers,
|
||||
proxy_result = forward(
|
||||
request.headers.get("Accept"),
|
||||
request.content_type,
|
||||
request.query_string or None,
|
||||
_request_body() if method != "GET" else None,
|
||||
)
|
||||
except (
|
||||
KnowledgeFSConfigurationError,
|
||||
@ -274,15 +282,77 @@ def _proxy_request(method: KnowledgeFSMethod, upstream_path: str) -> Response:
|
||||
)
|
||||
|
||||
|
||||
def _proxy_request(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
) -> Response:
|
||||
"""Authorize and forward the current request through the combined service use case."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
def _proxy_authorized_request(authorization: KnowledgeFSAuthorization) -> Response:
|
||||
"""Forward the current request using one previously authorized operation capability.
|
||||
|
||||
Args:
|
||||
authorization: Request-scoped capability produced before billing and body parsing.
|
||||
|
||||
Returns:
|
||||
The filtered response returned by KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
HTTPException: The integration is disabled or forwarding fails.
|
||||
"""
|
||||
operation = authorization.operation
|
||||
tenant_id = authorization.tenant_id
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_authorized_knowledge_fs_request(
|
||||
authorization=authorization,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=operation.method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
@_knowledge_fs_enabled
|
||||
@_knowledge_fs_operation_access_required
|
||||
@cloud_edition_billing_rate_limit_check("knowledge")
|
||||
def _proxy_knowledge_fs_non_get(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
authorization: KnowledgeFSAuthorization,
|
||||
) -> ResponseReturnValue:
|
||||
"""Apply knowledge billing checks to one allowlisted non-GET operation."""
|
||||
return _proxy_request(method, upstream_path)
|
||||
return _proxy_authorized_request(authorization)
|
||||
|
||||
|
||||
@bp.route(
|
||||
|
||||
@ -12,6 +12,7 @@ import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
@ -24,6 +25,16 @@ LOCK_PATH = API_ROOT / "knowledge-fs-contract.lock.json"
|
||||
DEFAULT_REPOSITORY = WORKSPACE_ROOT.parent / "knowledge-fs"
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
CONSOLE_PROXY_ERROR_SCHEMA_NAME = "ConsoleProxyError"
|
||||
CONSOLE_PROXY_ERROR_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"required": ["code", "message", "status"],
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"status": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
@ -38,6 +49,7 @@ class ContractDeclaration(TypedDict):
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
error_status_map: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
@ -106,8 +118,11 @@ def main() -> None:
|
||||
validate_declarations(document, declarations)
|
||||
|
||||
if args.output_openapi:
|
||||
filtered_document = filter_openapi_document(document, declarations)
|
||||
filtered_document["x-dify-source-openapi-sha256"] = openapi_sha256
|
||||
filtered_document["x-dify-console-declarations-sha256"] = contract_declarations_sha256(declarations)
|
||||
args.output_openapi.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_openapi.write_text(json.dumps(filter_openapi_document(document, declarations), indent=2) + "\n")
|
||||
args.output_openapi.write_text(json.dumps(filtered_document, indent=2) + "\n")
|
||||
|
||||
if args.update_lock:
|
||||
LOCK_PATH.write_text(
|
||||
@ -153,8 +168,7 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
|
||||
if method not in PROXY_METHODS:
|
||||
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
|
||||
expected: ContractDeclaration = {
|
||||
"operation_id": operation_id,
|
||||
expected: dict[DeclarationField, object] = {
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(operation),
|
||||
@ -172,6 +186,7 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
f"KnowledgeFS operation {operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}"
|
||||
)
|
||||
validate_error_status_map(operation_id, declaration["error_status_map"])
|
||||
|
||||
|
||||
def filter_openapi_document(
|
||||
@ -191,19 +206,65 @@ def filter_openapi_document(
|
||||
source_path_item = source_paths[path]
|
||||
path_metadata = {key: value for key, value in source_path_item.items() if key not in OPENAPI_METHODS}
|
||||
filtered_path_item = filtered_paths.setdefault(path, path_metadata)
|
||||
filtered_path_item[method] = source_path_item[method]
|
||||
filtered_operation = deepcopy(source_path_item[method])
|
||||
_rewrite_proxy_error_responses(filtered_operation, declaration["error_status_map"])
|
||||
filtered_path_item[method] = filtered_operation
|
||||
|
||||
filtered_document["paths"] = filtered_paths
|
||||
|
||||
source_components = document.get("components", {})
|
||||
filtered_components = {key: value for key, value in source_components.items() if key != "schemas"}
|
||||
source_schemas = source_components.get("schemas", {})
|
||||
schema_names = _referenced_schema_names(filtered_paths, source_schemas)
|
||||
filtered_components["schemas"] = {name: schema for name, schema in source_schemas.items() if name in schema_names}
|
||||
available_schemas = {**source_schemas, CONSOLE_PROXY_ERROR_SCHEMA_NAME: CONSOLE_PROXY_ERROR_SCHEMA}
|
||||
schema_names = _referenced_schema_names(filtered_paths, available_schemas)
|
||||
filtered_components["schemas"] = {
|
||||
name: schema for name, schema in available_schemas.items() if name in schema_names
|
||||
}
|
||||
filtered_document["components"] = filtered_components
|
||||
return filtered_document
|
||||
|
||||
|
||||
def validate_error_status_map(operation_id: str, error_status_map: tuple[tuple[int, int], ...]) -> None:
|
||||
"""Validate the status normalization advertised by one Console operation."""
|
||||
upstream_statuses: set[int] = set()
|
||||
for upstream_status, console_status in error_status_map:
|
||||
if upstream_status in upstream_statuses:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has duplicate error status: {upstream_status}")
|
||||
if not 400 <= upstream_status <= 599 or not 400 <= console_status <= 599:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has invalid error status mapping")
|
||||
upstream_statuses.add(upstream_status)
|
||||
|
||||
|
||||
def _rewrite_proxy_error_responses(
|
||||
operation: dict[str, Any],
|
||||
error_status_map: tuple[tuple[int, int], ...],
|
||||
) -> None:
|
||||
responses = operation.setdefault("responses", {})
|
||||
proxy_error_response = {
|
||||
"description": "Error normalized by the Dify Console KnowledgeFS proxy.",
|
||||
"content": {
|
||||
"application/json": {"schema": {"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"}}
|
||||
},
|
||||
}
|
||||
for upstream_status, console_status in error_status_map:
|
||||
existing_target = responses.get(str(console_status)) if upstream_status != console_status else None
|
||||
responses.pop(str(upstream_status), None)
|
||||
normalized_response: dict[str, Any] = deepcopy(proxy_error_response)
|
||||
existing_schema = (
|
||||
existing_target.get("content", {}).get("application/json", {}).get("schema")
|
||||
if isinstance(existing_target, dict)
|
||||
else None
|
||||
)
|
||||
if existing_schema is not None:
|
||||
normalized_response["content"]["application/json"]["schema"] = {
|
||||
"oneOf": [
|
||||
deepcopy(existing_schema),
|
||||
{"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"},
|
||||
]
|
||||
}
|
||||
responses[str(console_status)] = normalized_response
|
||||
|
||||
|
||||
def _referenced_schema_names(value: Any, schemas: dict[str, Any]) -> set[str]:
|
||||
reference_prefix = "#/components/schemas/"
|
||||
selected: set[str] = set()
|
||||
@ -232,7 +293,7 @@ def _referenced_schema_names(value: Any, schemas: dict[str, Any]) -> set[str]:
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""Return transport declarations from the runtime Console operation registry."""
|
||||
from services.knowledge_fs_proxy import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
from services.knowledge_fs_operations import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
{
|
||||
@ -245,11 +306,18 @@ def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
"error_status_map": operation.error_status_map,
|
||||
}
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
)
|
||||
|
||||
|
||||
def contract_declarations_sha256(declarations: tuple[ContractDeclaration, ...]) -> str:
|
||||
"""Return a stable digest for the runtime Console operation declarations."""
|
||||
content = json.dumps(declarations, separators=(",", ":"), sort_keys=True).encode()
|
||||
return sha256(content)
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
media_types = response_media_types(operation)
|
||||
if "text/event-stream" in media_types:
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"commit": "4310e2d582d25e7de58183f27720afab01e123cf",
|
||||
"openapiSha256": "5827ca930ce38462bfd1b2bef387efbf37eb7ffcaedde4558af2fbaeccbfbc4b",
|
||||
"commit": "a0f50470612cc0b3656f89e4f2435aaf412e6e3b",
|
||||
"openapiSha256": "f18910e9c45a64f0855e0643a7a626fb2889021b4f943458de86c6bd2469facb",
|
||||
"repository": "https://github.com/langgenius/knowledge-fs"
|
||||
}
|
||||
|
||||
502
api/services/knowledge_fs_operations.py
Normal file
502
api/services/knowledge_fs_operations.py
Normal file
@ -0,0 +1,502 @@
|
||||
"""Product-facing KnowledgeFS operation and authorization declarations.
|
||||
|
||||
This registry is Dify's explicit Console surface. Transport concerns live in
|
||||
knowledge_fs_proxy so contract review does not require reading proxy mechanics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final, Literal, NamedTuple
|
||||
|
||||
from core.rbac import RBACPermission
|
||||
|
||||
type KnowledgeFSMethod = Literal["DELETE", "GET", "PATCH", "POST", "PUT"]
|
||||
type KnowledgeFSResponseKind = Literal["binary", "buffered", "stream"]
|
||||
type KnowledgeFSRequiredScope = Literal["knowledge-spaces:read", "knowledge-spaces:write"]
|
||||
type KnowledgeFSLegacyRole = Literal["reader", "dataset_editor", "admin"]
|
||||
type KnowledgeFSErrorStatusMap = tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
class KnowledgeFSOperation(NamedTuple):
|
||||
operation_id: str
|
||||
method: KnowledgeFSMethod
|
||||
path: str
|
||||
response_kind: KnowledgeFSResponseKind
|
||||
required_scope: KnowledgeFSRequiredScope
|
||||
rbac_permission: RBACPermission
|
||||
legacy_role: KnowledgeFSLegacyRole
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
error_status_map: KnowledgeFSErrorStatusMap
|
||||
|
||||
|
||||
def _console_operation(
|
||||
operation_id: str,
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
*,
|
||||
rbac_permission: RBACPermission,
|
||||
legacy_role: KnowledgeFSLegacyRole,
|
||||
max_response_bytes: int = 1_048_576,
|
||||
request_headers: tuple[str, ...] = ("x-trace-id",),
|
||||
response_kind: KnowledgeFSResponseKind = "buffered",
|
||||
response_media_types: tuple[str, ...] = ("application/json",),
|
||||
error_status_map: KnowledgeFSErrorStatusMap = ((401, 502), (403, 403)),
|
||||
) -> KnowledgeFSOperation:
|
||||
"""Declare one contract-pinned operation with an explicit Dify authorization policy."""
|
||||
is_read = method == "GET"
|
||||
return KnowledgeFSOperation(
|
||||
operation_id=operation_id,
|
||||
method=method,
|
||||
path=path,
|
||||
response_kind=response_kind,
|
||||
required_scope="knowledge-spaces:read" if is_read else "knowledge-spaces:write",
|
||||
rbac_permission=rbac_permission,
|
||||
legacy_role=legacy_role,
|
||||
max_response_bytes=max_response_bytes,
|
||||
request_headers=request_headers,
|
||||
response_headers=("x-trace-id",),
|
||||
response_media_types=response_media_types,
|
||||
error_status_map=error_status_map,
|
||||
)
|
||||
|
||||
|
||||
def _dataset_read_operation(operation_id: str, path: str) -> KnowledgeFSOperation:
|
||||
"""Declare a dataset-readable buffered JSON operation."""
|
||||
return _console_operation(
|
||||
operation_id,
|
||||
"GET",
|
||||
path,
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
)
|
||||
|
||||
|
||||
def _dataset_edit_operation(
|
||||
operation_id: str,
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
*,
|
||||
request_headers: tuple[str, ...] = ("x-trace-id",),
|
||||
) -> KnowledgeFSOperation:
|
||||
"""Declare a dataset-editable buffered JSON operation."""
|
||||
return _console_operation(
|
||||
operation_id,
|
||||
method,
|
||||
path,
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
def _external_source_operation(
|
||||
operation_id: str,
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
*,
|
||||
request_headers: tuple[str, ...] = ("x-trace-id",),
|
||||
) -> KnowledgeFSOperation:
|
||||
"""Declare a source-connection operation restricted to dataset editors."""
|
||||
return _console_operation(
|
||||
operation_id,
|
||||
method,
|
||||
path,
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS: Final[tuple[KnowledgeFSOperation, ...]] = (
|
||||
_console_operation(
|
||||
operation_id="listKnowledgeSpaces",
|
||||
method="GET",
|
||||
path="knowledge-spaces",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="createKnowledgeSpace",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
rbac_permission=RBACPermission.DATASET_CREATE_AND_MANAGEMENT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesById",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_edit_operation("patchKnowledgeSpacesById", "PATCH", "knowledge-spaces/{id}"),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesById",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_read_operation("getKnowledgeSpacesByIdStats", "knowledge-spaces/{id}/stats"),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdAccessPolicy",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/access-policy",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="patchKnowledgeSpacesByIdAccessPolicy",
|
||||
method="PATCH",
|
||||
path="knowledge-spaces/{id}/access-policy",
|
||||
rbac_permission=RBACPermission.DATASET_ACCESS_CONFIG,
|
||||
legacy_role="admin",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getSourceProviders",
|
||||
method="GET",
|
||||
path="source-providers",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourceConnections",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/source-connections",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceConnections",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-connections",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourceConnectionsOauth",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/source-connections/oauth",
|
||||
),
|
||||
_external_source_operation("postSourceOauthCallback", "POST", "source-oauth/callback"),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/source-connections/{connectionId}",
|
||||
),
|
||||
_external_source_operation(
|
||||
"deleteKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/source-connections/{connectionId}",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-connections/{connectionId}/refresh",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSources",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/sources",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSources",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/sources",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}",
|
||||
),
|
||||
_external_source_operation(
|
||||
"patchKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"PATCH",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}",
|
||||
),
|
||||
_external_source_operation(
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"PUT",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/credentials",
|
||||
),
|
||||
_external_source_operation(
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/credentials",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdSync",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/sync",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/sources/{sourceId}/crawl-preview",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdWorkflowImports",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/workflow-imports",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdPages",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/pages",
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdFiles",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/files",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawl",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/crawl",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImport",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/import",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdTest",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/test",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImportFiles",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/import-files",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBulk",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/bulk",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourceWorkflows",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/source-workflows",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourceWorkflowsByRunId",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdBulkItems",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/source-workflows/{runId}/bulk-items",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/pages",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/cancel",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/retry",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/selection",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
method="PUT",
|
||||
path="knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_dataset_read_operation("getKnowledgeSpacesByIdDocuments", "knowledge-spaces/{id}/documents"),
|
||||
_dataset_edit_operation("postKnowledgeSpacesByIdDocuments", "POST", "knowledge-spaces/{id}/documents"),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesByIdDocumentsBulk",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/documents/bulk",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsBulk",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/bulk",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsBulkReindex",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/bulk/reindex",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"knowledge-spaces/{id}/documents/{documentId}",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/documents/{documentId}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdLogicalDocuments",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/logical-documents",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/logical-documents/{documentId}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdOutline",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/outline",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/logical-documents/{documentId}",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/revisions",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionRollback",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/rollback",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"patchKnowledgeSpacesByIdDocumentsByDocumentIdMetadata",
|
||||
"PATCH",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/metadata",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkId",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkIdState",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}/state",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdProcessingTasks",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/processing-tasks",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasks",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/processing-tasks",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}/events",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
max_response_bytes=67_108_864,
|
||||
request_headers=("last-event-id", "x-trace-id"),
|
||||
response_kind="stream",
|
||||
response_media_types=("text/event-stream",),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
method="DELETE",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}",
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}/retry",
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/settings",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"putKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"PUT",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/settings",
|
||||
),
|
||||
_dataset_read_operation("getJobsById", "jobs/{id}"),
|
||||
_dataset_edit_operation("deleteJobsById", "DELETE", "jobs/{id}"),
|
||||
_dataset_edit_operation("postJobsByIdRetry", "POST", "jobs/{id}/retry"),
|
||||
_dataset_read_operation("getDeletionJobsByJobId", "deletion-jobs/{jobId}"),
|
||||
_dataset_edit_operation(
|
||||
"postDeletionJobsByJobIdRetry",
|
||||
"POST",
|
||||
"deletion-jobs/{jobId}/retry",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_read_operation("getBulkJobsById", "bulk-jobs/{id}"),
|
||||
)
|
||||
@ -1,33 +1,32 @@
|
||||
"""Transport-only forwarding for the explicitly enabled KnowledgeFS Console operations.
|
||||
"""Authorize and forward the explicitly enabled KnowledgeFS Console operations.
|
||||
|
||||
KnowledgeFS owns the request and response contract. This module binds short-lived
|
||||
account and workspace identities, enforces Dify's coarse workspace policy, and
|
||||
normalizes transport failures. Dify deliberately maintains a small product-facing
|
||||
operation registry instead of exposing the full upstream OpenAPI surface. The
|
||||
dedicated request path uses Dify's shared SSRF policy, never follows redirects,
|
||||
bounds buffered responses, and rejects compressed responses.
|
||||
The dedicated request path uses Dify's shared SSRF policy, never follows redirects,
|
||||
bounds buffered responses, and rejects compressed streaming responses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
from typing import Final, Literal, NamedTuple, Protocol
|
||||
from typing import NamedTuple, Protocol
|
||||
|
||||
import httpx
|
||||
import jwt
|
||||
|
||||
from configs import dify_config
|
||||
from core.helper import ssrf_proxy
|
||||
from core.rbac import RBACPermission, RBACResourceScope
|
||||
from core.rbac import RBACResourceScope
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from models import Account
|
||||
from services.enterprise.rbac_service import RBACService
|
||||
|
||||
type KnowledgeFSMethod = Literal["DELETE", "GET", "PATCH", "POST", "PUT"]
|
||||
type KnowledgeFSResponseKind = Literal["binary", "buffered", "stream"]
|
||||
type KnowledgeFSRequiredScope = Literal["knowledge-spaces:read", "knowledge-spaces:write"]
|
||||
from services.knowledge_fs_operations import (
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSOperation,
|
||||
KnowledgeFSResponseKind,
|
||||
)
|
||||
|
||||
_JWT_AUDIENCE = "knowledge-fs"
|
||||
_JWT_ISSUER = "dify"
|
||||
@ -35,56 +34,55 @@ _JWT_TTL_SECONDS = 60
|
||||
_MAX_BUFFERED_RESPONSE_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
class KnowledgeFSOperation(NamedTuple):
|
||||
operation_id: str
|
||||
method: KnowledgeFSMethod
|
||||
path: str
|
||||
response_kind: KnowledgeFSResponseKind
|
||||
required_scope: KnowledgeFSRequiredScope
|
||||
rbac_permission: RBACPermission
|
||||
requires_dataset_editor: bool
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
|
||||
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS: Final[tuple[KnowledgeFSOperation, ...]] = (
|
||||
KnowledgeFSOperation(
|
||||
operation_id="listKnowledgeSpaces",
|
||||
method="GET",
|
||||
path="knowledge-spaces",
|
||||
response_kind="buffered",
|
||||
required_scope="knowledge-spaces:read",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
requires_dataset_editor=False,
|
||||
max_response_bytes=1_048_576,
|
||||
request_headers=("x-trace-id",),
|
||||
response_headers=("x-trace-id",),
|
||||
response_media_types=("application/json",),
|
||||
),
|
||||
KnowledgeFSOperation(
|
||||
operation_id="createKnowledgeSpace",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
response_kind="buffered",
|
||||
required_scope="knowledge-spaces:write",
|
||||
rbac_permission=RBACPermission.DATASET_CREATE_AND_MANAGEMENT,
|
||||
requires_dataset_editor=True,
|
||||
max_response_bytes=1_048_576,
|
||||
request_headers=("x-trace-id",),
|
||||
response_headers=("x-trace-id",),
|
||||
response_media_types=("application/json",),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSUpstreamResponse(NamedTuple):
|
||||
response: httpx.Response
|
||||
response_kind: KnowledgeFSResponseKind
|
||||
operation: KnowledgeFSOperation
|
||||
|
||||
|
||||
_AUTHORIZATION_MARKER = object()
|
||||
|
||||
|
||||
@dataclass(eq=False, frozen=True, init=False, slots=True)
|
||||
class KnowledgeFSAuthorization:
|
||||
"""Single-use forwarding capability created after Dify workspace policy checks.
|
||||
|
||||
Callers obtain this value from :func:`authorize_knowledge_fs_request`. Direct
|
||||
construction and repeated forwarding are rejected before outbound I/O.
|
||||
"""
|
||||
|
||||
account_id: str
|
||||
tenant_id: str
|
||||
operation: KnowledgeFSOperation
|
||||
_used: bool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
account_id: str,
|
||||
tenant_id: str,
|
||||
operation: KnowledgeFSOperation,
|
||||
*,
|
||||
_marker: object | None = None,
|
||||
) -> None:
|
||||
if _marker is not _AUTHORIZATION_MARKER:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS authorization must be created by workspace authorization")
|
||||
object.__setattr__(self, "account_id", account_id)
|
||||
object.__setattr__(self, "tenant_id", tenant_id)
|
||||
object.__setattr__(self, "operation", operation)
|
||||
object.__setattr__(self, "_used", False)
|
||||
|
||||
def consume(self) -> tuple[str, str, KnowledgeFSOperation]:
|
||||
"""Return the authorized principals and canonical operation exactly once.
|
||||
|
||||
Raises:
|
||||
KnowledgeFSAccessDeniedError: The capability was already consumed.
|
||||
"""
|
||||
if self._used:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS authorization has already been used")
|
||||
object.__setattr__(self, "_used", True)
|
||||
return self.account_id, self.tenant_id, self.operation
|
||||
|
||||
|
||||
class _RequestHeaders(Protocol):
|
||||
def items(self) -> Iterable[tuple[str, str]]: ...
|
||||
|
||||
@ -113,20 +111,29 @@ def authorize_knowledge_fs_request(
|
||||
*,
|
||||
account: Account,
|
||||
tenant_id: str,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
) -> KnowledgeFSAuthorization:
|
||||
"""Enforce Dify's workspace policy before KFS performs resource authorization.
|
||||
|
||||
Args:
|
||||
account: Authenticated Dify account with its current workspace role.
|
||||
tenant_id: Current Dify workspace identifier.
|
||||
operation: Dify-maintained KnowledgeFS operation and policy metadata.
|
||||
method: Requested upstream HTTP method.
|
||||
path: Requested relative KnowledgeFS path.
|
||||
|
||||
Raises:
|
||||
KnowledgeFSRouteNotAllowedError: The method and path do not resolve to a declared operation.
|
||||
KnowledgeFSAccessDeniedError: The account lacks a required legacy or enterprise permission.
|
||||
|
||||
Returns:
|
||||
A request-scoped capability binding the authorized account, workspace, and operation.
|
||||
"""
|
||||
if operation.requires_dataset_editor and not account.is_dataset_editor:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS mutations require dataset edit access")
|
||||
operation = get_knowledge_fs_operation(method, path)
|
||||
if operation.legacy_role == "dataset_editor" and not account.is_dataset_editor:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS operation requires dataset edit access")
|
||||
if operation.legacy_role == "admin" and not account.is_admin_or_owner:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS operation requires workspace administration access")
|
||||
if not RBACService.CheckAccess.check(
|
||||
tenant_id,
|
||||
account.id,
|
||||
@ -134,6 +141,7 @@ def authorize_knowledge_fs_request(
|
||||
resource_type=RBACResourceScope.DATASET.value,
|
||||
):
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS operation is denied by workspace RBAC")
|
||||
return KnowledgeFSAuthorization(account.id, tenant_id, operation, _marker=_AUTHORIZATION_MARKER)
|
||||
|
||||
|
||||
def proxy_knowledge_fs_request(
|
||||
@ -149,20 +157,62 @@ def proxy_knowledge_fs_request(
|
||||
request_headers: _RequestHeaders | None = None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
"""Authorize and forward one allowlisted KnowledgeFS request as a single use case."""
|
||||
operation = get_knowledge_fs_operation(method, path)
|
||||
authorize_knowledge_fs_request(
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id=tenant_id,
|
||||
operation=operation,
|
||||
method=method,
|
||||
path=path,
|
||||
)
|
||||
|
||||
return proxy_authorized_knowledge_fs_request(
|
||||
authorization=authorization,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
def proxy_authorized_knowledge_fs_request(
|
||||
*,
|
||||
authorization: KnowledgeFSAuthorization,
|
||||
accept: str | None = None,
|
||||
content_type: str | None = None,
|
||||
query: bytes | None = None,
|
||||
body: bytes | None = None,
|
||||
request_headers: _RequestHeaders | None = None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
"""Forward one request whose operation and workspace policy were already authorized.
|
||||
|
||||
This performs one outbound KnowledgeFS request and does not repeat Dify RBAC checks.
|
||||
|
||||
Args:
|
||||
authorization: Request-scoped capability returned by :func:`authorize_knowledge_fs_request`.
|
||||
accept: Original Accept header, when present.
|
||||
content_type: Original request Content-Type header, when present.
|
||||
query: Original encoded query string from the Console request.
|
||||
body: Original request body, when present.
|
||||
request_headers: Incoming headers; only names declared by the operation are forwarded.
|
||||
|
||||
Returns:
|
||||
The bounded KnowledgeFS response together with its transport metadata.
|
||||
|
||||
Raises:
|
||||
KnowledgeFSConfigurationError: The connection is incomplete or blocked by outbound policy.
|
||||
KnowledgeFSRouteNotAllowedError: A forwarded request header is outside the operation contract.
|
||||
KnowledgeFSTimeoutError: KnowledgeFS exceeds the configured timeout.
|
||||
KnowledgeFSTransportError: The request fails or its response violates transport bounds.
|
||||
"""
|
||||
account_id, tenant_id, operation = authorization.consume()
|
||||
incoming_request_headers = {name.lower(): value for name, value in (request_headers or {}).items()}
|
||||
contract_request_headers = {
|
||||
name: incoming_request_headers[name] for name in operation.request_headers if name in incoming_request_headers
|
||||
}
|
||||
return _forward_knowledge_fs_request(
|
||||
account_id=account.id,
|
||||
method=method,
|
||||
path=path,
|
||||
account_id=account_id,
|
||||
method=operation.method,
|
||||
path=operation.path,
|
||||
tenant_id=tenant_id,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
|
||||
@ -7,9 +7,11 @@ from unittest.mock import MagicMock
|
||||
import httpx
|
||||
import pytest
|
||||
from flask import Flask, Response
|
||||
from pydantic import SecretStr
|
||||
from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
HTTPException,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
@ -26,12 +28,14 @@ from controllers.console.knowledge_fs_proxy import (
|
||||
proxy_knowledge_fs_write,
|
||||
)
|
||||
from controllers.console.wraps import RBACPermission
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
from services.knowledge_fs_operations import (
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSOperation,
|
||||
KnowledgeFSResponseKind,
|
||||
)
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
get_knowledge_fs_operation,
|
||||
@ -47,6 +51,7 @@ def _upstream(
|
||||
response: httpx.Response,
|
||||
kind: KnowledgeFSResponseKind = "buffered",
|
||||
*,
|
||||
error_status_map: tuple[tuple[int, int], ...] = ((401, 502), (403, 403)),
|
||||
max_response_bytes: int | None = None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
operation = KnowledgeFSOperation(
|
||||
@ -56,7 +61,7 @@ def _upstream(
|
||||
response_kind=kind,
|
||||
required_scope="knowledge-spaces:read",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
requires_dataset_editor=False,
|
||||
legacy_role="reader",
|
||||
max_response_bytes=max_response_bytes
|
||||
or (64 * 1024 * 1024 if kind == "stream" else 25 * 1024 * 1024 if kind == "binary" else 1024 * 1024),
|
||||
request_headers=(),
|
||||
@ -67,6 +72,7 @@ def _upstream(
|
||||
"x-session-id",
|
||||
),
|
||||
response_media_types=(),
|
||||
error_status_map=error_status_map,
|
||||
)
|
||||
return KnowledgeFSUpstreamResponse(response, kind, operation)
|
||||
|
||||
@ -93,7 +99,7 @@ def _set_current_workspace(
|
||||
def _bypass_policy_wrappers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.knowledge_fs_proxy._proxy_knowledge_fs_non_get",
|
||||
unwrap(_proxy_knowledge_fs_non_get),
|
||||
lambda method, path: _proxy_request(method, path),
|
||||
)
|
||||
|
||||
|
||||
@ -287,10 +293,8 @@ def test_read_post_applies_knowledge_rate_limit_once(
|
||||
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy.current_account_with_tenant", current_workspace)
|
||||
monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", current_workspace)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.RBACService.CheckAccess.check",
|
||||
MagicMock(return_value=True),
|
||||
)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.wraps.FeatureService.get_knowledge_rate_limit",
|
||||
MagicMock(return_value=MagicMock(enabled=True, limit=10)),
|
||||
@ -300,14 +304,19 @@ def test_read_post_applies_knowledge_rate_limit_once(
|
||||
monkeypatch.setattr("controllers.console.wraps.redis_client.zremrangebyscore", MagicMock())
|
||||
monkeypatch.setattr("controllers.console.wraps.redis_client.zcard", MagicMock(return_value=1))
|
||||
proxy = MagicMock(return_value=Response(status=200))
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy._proxy_request", proxy)
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy._proxy_authorized_request", proxy)
|
||||
|
||||
with app.test_request_context("/console/api/knowledge-fs/knowledge-spaces", method="POST"):
|
||||
response = _proxy_knowledge_fs_non_get("POST", "knowledge-spaces")
|
||||
|
||||
assert isinstance(response, Response)
|
||||
zadd.assert_called_once()
|
||||
proxy.assert_called_once_with("POST", "knowledge-spaces")
|
||||
proxy.assert_called_once()
|
||||
authorization = proxy.call_args.args[0]
|
||||
assert authorization.account_id == "account-1"
|
||||
assert authorization.tenant_id == "tenant-1"
|
||||
assert authorization.operation.operation_id == "createKnowledgeSpace"
|
||||
check_access.assert_called_once()
|
||||
|
||||
|
||||
def test_denied_write_does_not_consume_the_workspace_rate_limit(
|
||||
@ -447,6 +456,65 @@ def test_generic_write_forwards_path_raw_body_and_current_tenant(
|
||||
assert response.get_json()["tenantId"] == "tenant-1"
|
||||
|
||||
|
||||
def test_generic_write_forwards_through_the_authorized_production_path(
|
||||
app: Flask,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
|
||||
def current_workspace() -> tuple[MagicMock, str]:
|
||||
return account, "tenant-1"
|
||||
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy.current_account_with_tenant", current_workspace)
|
||||
monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", current_workspace)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.wraps.FeatureService.get_knowledge_rate_limit",
|
||||
MagicMock(return_value=MagicMock(enabled=False)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.dify_config.KNOWLEDGE_FS_BASE_URL",
|
||||
"http://knowledge-fs.test",
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.dify_config.KNOWLEDGE_FS_JWT_SECRET",
|
||||
SecretStr("production-secret-with-at-least-32-bytes"),
|
||||
raising=False,
|
||||
)
|
||||
upstream_request = MagicMock(
|
||||
return_value=httpx.Response(
|
||||
201,
|
||||
content=b'{"id":"space-1","tenantId":"tenant-1"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", upstream_request)
|
||||
route = unwrap(proxy_knowledge_fs_write)
|
||||
body = b'{"idempotencyKey":"create-product-docs","name":"Product docs"}'
|
||||
|
||||
with app.test_request_context(
|
||||
"/console/api/knowledge-fs/knowledge-spaces",
|
||||
method="POST",
|
||||
query_string={"source": "console"},
|
||||
data=body,
|
||||
content_type="application/json",
|
||||
headers={"X-Trace-Id": "trace-1"},
|
||||
):
|
||||
response = route("knowledge-spaces")
|
||||
|
||||
assert isinstance(response, Response)
|
||||
assert response.status_code == 201
|
||||
assert response.get_json() == {"id": "space-1", "tenantId": "tenant-1"}
|
||||
check_access.assert_called_once()
|
||||
assert upstream_request.call_args.kwargs["method"] == "POST"
|
||||
assert upstream_request.call_args.kwargs["url"] == "http://knowledge-fs.test/knowledge-spaces"
|
||||
assert upstream_request.call_args.kwargs["params"] == b"source=console"
|
||||
assert upstream_request.call_args.kwargs["content"] == body
|
||||
assert upstream_request.call_args.kwargs["headers"]["x-trace-id"] == "trace-1"
|
||||
|
||||
|
||||
def test_generic_write_forwards_contract_declared_request_headers(
|
||||
app: Flask,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@ -617,6 +685,43 @@ def test_resource_authorization_rejection_is_exposed_as_forbidden(
|
||||
route("knowledge-spaces")
|
||||
|
||||
|
||||
def test_proxy_response_applies_operation_specific_error_status_mapping() -> None:
|
||||
upstream = httpx.Response(
|
||||
429,
|
||||
content=b'{"error":"rate limited"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
with pytest.raises(ServiceUnavailable):
|
||||
_proxy_response(
|
||||
_upstream(upstream, error_status_map=((429, 503),)),
|
||||
tenant_id="tenant-1",
|
||||
contract_response_headers=(),
|
||||
max_response_bytes=1024 * 1024,
|
||||
)
|
||||
|
||||
assert upstream.is_closed
|
||||
|
||||
|
||||
def test_proxy_response_preserves_nonstandard_mapped_error_status() -> None:
|
||||
upstream = httpx.Response(
|
||||
429,
|
||||
content=b'{"error":"rate limited"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_proxy_response(
|
||||
_upstream(upstream, error_status_map=((429, 499),)),
|
||||
tenant_id="tenant-1",
|
||||
contract_response_headers=(),
|
||||
max_response_bytes=1024 * 1024,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == 499
|
||||
assert upstream.is_closed
|
||||
|
||||
|
||||
def test_contract_response_headers_are_deduplicated_case_insensitively() -> None:
|
||||
upstream = httpx.Response(
|
||||
200,
|
||||
@ -671,6 +776,7 @@ def test_disallowed_non_get_route_is_hidden_as_not_found(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
method: KnowledgeFSMethod,
|
||||
) -> None:
|
||||
_set_current_workspace(monkeypatch)
|
||||
route = unwrap(proxy_knowledge_fs_write)
|
||||
|
||||
with app.test_request_context("/console/api/knowledge-fs/not-a-route", method=method):
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@ -15,7 +16,7 @@ from dev.generate_knowledge_fs_contract import (
|
||||
filter_openapi_document,
|
||||
validate_declarations,
|
||||
)
|
||||
from services.knowledge_fs_proxy import KNOWLEDGE_FS_CONSOLE_OPERATIONS, KnowledgeFSOperation
|
||||
from services.knowledge_fs_operations import KNOWLEDGE_FS_CONSOLE_OPERATIONS, KnowledgeFSOperation
|
||||
|
||||
|
||||
def test_contract_cli_updates_checks_and_detects_openapi_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@ -111,7 +112,7 @@ def test_contract_script_loads_runtime_registry_outside_api_directory(tmp_path:
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.stdout.strip() == "2"
|
||||
assert result.stdout.strip() == str(len(KNOWLEDGE_FS_CONSOLE_OPERATIONS))
|
||||
|
||||
|
||||
def test_validate_declarations_accepts_matching_contract() -> None:
|
||||
@ -178,47 +179,115 @@ def test_filter_openapi_document_keeps_only_declared_operations_and_referenced_s
|
||||
|
||||
assert set(filtered["paths"]) == {"/knowledge-spaces"}
|
||||
assert set(filtered["paths"]["/knowledge-spaces"]) == {"get"}
|
||||
assert set(filtered["components"]["schemas"]) == {"KnowledgeSpaceList", "KnowledgeSpace"}
|
||||
assert set(filtered["components"]["schemas"]) == {
|
||||
"ConsoleProxyError",
|
||||
"KnowledgeSpaceList",
|
||||
"KnowledgeSpace",
|
||||
}
|
||||
assert filtered["components"]["securitySchemes"] == document["components"]["securitySchemes"]
|
||||
|
||||
|
||||
def test_console_operation_registry_matches_contract() -> None:
|
||||
list_route = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
create_route = operation("knowledge-spaces:write", "createKnowledgeSpace")
|
||||
for route in (list_route, create_route):
|
||||
route["parameters"] = [{"in": "header", "name": "X-Trace-Id"}]
|
||||
route["responses"] = {
|
||||
"200": {
|
||||
"content": {"application/json": {}},
|
||||
"headers": {"X-Trace-Id": {}},
|
||||
}
|
||||
}
|
||||
def test_filter_openapi_document_keeps_sse_for_streaming_orpc_contracts() -> None:
|
||||
json_declaration = declaration()
|
||||
stream_declaration = declaration(
|
||||
operation_id="streamTask",
|
||||
path="tasks/{id}/events",
|
||||
response_kind="stream",
|
||||
response_media_types=("text/event-stream",),
|
||||
)
|
||||
json_operation = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
stream_operation = operation(
|
||||
"knowledge-spaces:read",
|
||||
"streamTask",
|
||||
responses={"200": {"content": {"text/event-stream": {"schema": {"$ref": "#/components/schemas/TaskEvent"}}}}},
|
||||
)
|
||||
|
||||
validate_declarations(
|
||||
filtered = filter_openapi_document(
|
||||
{
|
||||
"paths": {
|
||||
"/knowledge-spaces": {
|
||||
"get": list_route,
|
||||
"post": create_route,
|
||||
}
|
||||
}
|
||||
"/knowledge-spaces": {"get": json_operation},
|
||||
"/tasks/{id}/events": {"get": stream_operation},
|
||||
},
|
||||
"components": {"schemas": {"TaskEvent": {"type": "object"}}},
|
||||
},
|
||||
(json_declaration, stream_declaration),
|
||||
)
|
||||
|
||||
assert set(filtered["paths"]) == {"/knowledge-spaces", "/tasks/{id}/events"}
|
||||
assert filtered["components"]["schemas"]["TaskEvent"] == {"type": "object"}
|
||||
|
||||
|
||||
def test_filter_openapi_document_rewrites_proxy_error_responses() -> None:
|
||||
route = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
route["responses"] = {
|
||||
"200": {"content": {"application/json": {}}},
|
||||
"401": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}},
|
||||
"403": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}},
|
||||
}
|
||||
document = {
|
||||
"paths": {"/knowledge-spaces": {"get": route}},
|
||||
"components": {"schemas": {"ErrorResponse": {"type": "object"}}},
|
||||
}
|
||||
|
||||
filtered = filter_openapi_document(
|
||||
document,
|
||||
(declaration(error_status_map=((401, 502), (403, 403))),),
|
||||
)
|
||||
|
||||
responses = filtered["paths"]["/knowledge-spaces"]["get"]["responses"]
|
||||
assert "401" not in responses
|
||||
assert responses["403"]["content"]["application/json"]["schema"] == {
|
||||
"$ref": "#/components/schemas/ConsoleProxyError"
|
||||
}
|
||||
assert responses["502"]["content"]["application/json"]["schema"] == {
|
||||
"$ref": "#/components/schemas/ConsoleProxyError"
|
||||
}
|
||||
assert filtered["components"]["schemas"]["ConsoleProxyError"]["required"] == ["code", "message", "status"]
|
||||
|
||||
|
||||
def test_console_operation_registry_matches_contract() -> None:
|
||||
validate_declarations(
|
||||
console_registry_document(),
|
||||
tuple(_contract_declaration(operation) for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS),
|
||||
)
|
||||
|
||||
|
||||
def test_generated_contract_metadata_matches_current_pin_and_registry() -> None:
|
||||
metadata = (
|
||||
contract_validator.WORKSPACE_ROOT / "packages/contracts/generated/knowledge-fs/metadata.gen.ts"
|
||||
).read_text()
|
||||
lock = json.loads(contract_validator.LOCK_PATH.read_text())
|
||||
|
||||
assert _metadata_string(metadata, "knowledgeFsSourceOpenapiSha256") == lock["openapiSha256"]
|
||||
assert _metadata_string(
|
||||
metadata,
|
||||
"knowledgeFsConsoleDeclarationsSha256",
|
||||
) == contract_validator.contract_declarations_sha256(contract_validator.console_contract_declarations())
|
||||
|
||||
|
||||
def _metadata_string(source: str, export_name: str) -> str:
|
||||
match = re.search(rf"export const {export_name}\s*=\s*'([0-9a-f]{{64}})'", source)
|
||||
assert match is not None, f"missing generated metadata export: {export_name}"
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def console_registry_document() -> dict[str, object]:
|
||||
list_route = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
create_route = operation("knowledge-spaces:write", "createKnowledgeSpace")
|
||||
for route in (list_route, create_route):
|
||||
route["parameters"] = [{"in": "header", "name": "X-Trace-Id"}]
|
||||
route["responses"] = {
|
||||
"200": {
|
||||
"content": {"application/json": {}},
|
||||
"headers": {"X-Trace-Id": {}},
|
||||
}
|
||||
}
|
||||
return {"paths": {"/knowledge-spaces": {"get": list_route, "post": create_route}}}
|
||||
paths: dict[str, dict[str, object]] = {}
|
||||
for console_operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS:
|
||||
route = operation(
|
||||
console_operation.required_scope,
|
||||
console_operation.operation_id,
|
||||
parameters=[{"in": "header", "name": name} for name in console_operation.request_headers],
|
||||
responses={
|
||||
"200": {
|
||||
"content": {media_type: {} for media_type in console_operation.response_media_types},
|
||||
"headers": {name: {} for name in console_operation.response_headers},
|
||||
}
|
||||
},
|
||||
)
|
||||
route["x-knowledge-fs-max-response-bytes"] = console_operation.max_response_bytes
|
||||
paths.setdefault(f"/{console_operation.path}", {})[console_operation.method.lower()] = route
|
||||
return {"paths": paths}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@ -377,6 +446,7 @@ def declaration(**overrides: object) -> ContractDeclaration:
|
||||
"request_headers": (),
|
||||
"response_headers": (),
|
||||
"response_media_types": ("application/json",),
|
||||
"error_status_map": ((401, 502), (403, 403)),
|
||||
}
|
||||
value.update(overrides)
|
||||
return cast(ContractDeclaration, value)
|
||||
@ -393,6 +463,7 @@ def _contract_declaration(operation: KnowledgeFSOperation) -> ContractDeclaratio
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
"error_status_map": operation.error_status_map,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -10,16 +10,21 @@ from pydantic import SecretStr
|
||||
from core.helper import ssrf_proxy
|
||||
from core.rbac import RBACPermission
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from services.knowledge_fs_proxy import (
|
||||
from services.knowledge_fs_operations import (
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS,
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSOperation,
|
||||
)
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSAuthorization,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_authorized_knowledge_fs_request,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
from services.knowledge_fs_proxy import (
|
||||
@ -28,16 +33,178 @@ from services.knowledge_fs_proxy import (
|
||||
|
||||
_JWT_SECRET = "production-secret-with-at-least-32-bytes"
|
||||
|
||||
_HAPPY_PATH_OPERATION_IDS = (
|
||||
"listKnowledgeSpaces",
|
||||
"createKnowledgeSpace",
|
||||
"getKnowledgeSpacesById",
|
||||
"getKnowledgeSpacesByIdAccessPolicy",
|
||||
"patchKnowledgeSpacesByIdAccessPolicy",
|
||||
"getSourceProviders",
|
||||
"getKnowledgeSpacesByIdSourceConnections",
|
||||
"postKnowledgeSpacesByIdSourceConnections",
|
||||
"postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh",
|
||||
"getKnowledgeSpacesByIdSources",
|
||||
"postKnowledgeSpacesByIdSources",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview",
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunId",
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages",
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel",
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry",
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
"getKnowledgeSpacesByIdLogicalDocuments",
|
||||
"getKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks",
|
||||
"getKnowledgeSpacesByIdProcessingTasks",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents",
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry",
|
||||
)
|
||||
|
||||
_REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS = {
|
||||
"patchKnowledgeSpacesById",
|
||||
"deleteKnowledgeSpacesById",
|
||||
"getKnowledgeSpacesByIdStats",
|
||||
"postKnowledgeSpacesByIdSourceConnectionsOauth",
|
||||
"postSourceOauthCallback",
|
||||
"getKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"deleteKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"patchKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdSync",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdWorkflowImports",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdPages",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdFiles",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawl",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImport",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdTest",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImportFiles",
|
||||
"postKnowledgeSpacesByIdSourcesBulk",
|
||||
"getKnowledgeSpacesByIdSourceWorkflows",
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdBulkItems",
|
||||
"getKnowledgeSpacesByIdDocuments",
|
||||
"postKnowledgeSpacesByIdDocuments",
|
||||
"deleteKnowledgeSpacesByIdDocumentsBulk",
|
||||
"postKnowledgeSpacesByIdDocumentsBulk",
|
||||
"postKnowledgeSpacesByIdDocumentsBulkReindex",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"deleteKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdOutline",
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionRollback",
|
||||
"patchKnowledgeSpacesByIdDocumentsByDocumentIdMetadata",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkId",
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkIdState",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasks",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"putKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"getJobsById",
|
||||
"deleteJobsById",
|
||||
"postJobsByIdRetry",
|
||||
"getDeletionJobsByJobId",
|
||||
"postDeletionJobsByJobIdRetry",
|
||||
"getBulkJobsById",
|
||||
}
|
||||
|
||||
_EXPANDED_EXTERNAL_SOURCE_OPERATION_IDS = {
|
||||
operation_id
|
||||
for operation_id in _REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS
|
||||
if "Source" in operation_id or operation_id == "postSourceOauthCallback"
|
||||
}
|
||||
|
||||
_OPERATION_AUTHORIZATION_POLICIES = {
|
||||
"listKnowledgeSpaces": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"createKnowledgeSpace": (RBACPermission.DATASET_CREATE_AND_MANAGEMENT, "dataset_editor"),
|
||||
"getKnowledgeSpacesById": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdAccessPolicy": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"patchKnowledgeSpacesByIdAccessPolicy": (RBACPermission.DATASET_ACCESS_CONFIG, "admin"),
|
||||
"getSourceProviders": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"getKnowledgeSpacesByIdSourceConnections": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"postKnowledgeSpacesByIdSourceConnections": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSources": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"postKnowledgeSpacesByIdSources": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunId": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy": (RBACPermission.DATASET_EDIT, "dataset_editor"),
|
||||
"getKnowledgeSpacesByIdLogicalDocuments": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdLogicalDocumentsByDocumentId": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks": (
|
||||
RBACPermission.DATASET_READONLY,
|
||||
"reader",
|
||||
),
|
||||
"getKnowledgeSpacesByIdProcessingTasks": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents": (
|
||||
RBACPermission.DATASET_READONLY,
|
||||
"reader",
|
||||
),
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId": (
|
||||
RBACPermission.DATASET_EDIT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry": (
|
||||
RBACPermission.DATASET_EDIT,
|
||||
"dataset_editor",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _materialized_path(operation: KnowledgeFSOperation) -> str:
|
||||
segments = []
|
||||
for segment in operation.path.split("/"):
|
||||
if segment == "{revision}":
|
||||
segments.append("1")
|
||||
elif segment.startswith("{"):
|
||||
segments.append("00000000-0000-4000-8000-000000000001")
|
||||
else:
|
||||
segments.append(segment)
|
||||
return "/".join(segments)
|
||||
|
||||
|
||||
def _set_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
base_url: str | None = "http://knowledge-fs.test",
|
||||
sse_read_timeout_seconds: float = 90.0,
|
||||
timeout_seconds: float = 7.5,
|
||||
jwt_secret: str | None = _JWT_SECRET,
|
||||
) -> None:
|
||||
values = {
|
||||
"KNOWLEDGE_FS_BASE_URL": base_url,
|
||||
"KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS": sse_read_timeout_seconds,
|
||||
"KNOWLEDGE_FS_TIMEOUT_SECONDS": timeout_seconds,
|
||||
"KNOWLEDGE_FS_JWT_SECRET": SecretStr(jwt_secret) if jwt_secret is not None else None,
|
||||
}
|
||||
@ -45,43 +212,68 @@ def _set_config(
|
||||
monkeypatch.setattr(f"services.knowledge_fs_proxy.dify_config.{name}", value, raising=False)
|
||||
|
||||
|
||||
def test_console_registry_starts_with_list_and_create_operations() -> None:
|
||||
assert tuple(operation.operation_id for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS) == (
|
||||
"listKnowledgeSpaces",
|
||||
"createKnowledgeSpace",
|
||||
def _processing_task_events_path() -> str:
|
||||
operation = next(
|
||||
operation
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
if operation.operation_id == "getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents"
|
||||
)
|
||||
return _materialized_path(operation)
|
||||
|
||||
|
||||
def test_console_registry_exposes_only_the_new_rag_happy_path_operations() -> None:
|
||||
operation_ids = {operation.operation_id for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS}
|
||||
|
||||
assert operation_ids == set(_HAPPY_PATH_OPERATION_IDS) | _REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS
|
||||
|
||||
|
||||
def test_console_registry_exposes_existing_upstream_contracts_needed_by_all_happy_path_pages() -> None:
|
||||
operation_ids = {operation.operation_id for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS}
|
||||
|
||||
assert operation_ids >= _REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS
|
||||
|
||||
|
||||
def test_console_registry_preserves_explicit_scope_and_authorization_policies() -> None:
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS:
|
||||
is_read = operation.method == "GET"
|
||||
assert operation.required_scope == f"knowledge-spaces:{'read' if is_read else 'write'}"
|
||||
expected_policy = _OPERATION_AUTHORIZATION_POLICIES.get(operation.operation_id)
|
||||
if expected_policy is None and operation.operation_id in _EXPANDED_EXTERNAL_SOURCE_OPERATION_IDS:
|
||||
expected_policy = (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor")
|
||||
if expected_policy is None:
|
||||
expected_policy = (
|
||||
(RBACPermission.DATASET_READONLY, "reader")
|
||||
if is_read
|
||||
else (RBACPermission.DATASET_EDIT, "dataset_editor")
|
||||
)
|
||||
assert (operation.rbac_permission, operation.legacy_role) == expected_policy
|
||||
assert operation.response_headers == ("x-trace-id",)
|
||||
|
||||
|
||||
def test_console_registry_preserves_special_transport_contracts() -> None:
|
||||
crawl_preview = get_knowledge_fs_operation(
|
||||
"POST",
|
||||
"knowledge-spaces/00000000-0000-4000-8000-000000000001/sources/"
|
||||
"00000000-0000-4000-8000-000000000002/crawl-preview",
|
||||
)
|
||||
selection = get_knowledge_fs_operation(
|
||||
"POST",
|
||||
"knowledge-spaces/00000000-0000-4000-8000-000000000001/source-workflows/"
|
||||
"00000000-0000-4000-8000-000000000002/selection",
|
||||
)
|
||||
events = get_knowledge_fs_operation(
|
||||
"GET",
|
||||
"knowledge-spaces/00000000-0000-4000-8000-000000000001/documents/"
|
||||
"00000000-0000-4000-8000-000000000002/processing-tasks/"
|
||||
"00000000-0000-4000-8000-000000000003/events",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "operation_id", "scope", "permission", "requires_dataset_editor"),
|
||||
[
|
||||
("GET", "listKnowledgeSpaces", "knowledge-spaces:read", RBACPermission.DATASET_READONLY, False),
|
||||
(
|
||||
"POST",
|
||||
"createKnowledgeSpace",
|
||||
"knowledge-spaces:write",
|
||||
RBACPermission.DATASET_CREATE_AND_MANAGEMENT,
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_console_registry_preserves_contract_and_policy(
|
||||
method: KnowledgeFSMethod,
|
||||
operation_id: str,
|
||||
scope: str,
|
||||
permission: RBACPermission,
|
||||
requires_dataset_editor: bool,
|
||||
) -> None:
|
||||
operation = get_knowledge_fs_operation(method, "knowledge-spaces")
|
||||
|
||||
assert operation.operation_id == operation_id
|
||||
assert operation.required_scope == scope
|
||||
assert operation.rbac_permission == permission
|
||||
assert operation.requires_dataset_editor is requires_dataset_editor
|
||||
assert operation.max_response_bytes == 1_048_576
|
||||
assert operation.request_headers == ("x-trace-id",)
|
||||
assert operation.response_headers == ("x-trace-id",)
|
||||
assert operation.response_media_types == ("application/json",)
|
||||
assert crawl_preview.request_headers == ("idempotency-key", "x-trace-id")
|
||||
assert selection.request_headers == ("idempotency-key", "x-trace-id")
|
||||
assert events.response_kind == "stream"
|
||||
assert events.max_response_bytes == 67_108_864
|
||||
assert events.request_headers == ("last-event-id", "x-trace-id")
|
||||
assert events.response_media_types == ("text/event-stream",)
|
||||
|
||||
|
||||
def test_unconfigured_kfs_is_rejected_before_external_io(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@ -148,7 +340,112 @@ def test_proxy_forwards_only_registry_declared_headers(monkeypatch: pytest.Monke
|
||||
assert forward.call_args.kwargs["request_headers"] == {"x-trace-id": "trace-1"}
|
||||
|
||||
|
||||
def test_authorization_rejects_workspace_rbac_denial(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_authorized_proxy_does_not_repeat_workspace_rbac(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
check_access = MagicMock(return_value=True)
|
||||
forward = MagicMock(return_value=MagicMock())
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy._forward_knowledge_fs_request", forward)
|
||||
operation = get_knowledge_fs_operation("POST", "knowledge-spaces")
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
proxy_authorized_knowledge_fs_request(authorization=authorization)
|
||||
|
||||
check_access.assert_called_once()
|
||||
assert forward.call_args.kwargs["account_id"] == "account-1"
|
||||
assert forward.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert forward.call_args.kwargs["method"] == "POST"
|
||||
assert forward.call_args.kwargs["path"] == "knowledge-spaces"
|
||||
|
||||
|
||||
def test_authorization_capability_cannot_be_constructed_directly() -> None:
|
||||
operation = get_knowledge_fs_operation("POST", "knowledge-spaces")
|
||||
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="must be created by workspace authorization"):
|
||||
KnowledgeFSAuthorization("account-1", "tenant-1", operation)
|
||||
|
||||
|
||||
def test_authorization_resolves_the_canonical_operation_policy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="dataset edit access"):
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
)
|
||||
|
||||
check_access.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "value"),
|
||||
[
|
||||
("account_id", "account-2"),
|
||||
("tenant_id", "tenant-2"),
|
||||
("operation", get_knowledge_fs_operation("GET", "knowledge-spaces")),
|
||||
],
|
||||
)
|
||||
def test_authorization_capability_binding_cannot_be_mutated(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
attribute: str,
|
||||
value: object,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.RBACService.CheckAccess.check",
|
||||
MagicMock(return_value=True),
|
||||
)
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
)
|
||||
|
||||
with pytest.raises(AttributeError):
|
||||
setattr(authorization, attribute, value)
|
||||
|
||||
assert authorization.account_id == "account-1"
|
||||
assert authorization.tenant_id == "tenant-1"
|
||||
assert authorization.operation == get_knowledge_fs_operation("POST", "knowledge-spaces")
|
||||
|
||||
|
||||
def test_authorization_capability_cannot_be_reused(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
forward = MagicMock(return_value=MagicMock())
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.RBACService.CheckAccess.check",
|
||||
MagicMock(return_value=True),
|
||||
)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy._forward_knowledge_fs_request", forward)
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
)
|
||||
|
||||
proxy_authorized_knowledge_fs_request(authorization=authorization)
|
||||
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="already been used"):
|
||||
proxy_authorized_knowledge_fs_request(authorization=authorization)
|
||||
forward.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", KNOWLEDGE_FS_CONSOLE_OPERATIONS, ids=lambda operation: operation.operation_id)
|
||||
def test_authorization_rejects_workspace_rbac_denial(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
check_access = MagicMock(return_value=False)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
@ -157,19 +454,28 @@ def test_authorization_rejects_workspace_rbac_denial(monkeypatch: pytest.MonkeyP
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
operation=get_knowledge_fs_operation("GET", "knowledge-spaces"),
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_called_once_with(
|
||||
"tenant-1",
|
||||
"account-1",
|
||||
scene="dataset_readonly",
|
||||
scene=operation.rbac_permission.value,
|
||||
resource_type="dataset",
|
||||
)
|
||||
|
||||
|
||||
def test_create_rejects_non_dataset_editor_before_rbac(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False)
|
||||
@pytest.mark.parametrize(
|
||||
"operation",
|
||||
tuple(operation for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS if operation.legacy_role == "dataset_editor"),
|
||||
ids=lambda operation: operation.operation_id,
|
||||
)
|
||||
def test_dataset_editor_operations_reject_legacy_viewers_before_rbac(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
|
||||
@ -177,41 +483,66 @@ def test_create_rejects_non_dataset_editor_before_rbac(monkeypatch: pytest.Monke
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
operation=get_knowledge_fs_operation("POST", "knowledge-spaces"),
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_not_called()
|
||||
|
||||
|
||||
def test_authorization_uses_the_declared_editor_policy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False)
|
||||
def test_admin_operation_rejects_legacy_editors_before_rbac(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
operation = get_knowledge_fs_operation("POST", "knowledge-spaces")._replace(requires_dataset_editor=False)
|
||||
operation = get_knowledge_fs_operation(
|
||||
"PATCH", "knowledge-spaces/00000000-0000-4000-8000-000000000001/access-policy"
|
||||
)
|
||||
|
||||
authorize_knowledge_fs_request(account=account, tenant_id="tenant-1", operation=operation)
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="administration access"):
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_not_called()
|
||||
|
||||
|
||||
def test_authorization_uses_the_declared_reader_policy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
operation = get_knowledge_fs_operation("GET", "knowledge-spaces")
|
||||
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "expected_scope"),
|
||||
[("GET", "knowledge-spaces:read"), ("POST", "knowledge-spaces:write")],
|
||||
)
|
||||
@pytest.mark.parametrize("operation", KNOWLEDGE_FS_CONSOLE_OPERATIONS, ids=lambda operation: operation.operation_id)
|
||||
def test_auth_signs_current_principals_and_declared_scope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
method: KnowledgeFSMethod,
|
||||
expected_scope: str,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
_set_config(monkeypatch)
|
||||
response = httpx.Response(200, content=b'{"items":[]}', headers={"Content-Type": "application/json"})
|
||||
response = httpx.Response(
|
||||
200,
|
||||
content=b"data" if operation.response_kind == "stream" else b'{"items":[]}',
|
||||
headers={"Content-Type": operation.response_media_types[0]},
|
||||
)
|
||||
request = MagicMock(return_value=response)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", request)
|
||||
|
||||
forward_knowledge_fs_request(
|
||||
account_id="account-1",
|
||||
method=method,
|
||||
path="knowledge-spaces",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
tenant_id="tenant-1",
|
||||
)
|
||||
|
||||
@ -226,7 +557,7 @@ def test_auth_signs_current_principals_and_declared_scope(
|
||||
assert claims["dify_account_id"] == "dify-account:account-1"
|
||||
assert claims["sub"] == "dify-workspace:tenant-1"
|
||||
assert claims["tenant_id"] == "tenant-1"
|
||||
assert claims["scopes"] == [expected_scope]
|
||||
assert claims["scopes"] == [operation.required_scope]
|
||||
assert claims["caller_kind"] == "interactive"
|
||||
assert claims["exp"] - claims["iat"] == 60
|
||||
|
||||
@ -244,6 +575,69 @@ def test_buffered_response_rejects_non_empty_body_without_content_type(monkeypat
|
||||
assert response.is_closed
|
||||
|
||||
|
||||
def test_sse_response_remains_streaming_and_uses_the_dedicated_read_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_set_config(monkeypatch, sse_read_timeout_seconds=120.0)
|
||||
request = httpx.Request(
|
||||
"GET",
|
||||
"http://knowledge-fs.test/events",
|
||||
extensions={"timeout": {"connect": 7.5, "read": 7.5}},
|
||||
)
|
||||
response = httpx.Response(
|
||||
200,
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
request=request,
|
||||
stream=httpx.ByteStream(b"event: progress\n\n"),
|
||||
)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", MagicMock(return_value=response))
|
||||
buffer_response = MagicMock()
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.buffer_response", buffer_response)
|
||||
|
||||
result = forward_knowledge_fs_request(
|
||||
account_id="account-dev",
|
||||
method="GET",
|
||||
path=_processing_task_events_path(),
|
||||
tenant_id="tenant-dev",
|
||||
)
|
||||
|
||||
assert result.response is response
|
||||
assert result.response_kind == "stream"
|
||||
assert request.extensions["timeout"]["read"] == 120.0
|
||||
assert not response.is_closed
|
||||
buffer_response.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("headers", "message"),
|
||||
[
|
||||
({"Content-Type": "application/json"}, "unsupported media type"),
|
||||
(
|
||||
{"Content-Type": "text/event-stream", "Content-Encoding": "gzip"},
|
||||
"unsupported encoding",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sse_response_rejects_invalid_stream_headers_and_closes_upstream(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
headers: dict[str, str],
|
||||
message: str,
|
||||
) -> None:
|
||||
_set_config(monkeypatch)
|
||||
response = httpx.Response(200, headers=headers, stream=httpx.ByteStream(b"data"))
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", MagicMock(return_value=response))
|
||||
|
||||
with pytest.raises(KnowledgeFSTransportError, match=message):
|
||||
forward_knowledge_fs_request(
|
||||
account_id="account-dev",
|
||||
method="GET",
|
||||
path=_processing_task_events_path(),
|
||||
tenant_id="tenant-dev",
|
||||
)
|
||||
|
||||
assert response.is_closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_exception"),
|
||||
[
|
||||
@ -277,10 +671,10 @@ def test_transport_failures_are_normalized(
|
||||
("method", "path"),
|
||||
[
|
||||
("GET", "openapi.json"),
|
||||
("GET", "knowledge-spaces/space-1"),
|
||||
("GET", "knowledge-spaces/space-1/manifest"),
|
||||
("PATCH", "knowledge-spaces"),
|
||||
("POST", "queries"),
|
||||
("POST", "knowledge-spaces/space-1/documents"),
|
||||
("POST", "knowledge-spaces/space-1/uploads"),
|
||||
],
|
||||
)
|
||||
def test_unregistered_route_is_rejected_before_external_io(
|
||||
|
||||
12
packages/contracts/generated/knowledge-fs/metadata.gen.ts
Normal file
12
packages/contracts/generated/knowledge-fs/metadata.gen.ts
Normal file
@ -0,0 +1,12 @@
|
||||
// This file is auto-generated by scripts/generate-knowledge-fs-contract.mjs.
|
||||
// Do not edit it manually.
|
||||
|
||||
export const knowledgeFsSourceOpenapiSha256 =
|
||||
'f18910e9c45a64f0855e0643a7a626fb2889021b4f943458de86c6bd2469facb'
|
||||
export const knowledgeFsConsoleDeclarationsSha256 =
|
||||
'8bd1924747fdd0d478ca085817cbe000eb7e8630b2c6a03f4f13a6a0fac07946'
|
||||
export const knowledgeFsGeneratedArtifactSha256 = {
|
||||
'orpc.gen.ts': 'e0d9954f817e97a4e95dd38c4522fb403c8659d741ce485fa671b1b4ce90a540',
|
||||
'types.gen.ts': 'a558ab80f32a8555bb5b44b7a596ef4a4a7a8cb7904390993aabcc587915f530',
|
||||
'zod.gen.ts': '6aa3d3e768fe0008ca405ecbe423ee325bd8f65f745dd2d1c86f2bdb887a97fd',
|
||||
} as const
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
47
packages/contracts/knowledge-fs-contract.test.mjs
Normal file
47
packages/contracts/knowledge-fs-contract.test.mjs
Normal file
@ -0,0 +1,47 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
knowledgeFsGeneratedArtifactSha256,
|
||||
knowledgeFsSourceOpenapiSha256,
|
||||
} from './generated/knowledge-fs/metadata.gen'
|
||||
import { getStreamingOperationIds } from './scripts/knowledge-fs-contract-utils.mjs'
|
||||
|
||||
const packageRoot = dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
describe('KnowledgeFS contract generation', () => {
|
||||
it.each(['200', '2XX'])('detects an SSE response declared with %s', (status) => {
|
||||
expect(
|
||||
getStreamingOperationIds({
|
||||
paths: {
|
||||
'/tasks/{id}/events': {
|
||||
get: {
|
||||
operationId: 'streamTaskEvents',
|
||||
responses: {
|
||||
[status]: {
|
||||
content: {
|
||||
'text/event-stream': {},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual(['streamTaskEvents'])
|
||||
})
|
||||
|
||||
it('matches the pinned source contract and committed generated artifacts', async () => {
|
||||
const lock = JSON.parse(
|
||||
await readFile(join(packageRoot, '../../api/knowledge-fs-contract.lock.json'), 'utf8'),
|
||||
)
|
||||
|
||||
expect(knowledgeFsSourceOpenapiSha256).toBe(lock.openapiSha256)
|
||||
for (const [fileName, expectedSha256] of Object.entries(knowledgeFsGeneratedArtifactSha256)) {
|
||||
const content = await readFile(join(packageRoot, 'generated/knowledge-fs', fileName))
|
||||
expect(createHash('sha256').update(content).digest('hex'), fileName).toBe(expectedSha256)
|
||||
}
|
||||
})
|
||||
})
|
||||
@ -1,8 +1,10 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { getStreamingOperationIds } from './knowledge-fs-contract-utils.mjs'
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const workspaceRoot = resolve(packageRoot, '../..')
|
||||
@ -31,11 +33,83 @@ try {
|
||||
run('pnpm', ['exec', 'openapi-ts', '-f', 'openapi-ts.knowledge-fs.config.ts'], packageRoot, {
|
||||
KNOWLEDGE_FS_OPENAPI: openapiPath,
|
||||
})
|
||||
await patchStreamingContracts(openapiPath)
|
||||
run('pnpm', ['exec', 'vp', 'fmt', 'generated/knowledge-fs'], packageRoot)
|
||||
await writeContractMetadata(openapiPath, await generatedArtifactSha256())
|
||||
run('pnpm', ['exec', 'vp', 'fmt', 'generated/knowledge-fs/metadata.gen.ts'], packageRoot)
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { force: true, recursive: true })
|
||||
}
|
||||
|
||||
async function patchStreamingContracts(openapiPath) {
|
||||
const document = JSON.parse(await readFile(openapiPath, 'utf8'))
|
||||
const streamingOperationIds = getStreamingOperationIds(document)
|
||||
if (streamingOperationIds.length === 0) return
|
||||
|
||||
const outputPath = join(packageRoot, 'generated/knowledge-fs/orpc.gen.ts')
|
||||
let source = await readFile(outputPath, 'utf8')
|
||||
source = replaceOnce(
|
||||
source,
|
||||
"import { oc } from '@orpc/contract'",
|
||||
"import { eventIterator, oc } from '@orpc/contract'",
|
||||
)
|
||||
|
||||
for (const operationId of streamingOperationIds) {
|
||||
const responseSchema = `z${capitalize(operationId)}Response`
|
||||
source = replaceOnce(
|
||||
source,
|
||||
`.output(${responseSchema})`,
|
||||
`.output(eventIterator(${responseSchema}))`,
|
||||
)
|
||||
}
|
||||
|
||||
await writeFile(outputPath, source)
|
||||
}
|
||||
|
||||
function capitalize(value) {
|
||||
return value.charAt(0).toUpperCase() + value.slice(1)
|
||||
}
|
||||
|
||||
function replaceOnce(source, target, replacement) {
|
||||
const firstIndex = source.indexOf(target)
|
||||
if (firstIndex === -1 || source.indexOf(target, firstIndex + target.length) !== -1)
|
||||
throw new Error(`Expected exactly one generated occurrence of ${target}`)
|
||||
|
||||
return source.slice(0, firstIndex) + replacement + source.slice(firstIndex + target.length)
|
||||
}
|
||||
|
||||
async function generatedArtifactSha256() {
|
||||
const generatedDirectory = join(packageRoot, 'generated/knowledge-fs')
|
||||
const fileNames = (await readdir(generatedDirectory))
|
||||
.filter((fileName) => fileName.endsWith('.gen.ts') && fileName !== 'metadata.gen.ts')
|
||||
.sort()
|
||||
|
||||
return Object.fromEntries(
|
||||
await Promise.all(
|
||||
fileNames.map(async (fileName) => [
|
||||
fileName,
|
||||
createHash('sha256')
|
||||
.update(await readFile(join(generatedDirectory, fileName)))
|
||||
.digest('hex'),
|
||||
]),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
async function writeContractMetadata(openapiPath, artifactSha256) {
|
||||
const document = JSON.parse(await readFile(openapiPath, 'utf8'))
|
||||
const source = [
|
||||
'// This file is auto-generated by scripts/generate-knowledge-fs-contract.mjs.',
|
||||
'// Do not edit it manually.',
|
||||
'',
|
||||
`export const knowledgeFsSourceOpenapiSha256 = ${JSON.stringify(document['x-dify-source-openapi-sha256'])}`,
|
||||
`export const knowledgeFsConsoleDeclarationsSha256 = ${JSON.stringify(document['x-dify-console-declarations-sha256'])}`,
|
||||
`export const knowledgeFsGeneratedArtifactSha256 = ${JSON.stringify(artifactSha256, null, 2)} as const`,
|
||||
'',
|
||||
].join('\n')
|
||||
await writeFile(join(packageRoot, 'generated/knowledge-fs/metadata.gen.ts'), source)
|
||||
}
|
||||
|
||||
function run(command, args, cwd, extraEnv = {}) {
|
||||
execFileSync(command, args, {
|
||||
cwd,
|
||||
|
||||
19
packages/contracts/scripts/knowledge-fs-contract-utils.mjs
Normal file
19
packages/contracts/scripts/knowledge-fs-contract-utils.mjs
Normal file
@ -0,0 +1,19 @@
|
||||
export function getStreamingOperationIds(document) {
|
||||
return Object.values(document.paths ?? {})
|
||||
.flatMap((pathItem) =>
|
||||
Object.values(pathItem).flatMap((operation) => {
|
||||
if (typeof operation !== 'object' || operation === null) return []
|
||||
const isEventStream = Object.entries(operation.responses ?? {}).some(
|
||||
([status, response]) =>
|
||||
(status === '2XX' || /^2\d\d$/.test(status)) &&
|
||||
typeof response === 'object' &&
|
||||
response !== null &&
|
||||
'text/event-stream' in (response.content ?? {}),
|
||||
)
|
||||
return isEventStream && typeof operation.operationId === 'string'
|
||||
? [operation.operationId]
|
||||
: []
|
||||
}),
|
||||
)
|
||||
.sort()
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
import type { ApiBasedExtensionResponse } from '@dify/contracts/api/console/api-based-extension/types.gen'
|
||||
import type { TagResponse as Tag } from '@dify/contracts/api/console/tags/types.gen'
|
||||
import type { DocumentProcessingTaskEvent } from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import type { MutationFunctionContext, QueryFunctionContext } from '@tanstack/react-query'
|
||||
import type { consoleQuery as ConsoleQuery } from './client'
|
||||
import { QueryClient } from '@tanstack/react-query'
|
||||
@ -358,6 +359,74 @@ describe('consoleQuery transport context', () => {
|
||||
)
|
||||
expect(request.mock.calls[0]![0]).not.toContain('ids%5B0%5D')
|
||||
})
|
||||
|
||||
it('should consume KnowledgeFS processing events through the generated stream contract', async () => {
|
||||
const request = vi.fn().mockResolvedValue(
|
||||
new Response(
|
||||
[
|
||||
'id: task-1:1',
|
||||
'event: message',
|
||||
'data: {"event":"progress","data":{"progressPercent":25,"stage":"parsed","state":"running","updatedAt":"2026-07-22T10:00:00.000Z"}}',
|
||||
'',
|
||||
'id: task-1:terminal',
|
||||
'event: message',
|
||||
'data: {"event":"terminal","data":{"state":"succeeded"}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n'),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'text/event-stream',
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
const consoleQuery = await loadConsoleQueryWithRequest(request)
|
||||
const queryOptions =
|
||||
consoleQuery.knowledgeFs.getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents.experimental_streamedOptions(
|
||||
{
|
||||
input: {
|
||||
headers: {
|
||||
'last-event-id': 'task-1:0',
|
||||
},
|
||||
params: {
|
||||
documentId: 'document-1',
|
||||
id: 'space-1',
|
||||
taskId: 'task-1',
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
const events = await queryOptions.queryFn({
|
||||
client: new QueryClient(),
|
||||
signal: new AbortController().signal,
|
||||
} as QueryFunctionContext)
|
||||
|
||||
expectTypeOf(events[0]!).toMatchTypeOf<DocumentProcessingTaskEvent>()
|
||||
expect(events).toEqual([
|
||||
{
|
||||
data: {
|
||||
progressPercent: 25,
|
||||
stage: 'parsed',
|
||||
state: 'running',
|
||||
updatedAt: '2026-07-22T10:00:00.000Z',
|
||||
},
|
||||
event: 'progress',
|
||||
},
|
||||
{ data: { state: 'succeeded' }, event: 'terminal' },
|
||||
])
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'/knowledge-fs/knowledge-spaces/space-1/documents/document-1/processing-tasks/task-1/events',
|
||||
),
|
||||
expect.any(Object),
|
||||
expect.objectContaining({
|
||||
fetchCompat: true,
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
// Scenario: console OpenAPI query arrays follow backend parser expectations.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user