opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-05 08:48:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(dataset): integrate New RAG with KnowledgeFS (#39621)

Browse Source
This commit is contained in:
Stephen Zhou 2026-07-27 12:52:44 +08:00 committed by GitHub
parent ce0dee9afb
commit 1cde846bcf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
154 changed files with 8666 additions and 9154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/.env.example
View File

@ -692,8 +692,6 @@ KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
# Legacy rollback-only HMAC; Capability v2 deployments leave this blank.
KNOWLEDGE_FS_JWT_SECRET=
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
@ -701,7 +699,6 @@ KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304

6
api/configs/extra/knowledge_fs_config.py
View File

@ -12,7 +12,7 @@ class KnowledgeFSConfig(BaseSettings):
KNOWLEDGE_FS_ENABLED: bool = Field(
default=False,
description="Enable the private KnowledgeFS Console bridge.",
description="Enable the KnowledgeFS control-plane product routes.",
)
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
default=False,
@ -34,6 +34,10 @@ class KnowledgeFSConfig(BaseSettings):
default=None,
description="Public KnowledgeFS origin returned with direct upload capabilities.",
)
KNOWLEDGE_FS_DIRECT_UPLOAD_READY: bool = Field(
default=False,
description="Confirm that KnowledgeFS direct upload and its browser origin policy are deployed and verified.",
)
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
default=False,
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",

2
api/constants/__init__.py
View File

@ -77,3 +77,5 @@ COOKIE_NAME_PASSPORT = "passport"
HEADER_NAME_CSRF_TOKEN = "X-CSRF-Token"
HEADER_NAME_APP_CODE = "X-App-Code"
HEADER_NAME_PASSPORT = "X-App-Passport"
HEADER_NAME_IDEMPOTENCY_KEY = "Idempotency-Key"
HEADER_NAME_REQUEST_ID = "X-Request-ID"

387
api/controllers/console/knowledge_fs/resources.py
View File

@ -64,6 +64,9 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSBulkDocumentDeletePayload,
KnowledgeFSBulkJobResponse,
KnowledgeFSCapabilityResponse,
KnowledgeFSCrawlPreviewPageListQuery,
KnowledgeFSCrawlPreviewPageListResponse,
KnowledgeFSCrawlPreviewSelectionPayload,
KnowledgeFSCredentialCreatePayload,
KnowledgeFSCredentialCreateResponse,
KnowledgeFSCredentialListResponse,
@ -86,6 +89,7 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSExternalAccessResponse,
KnowledgeFSIdempotencyHeader,
KnowledgeFSJWKSResponse,
KnowledgeFSLogicalDocumentListResponse,
KnowledgeFSLogicalDocumentResponse,
KnowledgeFSMembersReplacePayload,
KnowledgeFSOverviewBaseStatsResponse,
@ -111,6 +115,11 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSSettingsPayload,
KnowledgeFSSettingsResponse,
KnowledgeFSSmallFileUploadResponse,
KnowledgeFSSourceConnectionCreatePayload,
KnowledgeFSSourceConnectionListQuery,
KnowledgeFSSourceConnectionListResponse,
KnowledgeFSSourceConnectionRefreshPayload,
KnowledgeFSSourceConnectionResponse,
KnowledgeFSSourceCrawlResponse,
KnowledgeFSSourceCreatePayload,
KnowledgeFSSourceCredentialTestResponse,
@ -124,8 +133,13 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSSourceListResponse,
KnowledgeFSSourcePagesQuery,
KnowledgeFSSourcePagesResponse,
KnowledgeFSSourceProviderListResponse,
KnowledgeFSSourceResponse,
KnowledgeFSSourceSyncPolicyPayload,
KnowledgeFSSourceSyncPolicyResponse,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSourceWorkflowCancelPayload,
KnowledgeFSSourceWorkflowResponse,
KnowledgeFSSpaceCreatePayload,
KnowledgeFSSpaceCreateResponse,
KnowledgeFSSpaceDetailResponse,
@ -143,6 +157,7 @@ from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemoteError,
KnowledgeFSProductRequestRejectedError,
KnowledgeFSProductResourceNotFoundError,
)
from services.knowledge_fs.runtime import KnowledgeFSRuntime, create_knowledge_fs_runtime
from services.knowledge_fs_capability import (
@ -164,6 +179,7 @@ register_schema_models(
KnowledgeFSDocumentMetadataPayload,
KnowledgeFSDocumentReindexPayload,
KnowledgeFSExternalAccessPayload,
KnowledgeFSCrawlPreviewPageListQuery,
KnowledgeFSMembersReplacePayload,
KnowledgeFSQueryCreatePayload,
KnowledgeFSResearchTaskPartialsQuery,
@ -171,6 +187,10 @@ register_schema_models(
KnowledgeFSResearchTaskCreatePayload,
KnowledgeFSSettingsPayload,
KnowledgeFSSourceCreatePayload,
KnowledgeFSSourceConnectionCreatePayload,
KnowledgeFSSourceConnectionListQuery,
KnowledgeFSSourceConnectionRefreshPayload,
KnowledgeFSCrawlPreviewSelectionPayload,
KnowledgeFSSourceDeletePayload,
KnowledgeFSSourceDeleteQuery,
KnowledgeFSSourceFilesQuery,
@ -178,6 +198,8 @@ register_schema_models(
KnowledgeFSSourceImportPagesPayload,
KnowledgeFSSourcePagesQuery,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSourceSyncPolicyPayload,
KnowledgeFSSourceWorkflowCancelPayload,
KnowledgeFSSpaceCreatePayload,
KnowledgeFSSpaceListQuery,
KnowledgeFSSpaceUpdatePayload,
@ -208,6 +230,7 @@ register_response_schema_models(
KnowledgeFSDurableDeletionAcceptedResponse,
KnowledgeFSExternalAccessResponse,
KnowledgeFSJWKSResponse,
KnowledgeFSLogicalDocumentListResponse,
KnowledgeFSPermissionListResponse,
KnowledgeFSQueryResponse,
KnowledgeFSQueryAdmissionResponse,
@ -218,6 +241,9 @@ register_response_schema_models(
KnowledgeFSResearchTaskListResponse,
KnowledgeFSSettingsResponse,
KnowledgeFSSmallFileUploadResponse,
KnowledgeFSCrawlPreviewPageListResponse,
KnowledgeFSSourceConnectionListResponse,
KnowledgeFSSourceConnectionResponse,
KnowledgeFSSourceListResponse,
KnowledgeFSSourceCrawlResponse,
KnowledgeFSSourceCredentialTestResponse,
@ -225,6 +251,9 @@ register_response_schema_models(
KnowledgeFSSourceImportResponse,
KnowledgeFSSourcePagesResponse,
KnowledgeFSSourceResponse,
KnowledgeFSSourceProviderListResponse,
KnowledgeFSSourceSyncPolicyResponse,
KnowledgeFSSourceWorkflowResponse,
KnowledgeFSSpaceCreateResponse,
KnowledgeFSSpaceDetailResponse,
KnowledgeFSSpaceListResponse,
@ -255,6 +284,8 @@ def _knowledge_fs_errors[**P, R](view: Callable[P, R]) -> Callable[P, R]:
raise KnowledgeFSSpaceNotFoundHTTPError() from exc
except KnowledgeFSOperationUnavailableError as exc:
raise KnowledgeFSOperationUnavailableHTTPError() from exc
except KnowledgeFSProductResourceNotFoundError as exc:
raise NotFound() from exc
except KnowledgeFSProductRemoteError as exc:
raise KnowledgeFSUpstreamUnavailableHTTPError() from exc
except KnowledgeFSOperationRateLimitExceededError as exc:
@ -290,6 +321,14 @@ _SMALL_FILE_UPLOAD_PARAMS = {
"required": True,
}
}
_IDEMPOTENCY_HEADER_PARAMS = {
"Idempotency-Key": {
"description": "Stable key used to make the mutation safe to retry",
"in": "header",
"required": True,
"type": "string",
}
}
_SMALL_FILE_MULTIPART_OVERHEAD_MAX_BYTES = 64 * 1024
_BACKGROUND_TASK_KIND_ADAPTER: TypeAdapter[Literal["document", "document_bulk", "source"]] = TypeAdapter(
Literal["document", "document_bulk", "source"]
@ -822,6 +861,52 @@ class KnowledgeFSSpaceOverviewHealthApi(Resource):
return dump_response(KnowledgeFSOverviewHealthResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/logical-documents")
class KnowledgeFSSpaceLogicalDocumentsApi(Resource):
@console_ns.doc(params=query_params_from_model(KnowledgeFSCursorQuery))
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS logical documents",
console_ns.models[KnowledgeFSLogicalDocumentListResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str):
actor_id, tenant_id = _actor()
query = KnowledgeFSCursorQuery.model_validate(request.args.to_dict())
result = _console_services().facade.list_logical_documents(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
cursor=query.cursor,
)
return dump_response(KnowledgeFSLogicalDocumentListResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/logical-documents/<string:document_id>")
class KnowledgeFSSpaceLogicalDocumentApi(Resource):
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS logical document",
console_ns.models[KnowledgeFSLogicalDocumentResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str, document_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.get_logical_document(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
document_id=document_id,
)
return dump_response(KnowledgeFSLogicalDocumentResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/documents")
class KnowledgeFSSpaceDocumentsApi(Resource):
@console_ns.doc(params=query_params_from_model(KnowledgeFSCursorQuery))
@ -866,6 +951,7 @@ class KnowledgeFSSpaceDocumentsApi(Resource):
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/documents/bulk")
class KnowledgeFSSpaceBulkDocumentsApi(Resource):
@console_ns.expect(console_ns.models[KnowledgeFSBulkDocumentDeletePayload.__name__])
@console_ns.doc(params=_IDEMPOTENCY_HEADER_PARAMS)
@console_ns.response(
HTTPStatus.ACCEPTED,
"KnowledgeFS document deletions accepted",
@ -953,6 +1039,7 @@ class KnowledgeFSSpaceDocumentApi(Resource):
return dump_response(KnowledgeFSLogicalDocumentResponse, result)
@console_ns.expect(console_ns.models[KnowledgeFSDocumentDeletePayload.__name__])
@console_ns.doc(params=_IDEMPOTENCY_HEADER_PARAMS)
@console_ns.response(
HTTPStatus.ACCEPTED,
"KnowledgeFS document deletion accepted",
@ -1225,6 +1312,96 @@ class KnowledgeFSSpaceBackgroundTaskRetryApi(Resource):
return dump_response(KnowledgeFSBackgroundTaskResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-providers")
class KnowledgeFSSourceProvidersApi(Resource):
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source providers",
console_ns.models[KnowledgeFSSourceProviderListResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.list_source_providers(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
)
return dump_response(KnowledgeFSSourceProviderListResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-connections")
class KnowledgeFSSourceConnectionsApi(Resource):
@console_ns.doc(params=query_params_from_model(KnowledgeFSSourceConnectionListQuery))
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source connections",
console_ns.models[KnowledgeFSSourceConnectionListResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str):
actor_id, tenant_id = _actor()
query = KnowledgeFSSourceConnectionListQuery.model_validate(request.args.to_dict())
result = _console_services().facade.list_source_connections(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
cursor=query.cursor,
limit=query.limit,
)
return dump_response(KnowledgeFSSourceConnectionListResponse, result)
@console_ns.expect(console_ns.models[KnowledgeFSSourceConnectionCreatePayload.__name__])
@console_ns.response(
HTTPStatus.CREATED,
"KnowledgeFS source connection created",
console_ns.models[KnowledgeFSSourceConnectionResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self, control_space_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.create_source_connection(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
payload=_payload(KnowledgeFSSourceConnectionCreatePayload),
)
return dump_response(KnowledgeFSSourceConnectionResponse, result), HTTPStatus.CREATED
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-connections/<string:connection_id>/refresh")
class KnowledgeFSSourceConnectionRefreshApi(Resource):
@console_ns.expect(console_ns.models[KnowledgeFSSourceConnectionRefreshPayload.__name__])
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source connection refreshed",
console_ns.models[KnowledgeFSSourceConnectionResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self, control_space_id: str, connection_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.refresh_source_connection(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
connection_id=connection_id,
payload=_payload(KnowledgeFSSourceConnectionRefreshPayload),
)
return dump_response(KnowledgeFSSourceConnectionResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/sources")
class KnowledgeFSSpaceSourcesApi(Resource):
@console_ns.doc(params=query_params_from_model(KnowledgeFSCursorQuery))
@ -1303,7 +1480,7 @@ class KnowledgeFSSpaceSourceApi(Resource):
return dump_response(KnowledgeFSSourceResponse, result)
@console_ns.expect(console_ns.models[KnowledgeFSSourceDeletePayload.__name__])
@console_ns.doc(params=query_params_from_model(KnowledgeFSSourceDeleteQuery))
@console_ns.doc(params=query_params_from_model(KnowledgeFSSourceDeleteQuery) | _IDEMPOTENCY_HEADER_PARAMS)
@console_ns.response(
HTTPStatus.ACCEPTED,
"KnowledgeFS source deletion accepted",
@ -1347,10 +1524,13 @@ class KnowledgeFSSpaceSourceTestApi(Resource):
return dump_response(KnowledgeFSSourceCredentialTestResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/crawl")
class KnowledgeFSSpaceSourceCrawlApi(Resource):
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/sync")
class KnowledgeFSSpaceSourceSyncApi(Resource):
@console_ns.doc(params=_IDEMPOTENCY_HEADER_PARAMS)
@console_ns.response(
HTTPStatus.OK, "KnowledgeFS source crawl", console_ns.models[KnowledgeFSSourceCrawlResponse.__name__]
HTTPStatus.ACCEPTED,
"KnowledgeFS source sync accepted",
console_ns.models[KnowledgeFSSourceWorkflowResponse.__name__],
)
@setup_required
@login_required
@ -1358,10 +1538,201 @@ class KnowledgeFSSpaceSourceCrawlApi(Resource):
@_knowledge_fs_errors
def post(self, control_space_id: str, source_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.crawl_source(
tenant_id=tenant_id, account_id=actor_id, control_space_id=control_space_id, source_id=source_id
result = _console_services().facade.sync_source(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
source_id=source_id,
idempotency_key=_idempotency_key(),
)
return dump_response(KnowledgeFSSourceCrawlResponse, result)
return dump_response(KnowledgeFSSourceWorkflowResponse, result), HTTPStatus.ACCEPTED
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/crawl-preview")
class KnowledgeFSSpaceSourceCrawlPreviewApi(Resource):
@console_ns.doc(params=_IDEMPOTENCY_HEADER_PARAMS)
@console_ns.response(
HTTPStatus.ACCEPTED,
"KnowledgeFS source crawl preview accepted",
console_ns.models[KnowledgeFSSourceWorkflowResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self, control_space_id: str, source_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.preview_source_crawl(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
source_id=source_id,
idempotency_key=_idempotency_key(),
)
return dump_response(KnowledgeFSSourceWorkflowResponse, result), HTTPStatus.ACCEPTED
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/sync-policy")
class KnowledgeFSSpaceSourceSyncPolicyApi(Resource):
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source sync policy",
console_ns.models[KnowledgeFSSourceSyncPolicyResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str, source_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.get_source_sync_policy(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
source_id=source_id,
)
return dump_response(KnowledgeFSSourceSyncPolicyResponse, result)
@console_ns.expect(console_ns.models[KnowledgeFSSourceSyncPolicyPayload.__name__])
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source sync policy updated",
console_ns.models[KnowledgeFSSourceSyncPolicyResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def put(self, control_space_id: str, source_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.update_source_sync_policy(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
source_id=source_id,
payload=_payload(KnowledgeFSSourceSyncPolicyPayload),
)
return dump_response(KnowledgeFSSourceSyncPolicyResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>")
class KnowledgeFSSourceWorkflowApi(Resource):
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source workflow",
console_ns.models[KnowledgeFSSourceWorkflowResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str, run_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.get_source_workflow(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
run_id=run_id,
)
return dump_response(KnowledgeFSSourceWorkflowResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/cancel")
class KnowledgeFSSourceWorkflowCancelApi(Resource):
@console_ns.expect(console_ns.models[KnowledgeFSSourceWorkflowCancelPayload.__name__])
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source workflow canceled",
console_ns.models[KnowledgeFSSourceWorkflowResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self, control_space_id: str, run_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.cancel_source_workflow(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
run_id=run_id,
payload=_payload(KnowledgeFSSourceWorkflowCancelPayload),
)
return dump_response(KnowledgeFSSourceWorkflowResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/retry")
class KnowledgeFSSourceWorkflowRetryApi(Resource):
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS source workflow retried",
console_ns.models[KnowledgeFSSourceWorkflowResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self, control_space_id: str, run_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.retry_source_workflow(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
run_id=run_id,
)
return dump_response(KnowledgeFSSourceWorkflowResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/pages")
class KnowledgeFSSourceWorkflowPagesApi(Resource):
@console_ns.doc(params=query_params_from_model(KnowledgeFSCrawlPreviewPageListQuery))
@console_ns.response(
HTTPStatus.OK,
"KnowledgeFS crawl preview pages",
console_ns.models[KnowledgeFSCrawlPreviewPageListResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def get(self, control_space_id: str, run_id: str):
actor_id, tenant_id = _actor()
query = KnowledgeFSCrawlPreviewPageListQuery.model_validate(request.args.to_dict())
result = _console_services().facade.list_crawl_preview_pages(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
run_id=run_id,
cursor=query.cursor,
limit=query.limit,
)
return dump_response(KnowledgeFSCrawlPreviewPageListResponse, result)
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/selection")
class KnowledgeFSSourceWorkflowSelectionApi(Resource):
@console_ns.expect(console_ns.models[KnowledgeFSCrawlPreviewSelectionPayload.__name__])
@console_ns.doc(params=_IDEMPOTENCY_HEADER_PARAMS)
@console_ns.response(
HTTPStatus.ACCEPTED,
"KnowledgeFS crawl preview selection accepted",
console_ns.models[KnowledgeFSSourceWorkflowResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self, control_space_id: str, run_id: str):
actor_id, tenant_id = _actor()
result = _console_services().facade.select_crawl_preview_pages(
tenant_id=tenant_id,
account_id=actor_id,
control_space_id=control_space_id,
run_id=run_id,
payload=_payload(KnowledgeFSCrawlPreviewSelectionPayload),
idempotency_key=_idempotency_key(),
)
return dump_response(KnowledgeFSSourceWorkflowResponse, result), HTTPStatus.ACCEPTED
@console_ns.route("/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/pages")
@ -1760,7 +2131,7 @@ class KnowledgeFSSpaceUploadCapabilitiesApi(Resource):
@_knowledge_fs_errors
def post(self, control_space_id: str):
direct_origin = dify_config.KNOWLEDGE_FS_DIRECT_ORIGIN
if direct_origin is None:
if direct_origin is None or not dify_config.KNOWLEDGE_FS_DIRECT_UPLOAD_READY:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS direct upload is not configured")
actor_id, tenant_id = _actor()
payload = _payload(KnowledgeFSUploadCapabilityPayload)

24
api/controllers/inner_api/plugin/wraps.py
View File

@ -1,5 +1,6 @@
from collections.abc import Callable
from functools import wraps
from uuid import UUID
from flask import current_app, request
from flask_login import user_logged_in
@ -55,14 +56,15 @@ def get_user(tenant_id: str, user_id: str | None) -> EndUser:
# session_id, id is auto-generated) and a fresh EndUser
# was created per call, breaking multi-turn chat
# continuation (see #36736).
user_model = session.scalar(
select(EndUser)
.where(
EndUser.id == user_id,
EndUser.tenant_id == tenant_id,
if _is_uuid(user_id):
user_model = session.scalar(
select(EndUser)
.where(
EndUser.id == user_id,
EndUser.tenant_id == tenant_id,
)
.limit(1)
)
.limit(1)
)
if user_model is None:
user_model = session.scalar(
select(EndUser)
@ -90,6 +92,14 @@ def get_user(tenant_id: str, user_id: str | None) -> EndUser:
return user_model
def _is_uuid(value: str) -> bool:
try:
UUID(value)
except ValueError:
return False
return True
def get_user_tenant[**P, R](view_func: Callable[P, R]) -> Callable[P, R]:
@wraps(view_func)
def decorated_view(*args: P.args, **kwargs: P.kwargs) -> R:

3
api/controllers/service_api/knowledge_fs/resources.py
View File

@ -95,6 +95,7 @@ from services.knowledge_fs.product_operations import product_operation_action
from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemoteError,
KnowledgeFSProductResourceNotFoundError,
)
from services.knowledge_fs.runtime import KnowledgeFSRuntime, create_knowledge_fs_runtime
@ -171,6 +172,8 @@ def _service_api_errors[**P, R](view: Callable[P, R]) -> Callable[P, R]:
raise KnowledgeFSInvalidCredentialHTTPError() from exc
except KnowledgeFSOperationUnavailableError as exc:
raise KnowledgeFSServiceOperationUnavailableHTTPError() from exc
except KnowledgeFSProductResourceNotFoundError as exc:
raise NotFound() from exc
except KnowledgeFSProductRemoteError as exc:
raise KnowledgeFSServiceUpstreamUnavailableHTTPError() from exc
except KnowledgeFSOperationRateLimitExceededError as exc:

17
api/extensions/ext_blueprints.py
View File

@ -1,10 +1,23 @@
"""Register API blueprints with their browser-facing CORS policies."""
from configs import dify_config
from constants import HEADER_NAME_APP_CODE, HEADER_NAME_CSRF_TOKEN, HEADER_NAME_PASSPORT
from constants import (
HEADER_NAME_APP_CODE,
HEADER_NAME_CSRF_TOKEN,
HEADER_NAME_IDEMPOTENCY_KEY,
HEADER_NAME_PASSPORT,
HEADER_NAME_REQUEST_ID,
)
from dify_app import DifyApp
BASE_CORS_HEADERS: tuple[str, ...] = ("Content-Type", HEADER_NAME_APP_CODE, HEADER_NAME_PASSPORT)
SERVICE_API_HEADERS: tuple[str, ...] = (*BASE_CORS_HEADERS, "Authorization")
AUTHENTICATED_HEADERS: tuple[str, ...] = (*SERVICE_API_HEADERS, HEADER_NAME_CSRF_TOKEN)
AUTHENTICATED_HEADERS: tuple[str, ...] = (
*SERVICE_API_HEADERS,
HEADER_NAME_CSRF_TOKEN,
HEADER_NAME_IDEMPOTENCY_KEY,
HEADER_NAME_REQUEST_ID,
)
FILES_HEADERS: tuple[str, ...] = (*BASE_CORS_HEADERS, HEADER_NAME_CSRF_TOKEN)
EMBED_HEADERS: tuple[str, ...] = ("Content-Type", HEADER_NAME_APP_CODE)
EXPOSED_HEADERS: tuple[str, ...] = ("X-Version", "X-Env", "X-Trace-Id")

6
api/knowledge-fs-contract.lock.json
View File

@ -1,9 +1,9 @@
{
"schemaVersion": 5,
"subtreeTree": "4a5f77139bfa81192ca1151aa2cea8aca8a19501",
"openapiSha256": "0a03c2cdc027c8d8d792da97db65f642c2e28ecfd8c4a9bc08aafb53fc38a64b",
"subtreeTree": "2745077bf08ffb7143abe8bd2e14fa235136d276",
"openapiSha256": "5e6d37b22f3e0441492bd928429899d890dfcda8d4645c671fa4d128faa8947d",
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
"productOperationManifestSha256": "8dc392325682a307702bc53420616123ab875e6875d3a473a453dca1ac40c61e",
"productOperationManifestSha256": "7b46eaf900d3db5b518d8ab52e3265b7c7bd0b5fbdb831e5647422c9f8975427",
"productOperationGapManifestSha256": "ccbae37fe658177a77529822211a2cf02e72b5815cbc9b7e7e65e6817ba5e0a9"
}

15
api/knowledge-fs-product-operations.json
View File

@ -11,6 +11,8 @@
{"productOperationId":"getOverviewHealth","kfsOperationId":"getKnowledgeSpaceProductHealth","method":"GET","path":"/knowledge-spaces/{id}/overview/health","action":"knowledge_spaces.overview.health.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSettings","kfsOperationId":"updateKnowledgeSpaceProductSettings","method":"PATCH","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocuments","kfsOperationId":"listDocuments","method":"GET","path":"/knowledge-spaces/{id}/documents","action":"documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listLogicalDocuments","kfsOperationId":"listLogicalDocuments","method":"GET","path":"/knowledge-spaces/{id}/logical-documents","action":"logical_documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getLogicalDocument","kfsOperationId":"getLogicalDocument","method":"GET","path":"/knowledge-spaces/{id}/logical-documents/{documentId}","action":"logical_documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocument","kfsOperationId":"getDocument","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocumentOutline","kfsOperationId":"getDocumentOutline","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/outline","action":"documents.outline.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocumentRevisions","kfsOperationId":"listDocumentRevisions","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions","action":"documents.revisions.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
@ -33,6 +35,19 @@
{"productOperationId":"updateSource","kfsOperationId":"updateKnowledgeSpaceSource","method":"PATCH","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.update","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"deleteSource","kfsOperationId":"requestSourceDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.delete","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"testSource","kfsOperationId":"testKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/test","action":"sources.test","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"syncSource","kfsOperationId":"createSourceSyncWorkflow","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/sync","action":"source_workflows.sync.create","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourceProviders","kfsOperationId":"listSourceProviders","method":"GET","path":"/source-providers","action":"source_providers.list","resource":"namespace","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createSourceConnection","kfsOperationId":"createSourceConnection","method":"POST","path":"/knowledge-spaces/{id}/source-connections","action":"source_connections.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourceConnections","kfsOperationId":"listSourceConnections","method":"GET","path":"/knowledge-spaces/{id}/source-connections","action":"source_connections.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"refreshSourceConnection","kfsOperationId":"refreshSourceConnection","method":"POST","path":"/knowledge-spaces/{id}/source-connections/{connectionId}/refresh","action":"source_connections.refresh","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"previewSourceCrawl","kfsOperationId":"createSourceCrawlPreviewWorkflow","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/crawl-preview","action":"source_workflows.preview.create","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSourceSyncPolicy","kfsOperationId":"getSourceSyncPolicy","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/sync-policy","action":"source_sync_policies.read","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSourceSyncPolicy","kfsOperationId":"putSourceSyncPolicy","method":"PUT","path":"/knowledge-spaces/{id}/sources/{sourceId}/sync-policy","action":"source_sync_policies.update","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSourceWorkflow","kfsOperationId":"getSourceWorkflow","method":"GET","path":"/knowledge-spaces/{id}/source-workflows/{runId}","action":"source_workflows.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"cancelSourceWorkflow","kfsOperationId":"cancelSourceWorkflow","method":"POST","path":"/knowledge-spaces/{id}/source-workflows/{runId}/cancel","action":"source_workflows.cancel","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"retrySourceWorkflow","kfsOperationId":"retrySourceWorkflow","method":"POST","path":"/knowledge-spaces/{id}/source-workflows/{runId}/retry","action":"source_workflows.retry","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listCrawlPreviewPages","kfsOperationId":"listCrawlPreviewPages","method":"GET","path":"/knowledge-spaces/{id}/source-workflows/{runId}/pages","action":"source_workflows.pages.list","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"selectCrawlPreviewPages","kfsOperationId":"selectCrawlPreviewPages","method":"POST","path":"/knowledge-spaces/{id}/source-workflows/{runId}/selection","action":"source_workflows.selection.create","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"crawlSource","kfsOperationId":"crawlKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/crawl","action":"sources.crawl","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourcePages","kfsOperationId":"listKnowledgeSpaceSourcePages","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/pages","action":"sources.pages.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"importSourcePages","kfsOperationId":"importKnowledgeSpaceSourcePages","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import","action":"sources.pages.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},

8
api/services/feature_service.py
View File

@ -186,6 +186,8 @@ class SystemFeatureModel(FeatureResponseModel):
enable_learn_app: bool = True
enable_step_by_step_tour: bool = False
rbac_enabled: bool = False
knowledge_fs_enabled: bool = False
knowledge_fs_upload_enabled: bool = False
class FeatureService:
@ -289,6 +291,12 @@ class FeatureService:
system_features.enable_learn_app = dify_config.ENABLE_LEARN_APP
system_features.webapp_auth.allow_public_access = dify_config.WEBAPP_PUBLIC_ACCESS_ENABLED
system_features.enable_step_by_step_tour = dify_config.ENABLE_STEP_BY_STEP_TOUR
system_features.knowledge_fs_enabled = dify_config.KNOWLEDGE_FS_ENABLED
system_features.knowledge_fs_upload_enabled = bool(
dify_config.KNOWLEDGE_FS_ENABLED
and dify_config.KNOWLEDGE_FS_DIRECT_ORIGIN
and dify_config.KNOWLEDGE_FS_DIRECT_UPLOAD_READY
)
@classmethod
def _fulfill_trial_models_from_env(cls) -> list[str]:

4
api/services/knowledge_fs/capability_broker.py
View File

@ -422,7 +422,9 @@ def _issue_request(
trace_id: str,
) -> CapabilityIssueRequest:
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
if capability_operation.resource_type == "knowledge_space":
if capability_operation.resource_type == "namespace":
resource = CapabilityResource(type="namespace", id=tenant_id)
elif capability_operation.resource_type == "knowledge_space":
resource = CapabilityResource(type="knowledge_space", id=knowledge_space_id)
elif capability_operation.resource_type in {
"document",

271
api/services/knowledge_fs/data_facade.py
View File

@ -18,6 +18,8 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSBulkDeletionAcceptedResponse,
KnowledgeFSBulkDocumentDeletePayload,
KnowledgeFSBulkJobResponse,
KnowledgeFSCrawlPreviewPageListResponse,
KnowledgeFSCrawlPreviewSelectionPayload,
KnowledgeFSDocumentChunkListResponse,
KnowledgeFSDocumentChunkResponse,
KnowledgeFSDocumentCompilationJobResponse,
@ -31,6 +33,7 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSDocumentResponse,
KnowledgeFSDocumentRevisionListResponse,
KnowledgeFSDurableDeletionAcceptedResponse,
KnowledgeFSLogicalDocumentListResponse,
KnowledgeFSLogicalDocumentResponse,
KnowledgeFSOverviewBaseStatsResponse,
KnowledgeFSOverviewHealthResponse,
@ -47,6 +50,10 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSSettingsPayload,
KnowledgeFSSettingsResponse,
KnowledgeFSSmallFileUploadResponse,
KnowledgeFSSourceConnectionCreatePayload,
KnowledgeFSSourceConnectionListResponse,
KnowledgeFSSourceConnectionRefreshPayload,
KnowledgeFSSourceConnectionResponse,
KnowledgeFSSourceCrawlResponse,
KnowledgeFSSourceCreatePayload,
KnowledgeFSSourceCredentialTestResponse,
@ -57,8 +64,13 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSSourceImportResponse,
KnowledgeFSSourceListResponse,
KnowledgeFSSourcePagesResponse,
KnowledgeFSSourceProviderListResponse,
KnowledgeFSSourceResponse,
KnowledgeFSSourceSyncPolicyPayload,
KnowledgeFSSourceSyncPolicyResponse,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSourceWorkflowCancelPayload,
KnowledgeFSSourceWorkflowResponse,
KnowledgeFSSpaceUpdatePayload,
KnowledgeFSTraceEntryListResponse,
KnowledgeFSTraceListResponse,
@ -194,6 +206,36 @@ class KnowledgeFSDataFacade:
)
return KnowledgeFSDocumentListResponse.model_validate(raw)
def list_logical_documents(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
cursor: str | None,
) -> KnowledgeFSLogicalDocumentListResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listLogicalDocuments",
query=(("cursor", cursor),) if cursor else (),
)
return KnowledgeFSLogicalDocumentListResponse.model_validate(raw)
def get_logical_document(
self, *, tenant_id: str, account_id: str, control_space_id: str, document_id: str
) -> KnowledgeFSLogicalDocumentResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getLogicalDocument",
resource_id=document_id,
path_parameters=(("documentId", document_id),),
)
return KnowledgeFSLogicalDocumentResponse.model_validate(raw)
def create_document(
self,
*,
@ -631,6 +673,235 @@ class KnowledgeFSDataFacade:
)
return KnowledgeFSSourceCredentialTestResponse.model_validate(raw)
def sync_source(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
idempotency_key: str,
) -> KnowledgeFSSourceWorkflowResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="syncSource",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
headers=(("Idempotency-Key", idempotency_key),),
)
return KnowledgeFSSourceWorkflowResponse.model_validate(raw)
def list_source_providers(
self, *, tenant_id: str, account_id: str, control_space_id: str
) -> KnowledgeFSSourceProviderListResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listSourceProviders",
)
return KnowledgeFSSourceProviderListResponse.model_validate(raw)
def create_source_connection(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSSourceConnectionCreatePayload,
) -> KnowledgeFSSourceConnectionResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="createSourceConnection",
payload=payload,
)
return KnowledgeFSSourceConnectionResponse.model_validate(raw)
def list_source_connections(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
cursor: str | None,
limit: int,
) -> KnowledgeFSSourceConnectionListResponse:
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listSourceConnections",
query=query,
)
return KnowledgeFSSourceConnectionListResponse.model_validate(raw)
def refresh_source_connection(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
connection_id: str,
payload: KnowledgeFSSourceConnectionRefreshPayload,
) -> KnowledgeFSSourceConnectionResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="refreshSourceConnection",
payload=payload,
path_parameters=(("connectionId", connection_id),),
)
return KnowledgeFSSourceConnectionResponse.model_validate(raw)
def preview_source_crawl(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
idempotency_key: str,
) -> KnowledgeFSSourceWorkflowResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="previewSourceCrawl",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
headers=(("Idempotency-Key", idempotency_key),),
)
return KnowledgeFSSourceWorkflowResponse.model_validate(raw)
def get_source_sync_policy(
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
) -> KnowledgeFSSourceSyncPolicyResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getSourceSyncPolicy",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
)
return KnowledgeFSSourceSyncPolicyResponse.model_validate(raw)
def update_source_sync_policy(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
payload: KnowledgeFSSourceSyncPolicyPayload,
) -> KnowledgeFSSourceSyncPolicyResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="updateSourceSyncPolicy",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
payload=payload,
)
return KnowledgeFSSourceSyncPolicyResponse.model_validate(raw)
def get_source_workflow(
self, *, tenant_id: str, account_id: str, control_space_id: str, run_id: str
) -> KnowledgeFSSourceWorkflowResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getSourceWorkflow",
resource_id=run_id,
path_parameters=(("runId", run_id),),
)
return KnowledgeFSSourceWorkflowResponse.model_validate(raw)
def cancel_source_workflow(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
run_id: str,
payload: KnowledgeFSSourceWorkflowCancelPayload,
) -> KnowledgeFSSourceWorkflowResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="cancelSourceWorkflow",
resource_id=run_id,
path_parameters=(("runId", run_id),),
payload=payload,
)
return KnowledgeFSSourceWorkflowResponse.model_validate(raw)
def retry_source_workflow(
self, *, tenant_id: str, account_id: str, control_space_id: str, run_id: str
) -> KnowledgeFSSourceWorkflowResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="retrySourceWorkflow",
resource_id=run_id,
path_parameters=(("runId", run_id),),
)
return KnowledgeFSSourceWorkflowResponse.model_validate(raw)
def list_crawl_preview_pages(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
run_id: str,
cursor: str | None,
limit: int,
) -> KnowledgeFSCrawlPreviewPageListResponse:
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listCrawlPreviewPages",
resource_id=run_id,
path_parameters=(("runId", run_id),),
query=query,
)
return KnowledgeFSCrawlPreviewPageListResponse.model_validate(raw)
def select_crawl_preview_pages(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
run_id: str,
payload: KnowledgeFSCrawlPreviewSelectionPayload,
idempotency_key: str,
) -> KnowledgeFSSourceWorkflowResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="selectCrawlPreviewPages",
resource_id=run_id,
path_parameters=(("runId", run_id),),
payload=payload,
headers=(("Idempotency-Key", idempotency_key),),
)
return KnowledgeFSSourceWorkflowResponse.model_validate(raw)
def crawl_source(
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
) -> KnowledgeFSSourceCrawlResponse:

181
api/services/knowledge_fs/product_dto.py
View File

@ -354,6 +354,7 @@ class KnowledgeFSOverviewStatsResponse(ResponseModel):
class KnowledgeFSSpaceListItemResponse(ResponseModel):
control_space_id: str
created_at: datetime
state: KnowledgeFSControlSpaceState
visibility: KnowledgeFSControlSpaceVisibility
owner_account_id: str
@ -362,6 +363,7 @@ class KnowledgeFSSpaceListItemResponse(ResponseModel):
permission_keys: list[KnowledgeFSProductPermission]
technical_status: Literal["available", "not_ready", "unavailable"]
technical_summary: KnowledgeFSTechnicalSummary | None = None
updated_at: datetime
class KnowledgeFSSpaceListResponse(ResponseModel):
@ -372,8 +374,7 @@ class KnowledgeFSSpaceListResponse(ResponseModel):
class KnowledgeFSSpaceDetailResponse(KnowledgeFSSpaceListItemResponse):
created_at: datetime
updated_at: datetime
pass
class KnowledgeFSSpaceCreateResponse(ResponseModel):
@ -662,6 +663,11 @@ class KnowledgeFSLogicalDocumentResponse(ResponseModel):
user_metadata: dict[str, object] = Field(validation_alias=AliasChoices("user_metadata", "userMetadata"))
class KnowledgeFSLogicalDocumentListResponse(ResponseModel):
data: list[KnowledgeFSLogicalDocumentResponse] = Field(validation_alias=AliasChoices("data", "items"))
next_cursor: str | None = Field(default=None, validation_alias=AliasChoices("next_cursor", "nextCursor"))
class KnowledgeFSDocumentRevisionListResponse(ResponseModel):
data: list[KnowledgeFSDocumentRevisionResponse] = Field(validation_alias=AliasChoices("data", "items"))
next_cursor: str | None = Field(default=None, validation_alias=AliasChoices("next_cursor", "nextCursor"))
@ -963,6 +969,162 @@ class KnowledgeFSSourceCredentialTestResponse(ResponseModel):
valid: bool
class KnowledgeFSSourceWorkflowResponse(ResponseModel):
canceled_at: datetime | None = Field(default=None, validation_alias=AliasChoices("canceled_at", "canceledAt"))
checkpoint: str
completed_at: datetime | None = Field(default=None, validation_alias=AliasChoices("completed_at", "completedAt"))
created_at: datetime = Field(validation_alias=AliasChoices("created_at", "createdAt"))
cursor: str | None = None
execution_attempts: int = Field(ge=0, validation_alias=AliasChoices("execution_attempts", "executionAttempts"))
id: str
knowledge_space_id: str = Field(validation_alias=AliasChoices("knowledge_space_id", "knowledgeSpaceId"))
kind: str
last_error_code: str | None = Field(default=None, validation_alias=AliasChoices("last_error_code", "lastErrorCode"))
max_execution_attempts: int = Field(
ge=1, validation_alias=AliasChoices("max_execution_attempts", "maxExecutionAttempts")
)
progress_completed: int = Field(ge=0, validation_alias=AliasChoices("progress_completed", "progressCompleted"))
progress_failed: int = Field(ge=0, validation_alias=AliasChoices("progress_failed", "progressFailed"))
progress_skipped: int = Field(ge=0, validation_alias=AliasChoices("progress_skipped", "progressSkipped"))
progress_total: int | None = Field(
default=None, ge=0, validation_alias=AliasChoices("progress_total", "progressTotal")
)
source_id: str | None = Field(default=None, validation_alias=AliasChoices("source_id", "sourceId"))
state: str
updated_at: datetime = Field(validation_alias=AliasChoices("updated_at", "updatedAt"))
class KnowledgeFSSourceProviderFieldResponse(ResponseModel):
description: str | None = None
format: Literal["password", "uri"] | None = None
name: str
required: bool
secret: bool
type: Literal["boolean", "integer", "string"]
class KnowledgeFSSourceProviderResponse(ResponseModel):
auth_kinds: list[Literal["api-key", "endpoint", "oauth2"]] = Field(
validation_alias=AliasChoices("auth_kinds", "authKinds")
)
available: bool
capabilities: list[Literal["website-crawl", "online-document", "online-drive"]]
configuration: list[KnowledgeFSSourceProviderFieldResponse]
display_name: str = Field(validation_alias=AliasChoices("display_name", "displayName"))
id: str
unavailable_reason: str | None = Field(
default=None, validation_alias=AliasChoices("unavailable_reason", "unavailableReason")
)
class KnowledgeFSSourceProviderListResponse(ResponseModel):
data: list[KnowledgeFSSourceProviderResponse] = Field(validation_alias=AliasChoices("data", "items"))
class KnowledgeFSSourceConnectionCreatePayload(BaseModel):
auth_kind: Literal["api-key", "endpoint"] = Field(alias="authKind")
configuration: dict[str, bool | int | str] = Field(default_factory=dict)
credentials: dict[str, object]
name: str = Field(min_length=1, max_length=160)
provider_id: str = Field(min_length=1, max_length=128, alias="providerId")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSSourceConnectionResponse(ResponseModel):
auth_kind: Literal["api-key", "endpoint", "oauth2"] = Field(validation_alias=AliasChoices("auth_kind", "authKind"))
configuration: dict[str, bool | int | str]
created_at: datetime = Field(validation_alias=AliasChoices("created_at", "createdAt"))
error_code: str | None = Field(default=None, validation_alias=AliasChoices("error_code", "errorCode"))
expires_at: datetime | None = Field(default=None, validation_alias=AliasChoices("expires_at", "expiresAt"))
id: str
knowledge_space_id: str = Field(validation_alias=AliasChoices("knowledge_space_id", "knowledgeSpaceId"))
name: str
provider_id: str = Field(validation_alias=AliasChoices("provider_id", "providerId"))
scopes: list[str]
status: Literal["provisioning", "active", "expired", "error", "revoked"]
updated_at: datetime = Field(validation_alias=AliasChoices("updated_at", "updatedAt"))
version: int = Field(ge=1)
class KnowledgeFSSourceConnectionListQuery(BaseModel):
cursor: str | None = Field(default=None, min_length=1, max_length=4_096)
limit: int = Field(default=50, ge=1, le=200)
model_config = ConfigDict(extra="forbid")
class KnowledgeFSSourceConnectionListResponse(ResponseModel):
data: list[KnowledgeFSSourceConnectionResponse] = Field(validation_alias=AliasChoices("data", "items"))
next_cursor: str | None = Field(default=None, validation_alias=AliasChoices("next_cursor", "nextCursor"))
class KnowledgeFSSourceConnectionRefreshPayload(BaseModel):
expected_version: int = Field(ge=1, alias="expectedVersion")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSSourceSyncPolicyResponse(ResponseModel):
created_at: datetime = Field(validation_alias=AliasChoices("created_at", "createdAt"))
custom_interval_seconds: int | None = Field(
default=None, validation_alias=AliasChoices("custom_interval_seconds", "customIntervalSeconds")
)
enabled: bool
expected_source_version: int = Field(
ge=1, validation_alias=AliasChoices("expected_source_version", "expectedSourceVersion")
)
id: str
knowledge_space_id: str = Field(validation_alias=AliasChoices("knowledge_space_id", "knowledgeSpaceId"))
mode: Literal["provider", "manual", "interval", "custom"]
next_run_at: datetime | None = Field(default=None, validation_alias=AliasChoices("next_run_at", "nextRunAt"))
revision: int = Field(ge=1)
source_id: str = Field(validation_alias=AliasChoices("source_id", "sourceId"))
updated_at: datetime = Field(validation_alias=AliasChoices("updated_at", "updatedAt"))
class KnowledgeFSSourceSyncPolicyPayload(BaseModel):
custom_interval_seconds: int | None = Field(default=None, ge=3_600, le=2_592_000, alias="customIntervalSeconds")
enabled: bool
expected_revision: int = Field(ge=0, alias="expectedRevision")
expected_source_version: int = Field(ge=1, alias="expectedSourceVersion")
mode: Literal["provider", "manual", "interval", "custom"]
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSSourceWorkflowCancelPayload(BaseModel):
reason: str | None = Field(default=None, max_length=1_000)
model_config = ConfigDict(extra="forbid")
class KnowledgeFSCrawlPreviewPageResponse(ResponseModel):
description: str | None = None
etag: str | None = None
page_id: str = Field(validation_alias=AliasChoices("page_id", "pageId"))
source_url: str = Field(validation_alias=AliasChoices("source_url", "sourceUrl"))
title: str | None = None
class KnowledgeFSCrawlPreviewPageListQuery(BaseModel):
cursor: str | None = Field(default=None, min_length=1, max_length=4_096)
limit: int = Field(default=50, ge=1, le=200)
model_config = ConfigDict(extra="forbid")
class KnowledgeFSCrawlPreviewPageListResponse(ResponseModel):
data: list[KnowledgeFSCrawlPreviewPageResponse] = Field(validation_alias=AliasChoices("data", "items"))
next_cursor: str | None = Field(default=None, validation_alias=AliasChoices("next_cursor", "nextCursor"))
class KnowledgeFSCrawlPreviewSelectionPayload(BaseModel):
page_ids: list[str] = Field(min_length=1, max_length=200, alias="pageIds")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSCrawledPageResponse(ResponseModel):
content: str
description: str | None = None
@ -1468,6 +1630,9 @@ __all__ = [
"KnowledgeFSBulkDocumentDeletePayload",
"KnowledgeFSBulkJobResponse",
"KnowledgeFSCapabilityResponse",
"KnowledgeFSCrawlPreviewPageListQuery",
"KnowledgeFSCrawlPreviewPageListResponse",
"KnowledgeFSCrawlPreviewSelectionPayload",
"KnowledgeFSCredentialCreatePayload",
"KnowledgeFSCredentialCreateResponse",
"KnowledgeFSCredentialItemResponse",
@ -1492,6 +1657,8 @@ __all__ = [
"KnowledgeFSIdempotencyHeader",
"KnowledgeFSJWKResponse",
"KnowledgeFSJWKSResponse",
"KnowledgeFSLogicalDocumentListResponse",
"KnowledgeFSLogicalDocumentResponse",
"KnowledgeFSMemberBindingPayload",
"KnowledgeFSMembersReplacePayload",
"KnowledgeFSModelIntent",
@ -1532,6 +1699,11 @@ __all__ = [
"KnowledgeFSSettingsPayload",
"KnowledgeFSSettingsResponse",
"KnowledgeFSSmallFileUploadResponse",
"KnowledgeFSSourceConnectionCreatePayload",
"KnowledgeFSSourceConnectionListQuery",
"KnowledgeFSSourceConnectionListResponse",
"KnowledgeFSSourceConnectionRefreshPayload",
"KnowledgeFSSourceConnectionResponse",
"KnowledgeFSSourceCrawlResponse",
"KnowledgeFSSourceCreatePayload",
"KnowledgeFSSourceCredentialTestResponse",
@ -1545,8 +1717,13 @@ __all__ = [
"KnowledgeFSSourceListResponse",
"KnowledgeFSSourcePagesQuery",
"KnowledgeFSSourcePagesResponse",
"KnowledgeFSSourceProviderListResponse",
"KnowledgeFSSourceResponse",
"KnowledgeFSSourceSyncPolicyPayload",
"KnowledgeFSSourceSyncPolicyResponse",
"KnowledgeFSSourceUpdatePayload",
"KnowledgeFSSourceWorkflowCancelPayload",
"KnowledgeFSSourceWorkflowResponse",
"KnowledgeFSSpaceCreatePayload",
"KnowledgeFSSpaceCreateResponse",
"KnowledgeFSSpaceDetailResponse",

183
api/services/knowledge_fs/product_operations.py
View File

@ -233,6 +233,30 @@ KNOWLEDGE_FS_PRODUCT_OPERATIONS: Final[MappingProxyType[str, KnowledgeFSProductO
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"listLogicalDocuments": _operation(
"GET",
"listLogicalDocuments",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/logical-documents",
"json",
resource_resolver="knowledge_space",
billing_cost=2,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"getLogicalDocument": _operation(
"GET",
"getLogicalDocument",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/logical-documents/{documentId}",
"json",
resource_resolver="document",
billing_cost=2,
max_request_bytes=0,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"createDocument": _operation(
"POST",
"uploadDocument",
@ -510,6 +534,165 @@ KNOWLEDGE_FS_PRODUCT_OPERATIONS: Final[MappingProxyType[str, KnowledgeFSProductO
max_response_bytes=256 * 1024,
stream_kind="json",
),
"syncSource": _operation(
"POST",
"createSourceSyncWorkflow",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/sync",
"json",
resource_resolver="source",
billing_cost=8,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"listSourceProviders": _operation(
"GET",
"listSourceProviders",
KnowledgeFSProductPermission.READ,
"/source-providers",
"json",
resource_resolver="namespace",
billing_cost=1,
max_request_bytes=0,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"createSourceConnection": _operation(
"POST",
"createSourceConnection",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/source-connections",
"json",
resource_resolver="knowledge_space",
billing_cost=5,
max_request_bytes=256 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"listSourceConnections": _operation(
"GET",
"listSourceConnections",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/source-connections",
"json",
resource_resolver="knowledge_space",
billing_cost=1,
max_request_bytes=16 * 1024,
max_response_bytes=1024 * 1024,
stream_kind="json",
),
"refreshSourceConnection": _operation(
"POST",
"refreshSourceConnection",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/source-connections/{connectionId}/refresh",
"json",
resource_resolver="knowledge_space",
billing_cost=3,
max_request_bytes=32 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"previewSourceCrawl": _operation(
"POST",
"createSourceCrawlPreviewWorkflow",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/crawl-preview",
"json",
resource_resolver="source",
billing_cost=8,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"getSourceSyncPolicy": _operation(
"GET",
"getSourceSyncPolicy",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
"json",
resource_resolver="source",
billing_cost=1,
max_request_bytes=0,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"updateSourceSyncPolicy": _operation(
"PUT",
"putSourceSyncPolicy",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
"json",
resource_resolver="source",
billing_cost=3,
max_request_bytes=32 * 1024,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"getSourceWorkflow": _operation(
"GET",
"getSourceWorkflow",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/source-workflows/{runId}",
"json",
resource_resolver="job",
billing_cost=1,
max_request_bytes=0,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"cancelSourceWorkflow": _operation(
"POST",
"cancelSourceWorkflow",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/source-workflows/{runId}/cancel",
"json",
resource_resolver="job",
billing_cost=2,
max_request_bytes=32 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"retrySourceWorkflow": _operation(
"POST",
"retrySourceWorkflow",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/source-workflows/{runId}/retry",
"json",
resource_resolver="job",
billing_cost=3,
max_request_bytes=0,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"listCrawlPreviewPages": _operation(
"GET",
"listCrawlPreviewPages",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/source-workflows/{runId}/pages",
"json",
resource_resolver="job",
billing_cost=1,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"selectCrawlPreviewPages": _operation(
"POST",
"selectCrawlPreviewPages",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/source-workflows/{runId}/selection",
"json",
resource_resolver="job",
billing_cost=8,
max_request_bytes=256 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"crawlSource": _operation(
"POST",
"crawlKnowledgeSpaceSource",

5
api/services/knowledge_fs/product_remote.py
View File

@ -13,6 +13,10 @@ class KnowledgeFSProductRemoteError(RuntimeError):
"""KnowledgeFS could not provide an authoritative product response."""
class KnowledgeFSProductResourceNotFoundError(KnowledgeFSProductRemoteError):
"""KnowledgeFS authoritatively reported that an authorized child resource is absent."""
class KnowledgeFSOperationUnavailableError(RuntimeError):
"""The Dify/KFS/Capability operation manifests are not yet aligned."""
@ -96,6 +100,7 @@ __all__ = [
"KnowledgeFSProductRemoteError",
"KnowledgeFSProductRemotePort",
"KnowledgeFSProductRequestRejectedError",
"KnowledgeFSProductResourceNotFoundError",
"KnowledgeFSRemoteBinaryRequest",
"KnowledgeFSRemoteJSONRequest",
"UnavailableKnowledgeFSProductRemote",

11
api/services/knowledge_fs/product_remote_http.py
View File

@ -20,6 +20,7 @@ from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemoteError,
KnowledgeFSProductRequestRejectedError,
KnowledgeFSProductResourceNotFoundError,
KnowledgeFSRemoteBinaryRequest,
KnowledgeFSRemoteJSONRequest,
)
@ -167,6 +168,8 @@ class HTTPKnowledgeFSProductRemoteClient:
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
if content_type != "application/json" and not content_type.endswith("+json"):
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an unsupported media type")
if response.status_code == HTTPStatus.NOT_FOUND:
raise KnowledgeFSProductResourceNotFoundError("KnowledgeFS resource was not found")
if not HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
raise KnowledgeFSProductRemoteError(f"KnowledgeFS returned HTTP {response.status_code}")
try:
@ -234,9 +237,17 @@ class HTTPKnowledgeFSProductRemoteClient:
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS request failed") from exc
try:
if response.status_code == 409:
raise KnowledgeFSProductRequestRejectedError(status_code=409)
if response.status_code == 413:
raise KnowledgeFSProductRequestRejectedError(status_code=413)
if response.status_code == 422:
raise KnowledgeFSProductRequestRejectedError(status_code=422)
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
if content_type != "application/json" and not content_type.endswith("+json"):
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an unsupported media type")
if response.status_code == HTTPStatus.NOT_FOUND:
raise KnowledgeFSProductResourceNotFoundError("KnowledgeFS resource was not found")
if not HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
raise KnowledgeFSProductRemoteError(f"KnowledgeFS returned HTTP {response.status_code}")
try:

8
api/services/knowledge_fs/product_service.py
View File

@ -212,11 +212,7 @@ class KnowledgeFSProductService:
trace_id=str(uuid.uuid4()),
)
item = _list_item(space, summaries=summaries, permission_keys=authorized.permission_keys)
return KnowledgeFSSpaceDetailResponse(
**item.model_dump(),
created_at=space.created_at,
updated_at=space.updated_at,
)
return KnowledgeFSSpaceDetailResponse(**item.model_dump())
def require_product_routes(self, *, tenant_id: str) -> None:
self._cutover_gate.require_product_routes(tenant_id=tenant_id)
@ -308,6 +304,7 @@ def _list_item(
technical_status = "available"
return KnowledgeFSSpaceListItemResponse(
control_space_id=space.id,
created_at=space.created_at,
state=space.state,
visibility=space.visibility,
owner_account_id=space.owner_account_id,
@ -316,6 +313,7 @@ def _list_item(
permission_keys=list(permission_keys),
technical_status=technical_status,
technical_summary=summary,
updated_at=space.updated_at,
)

105
api/services/knowledge_fs_capability.py
View File

@ -388,6 +388,20 @@ KNOWLEDGE_FS_CAPABILITY_OPERATIONS: Final[Mapping[str, KnowledgeFSCapabilityOper
"/knowledge-spaces/{id}/documents",
"knowledge_space",
),
"listLogicalDocuments": KnowledgeFSCapabilityOperation(
"logical_documents.list",
_STANDARD_CALLERS,
"GET",
"/knowledge-spaces/{id}/logical-documents",
"knowledge_space",
),
"getLogicalDocument": KnowledgeFSCapabilityOperation(
"logical_documents.read",
_STANDARD_CALLERS,
"GET",
"/knowledge-spaces/{id}/logical-documents/{documentId}",
"document",
),
"uploadDocument": KnowledgeFSCapabilityOperation(
"documents.create",
_STANDARD_CALLERS,
@ -533,6 +547,97 @@ KNOWLEDGE_FS_CAPABILITY_OPERATIONS: Final[Mapping[str, KnowledgeFSCapabilityOper
"/knowledge-spaces/{id}/sources/{sourceId}/test",
"source",
),
"createSourceSyncWorkflow": KnowledgeFSCapabilityOperation(
"source_workflows.sync.create",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/sources/{sourceId}/sync",
"source",
),
"listSourceProviders": KnowledgeFSCapabilityOperation(
"source_providers.list",
_STANDARD_CALLERS,
"GET",
"/source-providers",
"namespace",
),
"createSourceConnection": KnowledgeFSCapabilityOperation(
"source_connections.create",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/source-connections",
"knowledge_space",
),
"listSourceConnections": KnowledgeFSCapabilityOperation(
"source_connections.list",
_STANDARD_CALLERS,
"GET",
"/knowledge-spaces/{id}/source-connections",
"knowledge_space",
),
"refreshSourceConnection": KnowledgeFSCapabilityOperation(
"source_connections.refresh",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/source-connections/{connectionId}/refresh",
"knowledge_space",
),
"createSourceCrawlPreviewWorkflow": KnowledgeFSCapabilityOperation(
"source_workflows.preview.create",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/sources/{sourceId}/crawl-preview",
"source",
),
"getSourceSyncPolicy": KnowledgeFSCapabilityOperation(
"source_sync_policies.read",
_STANDARD_CALLERS,
"GET",
"/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
"source",
),
"putSourceSyncPolicy": KnowledgeFSCapabilityOperation(
"source_sync_policies.update",
_STANDARD_CALLERS,
"PUT",
"/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
"source",
),
"getSourceWorkflow": KnowledgeFSCapabilityOperation(
"source_workflows.read",
_STANDARD_CALLERS,
"GET",
"/knowledge-spaces/{id}/source-workflows/{runId}",
"job",
),
"cancelSourceWorkflow": KnowledgeFSCapabilityOperation(
"source_workflows.cancel",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/source-workflows/{runId}/cancel",
"job",
),
"retrySourceWorkflow": KnowledgeFSCapabilityOperation(
"source_workflows.retry",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/source-workflows/{runId}/retry",
"job",
),
"listCrawlPreviewPages": KnowledgeFSCapabilityOperation(
"source_workflows.pages.list",
_STANDARD_CALLERS,
"GET",
"/knowledge-spaces/{id}/source-workflows/{runId}/pages",
"job",
),
"selectCrawlPreviewPages": KnowledgeFSCapabilityOperation(
"source_workflows.selection.create",
_STANDARD_CALLERS,
"POST",
"/knowledge-spaces/{id}/source-workflows/{runId}/selection",
"job",
),
"crawlKnowledgeSpaceSource": KnowledgeFSCapabilityOperation(
"sources.crawl",
_STANDARD_CALLERS,

2
api/tests/unit_tests/configs/test_knowledge_fs_config.py
View File

@ -17,6 +17,7 @@ _KNOWLEDGE_FS_DOCKER_VARIABLES = (
"KNOWLEDGE_FS_ENABLED",
"KNOWLEDGE_FS_BASE_URL",
"KNOWLEDGE_FS_DIRECT_ORIGIN",
"KNOWLEDGE_FS_DIRECT_UPLOAD_READY",
"KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED",
"KNOWLEDGE_FS_INTEGRATED_PROVISION_READY",
"KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY",
@ -68,6 +69,7 @@ def test_knowledge_fs_lifecycle_worker_is_disabled_by_default() -> None:
assert config.KNOWLEDGE_FS_INTEGRATED_PROVISION_READY is False
assert config.KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY is False
assert config.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED is False
assert config.KNOWLEDGE_FS_DIRECT_UPLOAD_READY is False
def test_capability_v2_requires_private_signing_configuration_when_enabled() -> None:

47
api/tests/unit_tests/controllers/inner_api/plugin/test_plugin_wraps.py
View File

@ -25,6 +25,8 @@ from models.base import TypeBase
from models.enums import EndUserType
from models.model import DefaultEndUserSessionID, EndUser
_USER_UUID = "00000000-0000-4000-8000-000000000001"
@pytest.fixture
def sqlite_plugin_engine(
@ -101,14 +103,14 @@ class TestGetUser:
"""Test returning existing user when found by ID"""
_persist_end_user(
sqlite_plugin_engine,
user_id="user123",
user_id=_USER_UUID,
session_id="existing-session",
)
with app.app_context():
result = get_user("tenant123", "user123")
result = get_user("tenant123", _USER_UUID)
assert result.id == "user123"
assert result.id == _USER_UUID
assert result.tenant_id == "tenant123"
def test_should_not_resolve_non_anonymous_users_across_tenants(
@ -158,6 +160,39 @@ class TestGetUser:
users = session.scalars(select(EndUser)).all()
assert [user.id for user in users] == ["persisted-user-id"]
def test_should_skip_uuid_id_lookup_for_text_session_id(
self,
sqlite_plugin_engine: Engine,
app: Flask,
):
"""Service actor names must not be compared with the UUID primary key."""
_persist_end_user(
sqlite_plugin_engine,
user_id="persisted-user-id",
session_id="knowledge-fs",
)
statements: list[str] = []
def _record_statement(
_connection: object,
_cursor: object,
statement: str,
_parameters: object,
_context: object,
_executemany: bool,
) -> None:
statements.append(statement)
event.listen(sqlite_plugin_engine, "before_cursor_execute", _record_statement)
try:
with app.app_context():
result = get_user("tenant123", "knowledge-fs")
finally:
event.remove(sqlite_plugin_engine, "before_cursor_execute", _record_statement)
assert result.id == "persisted-user-id"
assert not any("WHERE end_users.id =" in statement for statement in statements)
def test_should_return_existing_anonymous_user_by_session_id(
self,
sqlite_plugin_engine: Engine,
@ -244,17 +279,17 @@ class TestGetUserTenant:
_persist_tenant(sqlite_plugin_engine)
_persist_end_user(
sqlite_plugin_engine,
user_id="user456",
user_id=_USER_UUID,
session_id="user-session",
)
with app.test_request_context(json={"tenant_id": "tenant123", "user_id": "user456"}):
with app.test_request_context(json={"tenant_id": "tenant123", "user_id": _USER_UUID}):
monkeypatch.setattr(app, "login_manager", MagicMock(), raising=False)
with patch("controllers.inner_api.plugin.wraps.user_logged_in"):
result = protected_view()
assert result["tenant"].id == "tenant123"
assert result["user"].id == "user456"
assert result["user"].id == _USER_UUID
def test_should_raise_error_when_tenant_id_missing(self, app: Flask):
"""Test that Pydantic ValidationError is raised when tenant_id is missing from payload"""

26
api/tests/unit_tests/controllers/test_knowledge_fs_product_controllers.py
View File

@ -46,7 +46,20 @@ def test_console_and_service_api_routes_are_registered() -> None:
"/knowledge-fs/spaces/<string:control_space_id>/credentials",
"/knowledge-fs/spaces/<string:control_space_id>/settings",
"/knowledge-fs/spaces/<string:control_space_id>/documents",
"/knowledge-fs/spaces/<string:control_space_id>/logical-documents",
"/knowledge-fs/spaces/<string:control_space_id>/logical-documents/<string:document_id>",
"/knowledge-fs/spaces/<string:control_space_id>/sources",
"/knowledge-fs/spaces/<string:control_space_id>/source-connections",
("/knowledge-fs/spaces/<string:control_space_id>/source-connections/<string:connection_id>/refresh"),
"/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/sync",
"/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/crawl-preview",
"/knowledge-fs/spaces/<string:control_space_id>/sources/<string:source_id>/sync-policy",
"/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>",
"/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/cancel",
"/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/retry",
"/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/pages",
"/knowledge-fs/spaces/<string:control_space_id>/source-workflows/<string:run_id>/selection",
"/knowledge-fs/spaces/<string:control_space_id>/source-providers",
"/knowledge-fs/spaces/<string:control_space_id>/queries",
"/knowledge-fs/spaces/<string:control_space_id>/research-tasks",
"/knowledge-fs/spaces/<string:control_space_id>/traces",
@ -114,6 +127,18 @@ def test_knowledge_fs_request_and_response_schemas_are_registered() -> None:
"KnowledgeFSStreamCapabilityResponse",
"KnowledgeFSJWKSResponse",
"KnowledgeFSSmallFileUploadResponse",
"KnowledgeFSCrawlPreviewPageListQuery",
"KnowledgeFSCrawlPreviewPageListResponse",
"KnowledgeFSCrawlPreviewSelectionPayload",
"KnowledgeFSSourceConnectionCreatePayload",
"KnowledgeFSSourceConnectionListQuery",
"KnowledgeFSSourceConnectionListResponse",
"KnowledgeFSSourceConnectionRefreshPayload",
"KnowledgeFSSourceProviderListResponse",
"KnowledgeFSSourceSyncPolicyPayload",
"KnowledgeFSSourceSyncPolicyResponse",
"KnowledgeFSSourceWorkflowCancelPayload",
"KnowledgeFSSourceWorkflowResponse",
}.issubset(console_ns.models)
assert {
"KnowledgeFSDocumentCreatePayload",
@ -524,6 +549,7 @@ def test_upload_and_task_stream_capabilities_use_direct_operation_admission(
runtime = SimpleNamespace(direct_operation_admission=DirectAdmission())
monkeypatch.setattr(console_resources.dify_config, "KNOWLEDGE_FS_DIRECT_ORIGIN", "https://kfs.test")
monkeypatch.setattr(console_resources.dify_config, "KNOWLEDGE_FS_DIRECT_UPLOAD_READY", True)
monkeypatch.setattr(console_resources, "_actor", lambda: ("account-1", "tenant-1"))
monkeypatch.setattr(console_resources, "_console_services", lambda: runtime)
app = Flask(__name__)

27
api/tests/unit_tests/controllers/test_knowledge_fs_resource_delegation.py
View File

@ -384,11 +384,11 @@ _CONSOLE_DELEGATION_CASES = (
{"control_space_id": "space-1", "source_id": "source-1"},
),
(
"KnowledgeFSSpaceSourceCrawlApi",
"KnowledgeFSSpaceSourceSyncApi",
"post",
("space-1", "source-1"),
"facade",
"crawl_source",
"sync_source",
{"control_space_id": "space-1", "source_id": "source-1"},
),
(
@ -987,6 +987,7 @@ def test_console_direct_capabilities_bind_the_authorized_resource(monkeypatch: p
]
)
monkeypatch.setattr(console_resources.dify_config, "KNOWLEDGE_FS_DIRECT_ORIGIN", "https://kfs.example/")
monkeypatch.setattr(console_resources.dify_config, "KNOWLEDGE_FS_DIRECT_UPLOAD_READY", True)
monkeypatch.setattr(console_resources, "_actor", lambda: ("account-1", "tenant-1"))
monkeypatch.setattr(
console_resources,
@ -1066,6 +1067,16 @@ def test_direct_routes_fail_before_admission_when_origin_is_unconfigured(
_invoke(resource_module, class_name, "post", "resource-1")
def test_console_upload_capability_fails_before_admission_until_upload_is_verified(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(console_resources.dify_config, "KNOWLEDGE_FS_DIRECT_ORIGIN", "https://kfs.example")
monkeypatch.setattr(console_resources.dify_config, "KNOWLEDGE_FS_DIRECT_UPLOAD_READY", False)
with pytest.raises(KnowledgeFSOperationUnavailableError, match="direct upload"):
_invoke(console_resources, "KnowledgeFSSpaceUploadCapabilitiesApi", "post", "space-1")
def test_console_resource_helpers_validate_feature_payload_headers_and_query_pairs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@ -1194,7 +1205,10 @@ def test_console_error_adapter_maps_every_domain_boundary_to_the_stable_http_con
from services.knowledge_fs.control_plane_service import KnowledgeFSControlPlaneInvariantError
from services.knowledge_fs.credential_service import KnowledgeFSCredentialPolicyError
from services.knowledge_fs.product_authorization import KnowledgeFSProductNotFoundError
from services.knowledge_fs.product_remote import KnowledgeFSProductRemoteError
from services.knowledge_fs.product_remote import (
KnowledgeFSProductRemoteError,
KnowledgeFSProductResourceNotFoundError,
)
from services.knowledge_fs_capability import KnowledgeFSCapabilityConfigurationError
with pytest.raises(ValidationError) as raised_validation:
@ -1202,6 +1216,7 @@ def test_console_error_adapter_maps_every_domain_boundary_to_the_stable_http_con
validation_error = raised_validation.value
mappings = (
(KnowledgeFSProductNotFoundError("hidden"), KnowledgeFSSpaceNotFoundHTTPError),
(KnowledgeFSProductResourceNotFoundError("missing child"), NotFound),
(KnowledgeFSOperationUnavailableError("manifest mismatch"), KnowledgeFSOperationUnavailableHTTPError),
(KnowledgeFSProductRemoteError("upstream unavailable"), KnowledgeFSUpstreamUnavailableHTTPError),
(KnowledgeFSAppBindingManagementError("invalid binding"), KnowledgeFSInvalidRequestHTTPError),
@ -1223,7 +1238,10 @@ def test_service_error_adapter_maps_every_domain_boundary_to_the_stable_http_con
from pydantic import ValidationError
from services.knowledge_fs.credential_service import KnowledgeFSCredentialValidationError
from services.knowledge_fs.product_remote import KnowledgeFSProductRemoteError
from services.knowledge_fs.product_remote import (
KnowledgeFSProductRemoteError,
KnowledgeFSProductResourceNotFoundError,
)
with pytest.raises(ValidationError) as raised_validation:
KnowledgeFSQueryCreatePayload.model_validate({"query": ""})
@ -1231,6 +1249,7 @@ def test_service_error_adapter_maps_every_domain_boundary_to_the_stable_http_con
mappings = (
(KnowledgeFSCredentialValidationError("revoked"), KnowledgeFSInvalidCredentialHTTPError),
(KnowledgeFSOperationUnavailableError("manifest mismatch"), KnowledgeFSServiceOperationUnavailableHTTPError),
(KnowledgeFSProductResourceNotFoundError("missing child"), NotFound),
(KnowledgeFSProductRemoteError("upstream unavailable"), KnowledgeFSServiceUpstreamUnavailableHTTPError),
(validation_error, KnowledgeFSServiceInvalidRequestHTTPError),
)

36
api/tests/unit_tests/extensions/test_ext_blueprints_cors.py Normal file
View File

@ -0,0 +1,36 @@
"""Regression coverage for authenticated browser CORS headers."""
from flask import Blueprint, Flask
from extensions.ext_blueprints import AUTHENTICATED_HEADERS, _apply_cors_once
def test_authenticated_cors_allows_request_metadata_headers() -> None:
app = Flask(__name__)
blueprint = Blueprint("cors_probe", __name__, url_prefix="/console/api")
@blueprint.post("/probe")
def probe() -> tuple[str, int]:
return "", 204
_apply_cors_once(
blueprint,
resources={r"/*": {"origins": ["http://localhost:3000"]}},
supports_credentials=True,
allow_headers=list(AUTHENTICATED_HEADERS),
methods=["POST", "OPTIONS"],
)
app.register_blueprint(blueprint)
response = app.test_client().options(
"/console/api/probe",
headers={
"Access-Control-Request-Headers": "Idempotency-Key, X-Request-ID",
"Access-Control-Request-Method": "POST",
"Origin": "http://localhost:3000",
},
)
allowed_headers = response.headers.get("Access-Control-Allow-Headers", "").lower()
assert "idempotency-key" in allowed_headers
assert "x-request-id" in allowed_headers

36
api/tests/unit_tests/services/test_feature_service_knowledge_fs.py Normal file
View File

@ -0,0 +1,36 @@
import pytest
from services import feature_service as feature_service_module
from services.feature_service import FeatureService
@pytest.mark.parametrize(
("enabled", "direct_origin", "direct_upload_ready", "upload_enabled"),
[
(True, "https://uploads.knowledge-fs.test", True, True),
(True, "https://uploads.knowledge-fs.test", False, False),
(True, None, True, False),
(False, "https://uploads.knowledge-fs.test", True, False),
],
)
def test_get_system_features_reads_knowledge_fs_availability(
monkeypatch: pytest.MonkeyPatch,
enabled: bool,
direct_origin: str | None,
direct_upload_ready: bool,
upload_enabled: bool,
) -> None:
monkeypatch.setattr(feature_service_module.dify_config, "KNOWLEDGE_FS_ENABLED", enabled)
monkeypatch.setattr(feature_service_module.dify_config, "KNOWLEDGE_FS_DIRECT_ORIGIN", direct_origin)
monkeypatch.setattr(
feature_service_module.dify_config,
"KNOWLEDGE_FS_DIRECT_UPLOAD_READY",
direct_upload_ready,
)
result = FeatureService.get_system_features()
assert result.knowledge_fs_enabled is enabled
assert result.knowledge_fs_upload_enabled is upload_enabled
assert result.model_dump()["knowledge_fs_enabled"] is enabled
assert result.model_dump()["knowledge_fs_upload_enabled"] is upload_enabled

105
api/tests/unit_tests/services/test_knowledge_fs_data_facade.py
View File

@ -604,6 +604,20 @@ def test_advanced_facade_binds_child_resources_parent_space_and_idempotency() ->
("get_overview_inventory", "KnowledgeFSOverviewInventoryResponse", "getOverviewInventory", {}, None),
("get_overview_health", "KnowledgeFSOverviewHealthResponse", "getOverviewHealth", {}, None),
("list_documents", "KnowledgeFSDocumentListResponse", "listDocuments", {"cursor": "cursor-1"}, None),
(
"list_logical_documents",
"KnowledgeFSLogicalDocumentListResponse",
"listLogicalDocuments",
{"cursor": "cursor-1"},
None,
),
(
"get_logical_document",
"KnowledgeFSLogicalDocumentResponse",
"getLogicalDocument",
{"document_id": "document-1"},
"document-1",
),
("get_document", "KnowledgeFSDocumentResponse", "getDocument", {"document_id": "document-1"}, "document-1"),
(
"get_document_outline",
@ -703,6 +717,97 @@ def test_advanced_facade_binds_child_resources_parent_space_and_idempotency() ->
},
"source-1",
),
(
"sync_source",
"KnowledgeFSSourceWorkflowResponse",
"syncSource",
{"source_id": "source-1", "idempotency_key": "sync-source-once"},
"source-1",
),
(
"list_source_providers",
"KnowledgeFSSourceProviderListResponse",
"listSourceProviders",
{},
None,
),
(
"create_source_connection",
"KnowledgeFSSourceConnectionResponse",
"createSourceConnection",
{"payload": MagicMock()},
None,
),
(
"list_source_connections",
"KnowledgeFSSourceConnectionListResponse",
"listSourceConnections",
{"cursor": "cursor-1", "limit": 25},
None,
),
(
"refresh_source_connection",
"KnowledgeFSSourceConnectionResponse",
"refreshSourceConnection",
{"connection_id": "connection-1", "payload": MagicMock()},
None,
),
(
"preview_source_crawl",
"KnowledgeFSSourceWorkflowResponse",
"previewSourceCrawl",
{"source_id": "source-1", "idempotency_key": "preview-source-once"},
"source-1",
),
(
"get_source_sync_policy",
"KnowledgeFSSourceSyncPolicyResponse",
"getSourceSyncPolicy",
{"source_id": "source-1"},
"source-1",
),
(
"update_source_sync_policy",
"KnowledgeFSSourceSyncPolicyResponse",
"updateSourceSyncPolicy",
{"source_id": "source-1", "payload": MagicMock()},
"source-1",
),
(
"get_source_workflow",
"KnowledgeFSSourceWorkflowResponse",
"getSourceWorkflow",
{"run_id": "run-1"},
"run-1",
),
(
"cancel_source_workflow",
"KnowledgeFSSourceWorkflowResponse",
"cancelSourceWorkflow",
{"run_id": "run-1", "payload": MagicMock()},
"run-1",
),
(
"retry_source_workflow",
"KnowledgeFSSourceWorkflowResponse",
"retrySourceWorkflow",
{"run_id": "run-1"},
"run-1",
),
(
"list_crawl_preview_pages",
"KnowledgeFSCrawlPreviewPageListResponse",
"listCrawlPreviewPages",
{"run_id": "run-1", "cursor": "cursor-1", "limit": 25},
"run-1",
),
(
"select_crawl_preview_pages",
"KnowledgeFSSourceWorkflowResponse",
"selectCrawlPreviewPages",
{"run_id": "run-1", "payload": MagicMock(), "idempotency_key": "selection-once"},
"run-1",
),
(
"crawl_source",
"KnowledgeFSSourceCrawlResponse",

15
api/tests/unit_tests/services/test_knowledge_fs_product_operations.py
View File

@ -29,11 +29,13 @@ def test_ready_product_operations_exactly_match_capability_method_path_and_actio
"cancelBackgroundTask",
"cancelCompilationJob",
"cancelResearchTask",
"cancelSourceWorkflow",
"completeUploadSession",
"crawlSource",
"createQuery",
"createResearchTask",
"createSource",
"createSourceConnection",
"createUploadSession",
"deleteDocument",
"deleteSource",
@ -42,6 +44,7 @@ def test_ready_product_operations_exactly_match_capability_method_path_and_actio
"getDocument",
"getDocumentChunk",
"getDocumentOutline",
"getLogicalDocument",
"getOverviewHealth",
"getOverviewInventory",
"getOverviewQueryOutcomes",
@ -49,6 +52,8 @@ def test_ready_product_operations_exactly_match_capability_method_path_and_actio
"getResearchTask",
"getSettings",
"getSource",
"getSourceSyncPolicy",
"getSourceWorkflow",
"getSpace",
"getTrace",
"importSourceFiles",
@ -56,26 +61,36 @@ def test_ready_product_operations_exactly_match_capability_method_path_and_actio
"listDocumentChunks",
"listDocumentRevisions",
"listDocuments",
"listLogicalDocuments",
"listBackgroundTasks",
"listCrawlPreviewPages",
"listResearchTaskPartials",
"listResearchTasks",
"listSources",
"listSourceConnections",
"listSourceFiles",
"listSourcePages",
"listSourceProviders",
"listTraceConflicts",
"listTraceEvidence",
"listTraceMissing",
"listTraces",
"planResearchTask",
"presignUploadSessionPart",
"previewSourceCrawl",
"reindexDocuments",
"refreshSourceConnection",
"retryBackgroundTask",
"retryCompilationJob",
"retrySourceWorkflow",
"selectCrawlPreviewPages",
"streamResearchTask",
"syncSource",
"testSource",
"updateDocumentMetadata",
"updateSettings",
"updateSource",
"updateSourceSyncPolicy",
"updateSpace",
"uploadSmallFile",
}

37
api/tests/unit_tests/services/test_knowledge_fs_product_remote_http.py
View File

@ -9,6 +9,7 @@ from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemoteError,
KnowledgeFSProductRequestRejectedError,
KnowledgeFSProductResourceNotFoundError,
KnowledgeFSRemoteBinaryRequest,
KnowledgeFSRemoteJSONRequest,
)
@ -520,17 +521,22 @@ def test_binary_remote_closes_and_maps_all_upstream_response_failures(
@pytest.mark.parametrize(
("status_code", "content_type", "body"),
("status_code", "content_type", "body", "error_type", "expected_status"),
[
(500, "application/json", b"{}"),
(200, "text/plain", b"ok"),
(200, "application/json", b"{"),
(409, "application/json", b"{}", KnowledgeFSProductRequestRejectedError, 409),
(413, "application/json", b"{}", KnowledgeFSProductRequestRejectedError, 413),
(422, "application/json", b"{}", KnowledgeFSProductRequestRejectedError, 422),
(500, "application/json", b"{}", KnowledgeFSProductRemoteError, None),
(200, "text/plain", b"ok", KnowledgeFSProductRemoteError, None),
(200, "application/json", b"{", KnowledgeFSProductRemoteError, None),
],
)
def test_json_remote_closes_and_maps_upstream_response_failures(
status_code: int,
content_type: str,
body: bytes,
error_type: type[Exception],
expected_status: int | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
response = httpx.Response(status_code, content=body, headers={"Content-Type": content_type})
@ -538,7 +544,28 @@ def test_json_remote_closes_and_maps_upstream_response_failures(
monkeypatch.setattr(ssrf_proxy, "buffer_response", lambda buffered, **_: buffered)
client = HTTPKnowledgeFSProductRemoteClient(base_url="https://knowledge-fs.test", timeout_seconds=3)
with pytest.raises(KnowledgeFSProductRemoteError):
with pytest.raises(error_type) as raised:
client.execute_json(_json_request())
if expected_status is not None:
assert isinstance(raised.value, KnowledgeFSProductRequestRejectedError)
assert raised.value.status_code == expected_status
assert response.is_closed
def test_json_remote_preserves_authoritative_resource_not_found(
monkeypatch: pytest.MonkeyPatch,
) -> None:
response = httpx.Response(
404,
json={"error": "Source sync policy not found"},
headers={"Content-Type": "application/json"},
)
monkeypatch.setattr(ssrf_proxy, "make_request", lambda **_: response)
monkeypatch.setattr(ssrf_proxy, "buffer_response", lambda buffered, **_: buffered)
client = HTTPKnowledgeFSProductRemoteClient(base_url="https://knowledge-fs.test", timeout_seconds=3)
with pytest.raises(KnowledgeFSProductResourceNotFoundError):
client.execute_json(_json_request())
assert response.is_closed

3
docker/envs/core-services/api.env.example
View File

@ -15,13 +15,14 @@ KNOWLEDGE_FS_ENABLED=${KNOWLEDGE_FS_ENABLED:-false}
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
KNOWLEDGE_FS_BASE_URL=
KNOWLEDGE_FS_DIRECT_ORIGIN=
# Set true only after the KnowledgeFS direct-upload service and browser origins below are verified.
KNOWLEDGE_FS_DIRECT_UPLOAD_READY=false
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
# Legacy rollback-only HMAC; leave blank when Capability v2 is selected.
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=

5
docker/envs/core-services/knowledge-fs.env.example
View File

@ -19,6 +19,11 @@ KNOWLEDGE_DOCUMENT_COMPILATION_RUNTIME=on
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
KNOWLEDGE_FS_CAPABILITY_V2_PUBLIC_JWKS=
# Direct upload remains hidden in Dify until this service is enabled, its allowed browser origins
# are verified, and the API service sets KNOWLEDGE_FS_DIRECT_UPLOAD_READY=true.
KNOWLEDGE_DIRECT_UPLOAD_ENABLED=false
KNOWLEDGE_DIRECT_UPLOAD_ALLOWED_ORIGINS=
# Complex-document parser reachable from the default Compose network or an external endpoint.
# Leave both values blank only when PDF/Office parsing is intentionally unavailable.
UNSTRUCTURED_API_URL=

58
knowledge-fs/apps/api/src/auth-options.test.ts
View File

@ -1,8 +1,57 @@
import { createHmac } from "node:crypto";
import { describe, expect, it } from "vitest";
import { createApiAuthVerifier } from "./auth-options";
describe("createApiAuthVerifier", () => {
it.each(["development", "production"])(
"accepts Dify-issued workspace JWTs in %s mode",
async (nodeEnvironment) => {
const secret = "test-secret-with-at-least-32-bytes";
const issuedAt = Math.floor(Date.now() / 1_000);
const token = signJwt(
{
caller_kind: "interactive",
scopes: ["knowledge-spaces:write"],
tenant_id: "tenant-1",
aud: "knowledge-fs",
exp: issuedAt + 60,
iat: issuedAt,
iss: "dify",
sub: "dify-workspace:tenant-1",
},
secret,
);
const verifier = createApiAuthVerifier({
KNOWLEDGE_FS_JWT_SECRET: secret,
NODE_ENV: nodeEnvironment,
});
await expect(verifier?.verify(token)).resolves.toEqual({
callerKind: "interactive",
subject: {
scopes: ["knowledge-spaces:write"],
subjectId: "dify-workspace:tenant-1",
tenantId: "tenant-1",
},
});
},
);
it("keeps explicit local auth available alongside Dify JWT auth", async () => {
const verifier = createApiAuthVerifier({
KNOWLEDGE_DEV_AUTH_TOKEN: "local-secret",
KNOWLEDGE_FS_JWT_SECRET: "test-secret-with-at-least-32-bytes",
NODE_ENV: "development",
});
await expect(verifier?.verify("local-secret")).resolves.toEqual({
scopes: ["knowledge-spaces:*"],
subjectId: "dev-user",
tenantId: "tenant-dev",
});
});
it.each(["development", "test"])(
"accepts the default local dev token in %s mode",
async (nodeEnvironment) => {
@ -56,3 +105,12 @@ describe("createApiAuthVerifier", () => {
},
);
});
function signJwt(payload: Readonly<Record<string, unknown>>, secret: string): string {
const header = Buffer.from(JSON.stringify({ alg: "HS256", typ: "JWT" })).toString("base64url");
const claims = Buffer.from(JSON.stringify(payload)).toString("base64url");
const input = `${header}.${claims}`;
const signature = createHmac("sha256", secret).update(input).digest("base64url");
return `${input}.${signature}`;
}

27
knowledge-fs/apps/api/src/auth-options.ts
View File

@ -1,22 +1,35 @@
import { type AuthVerifier, createStaticAuthVerifier } from "@knowledge/api";
import { type AuthVerifier, createJwtAuthVerifier, createStaticAuthVerifier } from "@knowledge/api";
const DEFAULT_LOCAL_AUTH_TOKEN = "dev-token";
const DIFY_JWT_AUDIENCE = "knowledge-fs";
const DIFY_JWT_ISSUER = "dify";
const DIFY_JWT_MAX_TTL_SECONDS = 60;
export interface ApiAuthEnv {
readonly KNOWLEDGE_DEV_AUTH_TOKEN?: string | undefined;
readonly KNOWLEDGE_DEV_SUBJECT_ID?: string | undefined;
readonly KNOWLEDGE_DEV_TENANT_ID?: string | undefined;
readonly KNOWLEDGE_FS_JWT_SECRET?: string | undefined;
readonly NODE_ENV?: string | undefined;
}
export function createApiAuthVerifier(env: ApiAuthEnv = process.env): AuthVerifier | undefined {
const difyJwtSecret = env.KNOWLEDGE_FS_JWT_SECRET?.trim();
const token = getLocalAuthToken(env);
const difyJwtAuth = difyJwtSecret
? createJwtAuthVerifier({
audience: DIFY_JWT_AUDIENCE,
issuer: DIFY_JWT_ISSUER,
maxTtlSeconds: DIFY_JWT_MAX_TTL_SECONDS,
secret: difyJwtSecret,
})
: undefined;
if (!token) {
return undefined;
return difyJwtAuth;
}
return createStaticAuthVerifier({
const localAuth = createStaticAuthVerifier({
subject: {
scopes: ["knowledge-spaces:*"],
subjectId: env.KNOWLEDGE_DEV_SUBJECT_ID?.trim() || "dev-user",
@ -24,6 +37,14 @@ export function createApiAuthVerifier(env: ApiAuthEnv = process.env): AuthVerifi
},
token,
});
if (!difyJwtAuth) {
return localAuth;
}
return {
verify: async (candidate) =>
(await difyJwtAuth.verify(candidate)) ?? localAuth.verify(candidate),
};
}
function getLocalAuthToken(env: ApiAuthEnv): string | undefined {

42
knowledge-fs/apps/api/src/dify-datasource-invocation-client.test.ts
View File

@ -61,6 +61,48 @@ describe("createDifyDatasourceInvocationClient", () => {
expect(JSON.stringify(getOnlineDocumentPages.mock.calls)).not.toContain("credentials");
});
it("maps product crawl options to the Firecrawl datasource parameters", async () => {
const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } }));
const adapter = createDifyDatasourceInvocationClient({
client: difyClient({ getWebsiteCrawl }),
});
const source: Source = {
...SOURCE,
metadata: {
credentialId: "dify-credential-1",
crawlOptions: { includeSubpages: false, limit: 1 },
datasource: "crawl",
parameters: { formats: ["markdown"] },
pluginId: "langgenius/firecrawl_datasource",
provider: "firecrawl",
},
type: "web",
uri: "https://example.com",
};
await collect(
adapter.dispatch({
operation: "get_website_crawl",
source,
tenantId: "tenant-1",
}),
);
expect(getWebsiteCrawl).toHaveBeenCalledWith({
credentialId: "dify-credential-1",
datasource: "crawl",
datasourceParameters: {
crawl_subpages: false,
formats: ["markdown"],
limit: 1,
url: "https://example.com",
},
pluginId: "langgenius/firecrawl_datasource",
provider: "firecrawl",
tenantId: "tenant-1",
});
});
it("rejects inline credentials in integrated mode", async () => {
const adapter = createDifyDatasourceInvocationClient({ client: difyClient() });
const source = {

24
knowledge-fs/apps/api/src/dify-datasource-invocation-client.ts
View File

@ -38,7 +38,11 @@ export function createDifyDatasourceInvocationClient(input: {
case "get_website_crawl":
yield* input.client.getWebsiteCrawl({
...common,
datasourceParameters: withCrawlUrl(config.parameters, invocation.source.uri),
datasourceParameters: withCrawlOptions(
config.parameters,
invocation.source,
invocation.source.uri,
),
});
return;
case "get_online_document_pages":
@ -145,6 +149,24 @@ function withCrawlUrl(parameters: Record<string, unknown>, uri: string): Record<
: { ...parameters, url: uri };
}
function withCrawlOptions(
parameters: Record<string, unknown>,
source: Source,
uri: string,
): Record<string, unknown> {
const crawlOptions = plainObject(source.metadata.crawlOptions);
const includeSubpages = crawlOptions.includeSubpages;
const limit = crawlOptions.limit;
return withCrawlUrl(
{
...parameters,
...(typeof includeSubpages === "boolean" ? { crawl_subpages: includeSubpages } : {}),
...(Number.isSafeInteger(limit) && Number(limit) > 0 ? { limit } : {}),
},
uri,
);
}
function decodeNextPageParameters(token: string): Record<string, unknown> {
try {
const value = JSON.parse(Buffer.from(token, "base64url").toString("utf8")) as unknown;

6
knowledge-fs/apps/api/src/website-crawl-options.test.ts
View File

@ -12,6 +12,7 @@ const WEB_SOURCE: WebsiteCrawlInput["source"] = {
id: "00000000-0000-4000-8000-000000000001",
knowledgeSpaceId: "10000000-0000-4000-8000-000000000001",
metadata: {
crawlOptions: { includeSubpages: false, limit: 1 },
datasource: "crawl",
parameters: { limit: 5 },
pluginId: "langgenius/firecrawl_datasource",
@ -62,7 +63,7 @@ describe("createApiWebsiteCrawlConnector", () => {
const result = await connector.crawl({ source: WEB_SOURCE, tenantId: "tenant-1" });
expect(result).toEqual({
completed: 2,
completed: 1,
pages: [
{
content: "# A",
@ -70,10 +71,9 @@ describe("createApiWebsiteCrawlConnector", () => {
sourceUrl: "https://example.com/a",
title: "A",
},
{ content: "# B", sourceUrl: "https://example.com/b" },
],
status: "completed",
total: 2,
total: 1,
});
expect(calls).toHaveLength(1);

17
knowledge-fs/apps/api/src/website-crawl-options.ts
View File

@ -20,6 +20,7 @@ export function createApiWebsiteCrawlConnector(input: {
return {
crawl: async ({ signal, source, tenantId, userId }): Promise<WebsiteCrawlResult> => {
const pages = new Map<string, CrawledPage>();
const pageLimit = crawlPageLimit(source.metadata.crawlOptions);
let status: string | undefined;
let total: number | undefined;
let completed: number | undefined;
@ -55,15 +56,25 @@ export function createApiWebsiteCrawlConnector(input: {
}
return {
pages: Array.from(pages.values()),
...(completed === undefined ? {} : { completed }),
pages: Array.from(pages.values()).slice(0, pageLimit),
...(completed === undefined ? {} : { completed: Math.min(completed, pageLimit) }),
...(status === undefined ? {} : { status }),
...(total === undefined ? {} : { total }),
...(total === undefined ? {} : { total: Math.min(total, pageLimit) }),
};
},
};
}
function crawlPageLimit(value: unknown): number {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return Number.POSITIVE_INFINITY;
}
const limit = (value as Readonly<Record<string, unknown>>).limit;
return Number.isSafeInteger(limit) && Number(limit) > 0
? Number(limit)
: Number.POSITIVE_INFINITY;
}
export function createApiWebsiteCrawlOptions(input: {
readonly client: ApiDatasourceInvocationClient;
}): {

4
knowledge-fs/packages/api/src/database-deletion-lifecycle-fence-reader.test.ts
View File

@ -60,6 +60,10 @@ describe.each(["postgres", "tidb"] as const)(
expect(calls[0]?.sql).toContain(
dialect === "postgres" ? '"knowledge_space_id" = $2' : "`knowledge_space_id` = ?",
);
if (dialect === "postgres") {
expect(calls[0]?.sql).toContain("$3::uuid IS NOT NULL");
expect(calls[0]?.sql).toContain("$4::uuid IS NOT NULL");
}
expect(calls[0]?.sql).toContain("'knowledge_space'");
expect(calls[0]?.sql).toContain("'source'");
expect(calls[0]?.sql).toContain("'document_asset'");

6
knowledge-fs/packages/api/src/database-deletion-lifecycle-fence-reader.ts
View File

@ -44,9 +44,13 @@ export function createDatabaseDeletionLifecycleFenceReader(
function tombstoneHierarchySql(database: DatabaseAdapter): string {
const q = (identifier: string) => quoteDatabaseIdentifier(database, identifier);
const p = (position: number) => databasePlaceholder(database, position);
const idParam = (position: number) =>
database.dialect === "postgres" ? `${p(position)}::uuid` : p(position);
const sourceId = idParam(3);
const documentAssetId = idParam(4);
const columns = ["id", "tenant_id", "knowledge_space_id", "target_type", "target_id"];
const selected = columns.map(q).join(", ");
return `SELECT ${selected} FROM (SELECT ${selected}, 0 AS ${q("fence_priority")} FROM ${q("deletion_jobs")} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ${q("active_slot")} = 1 UNION ALL SELECT ${selected}, 1 AS ${q("fence_priority")} FROM ${q(tombstoneTable)} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ((${q("target_type")} = 'knowledge_space' AND ${q("target_id")} = ${p(2)}) OR (${q("target_type")} = 'source' AND ((${p(3)} IS NOT NULL AND ${q("target_id")} = ${p(3)}) OR (${p(4)} IS NOT NULL AND ${q("target_id")} IN (SELECT source_document.${q("source_id")} FROM ${q("document_assets")} source_document WHERE source_document.${q("knowledge_space_id")} = ${p(2)} AND source_document.${q("id")} = ${p(4)} AND source_document.${q("source_id")} IS NOT NULL)))) OR (${p(4)} IS NOT NULL AND ${q("target_type")} = 'document_asset' AND ${q("target_id")} = ${p(4)}) OR (${p(4)} IS NOT NULL AND ${q("target_type")} = 'logical_document' AND ${q("target_id")} IN (SELECT logical_revision.${q("document_id")} FROM ${q("document_revisions")} logical_revision WHERE logical_revision.${q("tenant_id")} = ${p(1)} AND logical_revision.${q("knowledge_space_id")} = ${p(2)} AND logical_revision.${q("document_asset_id")} = ${p(4)}))) AS lifecycle_fence ORDER BY ${q("fence_priority")} ASC, CASE ${q("target_type")} WHEN 'knowledge_space' THEN 0 WHEN 'source' THEN 1 ELSE 2 END ASC LIMIT 1;`;
return `SELECT ${selected} FROM (SELECT ${selected}, 0 AS ${q("fence_priority")} FROM ${q("deletion_jobs")} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ${q("active_slot")} = 1 UNION ALL SELECT ${selected}, 1 AS ${q("fence_priority")} FROM ${q(tombstoneTable)} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ((${q("target_type")} = 'knowledge_space' AND ${q("target_id")} = ${p(2)}) OR (${q("target_type")} = 'source' AND ((${sourceId} IS NOT NULL AND ${q("target_id")} = ${sourceId}) OR (${documentAssetId} IS NOT NULL AND ${q("target_id")} IN (SELECT source_document.${q("source_id")} FROM ${q("document_assets")} source_document WHERE source_document.${q("knowledge_space_id")} = ${p(2)} AND source_document.${q("id")} = ${documentAssetId} AND source_document.${q("source_id")} IS NOT NULL)))) OR (${documentAssetId} IS NOT NULL AND ${q("target_type")} = 'document_asset' AND ${q("target_id")} = ${documentAssetId}) OR (${documentAssetId} IS NOT NULL AND ${q("target_type")} = 'logical_document' AND ${q("target_id")} IN (SELECT logical_revision.${q("document_id")} FROM ${q("document_revisions")} logical_revision WHERE logical_revision.${q("tenant_id")} = ${p(1)} AND logical_revision.${q("knowledge_space_id")} = ${p(2)} AND logical_revision.${q("document_asset_id")} = ${documentAssetId})))) AS lifecycle_fence ORDER BY ${q("fence_priority")} ASC, CASE ${q("target_type")} WHEN 'knowledge_space' THEN 0 WHEN 'source' THEN 1 ELSE 2 END ASC LIMIT 1;`;
}
function mapFence(row: DatabaseRow): ActiveDeletionLifecycleFence {

148
knowledge-fs/packages/api/src/database-durable-deletion-target-capabilities.test.ts
View File

@ -279,9 +279,14 @@ describe("database durable deletion target capabilities", () => {
signal,
}),
).resolves.toEqual({ complete: true, items: [], scanPhase: "document_objects:6" });
expect(
calls.filter((call) => call.tableName === "document_multimodal_manifests"),
).toHaveLength(3);
const manifestDatabaseCalls = calls.filter(
(call) => call.tableName === "document_multimodal_manifests",
);
expect(manifestDatabaseCalls).toHaveLength(3);
expect(manifestDatabaseCalls[0]?.params).toEqual(["tenant-a", spaceId, targetDocumentId]);
expect(manifestDatabaseCalls[0]?.sql).not.toContain(
dialect === "postgres" ? 'manifest."id" >' : "manifest.`id` >",
);
});
it(`inventories and executes space objects, lifecycle secrets, source secrets, and cache items (${dialect})`, async () => {
@ -371,6 +376,18 @@ describe("database durable deletion target capabilities", () => {
],
scanPhase: "source_secrets",
});
expect(
calls.find((call) => call.operation === "select" && call.tableName === "sources")?.sql,
).toContain(dialect === "postgres" ? `"credential_ref" <> ''` : "`credential_ref` <> ''");
expect(
calls.find(
(call) =>
call.operation === "select" && call.tableName === "source_secret_lifecycle_refs",
)?.params,
).toEqual(["tenant-a", spaceId, null, 2]);
expect(
calls.find((call) => call.operation === "select" && call.tableName === "sources")?.params,
).toEqual(["tenant-a", spaceId, null, 2]);
await capabilities.executeExternalItem({
item: deletionItem("object", { objectKey: firstObjectKey }),
@ -419,6 +436,26 @@ describe("database durable deletion target capabilities", () => {
}
});
it(`uses a nullable UUID cursor for the first document inventory page (${dialect})`, async () => {
const calls: DatabaseExecuteInput[] = [];
const capabilities = capabilitiesFor(dialect, async (input) => {
calls.push(input);
return result([]);
});
await capabilities.inventory({
job: job({ targetType: "source" }),
limit: 2,
signal: new AbortController().signal,
});
const documentCall = calls.find(
(call) => call.operation === "select" && call.tableName === "document_assets",
);
expect(documentCall?.params).toEqual([spaceId, null, targetDocumentId]);
expect(documentCall?.sql).toContain("COALESCE");
});
it(`publishes a target-free, graph-closed head while preserving unrelated Deep members (${dialect})`, async () => {
const calls: DatabaseExecuteInput[] = [];
let targetProbeCount = 0;
@ -861,9 +898,11 @@ describe("database durable deletion target capabilities", () => {
(call) => call.operation === "select" && call.tableName === "knowledge_fs_sessions",
),
).toBe(true);
expect(
calls.some((call) => call.operation === "select" && call.tableName === "golden_questions"),
).toBe(true);
const goldenQuestionSelect = calls.find(
(call) => call.operation === "select" && call.tableName === "golden_questions",
);
expect(goldenQuestionSelect?.params).toEqual([spaceId, 7]);
expect(goldenQuestionSelect?.sql).toContain("1 = 1");
expect(
calls.some(
(call) => call.operation === "select" && call.tableName === "research_task_jobs",
@ -890,6 +929,45 @@ describe("database durable deletion target capabilities", () => {
expect(calls.some((call) => call.tableName === "answer_traces")).toBe(false);
});
it(`preserves the original derived-cleanup error when the transaction is aborted (${dialect})`, async () => {
let fenceChecks = 0;
const execute = async (input: DatabaseExecuteInput): Promise<DatabaseExecuteResult> => {
if (input.operation === "select" && input.tableName === "deletion_jobs") {
fenceChecks += 1;
if (fenceChecks > 1) throw new Error("transaction aborted");
return result([{ id: job().id }]);
}
if (input.operation === "select" && input.tableName === "golden_questions") {
throw new Error("golden question cleanup failed");
}
return result([]);
};
const database = createSchemaDatabaseAdapter({
executor: execute,
kind: dialect,
transaction: async (callback) => callback({ execute }),
});
const capabilities = createDatabaseDurableDeletionTargetCapabilities({
cache: createMemoryCacheAdapter({ maxEntries: 10 }),
database,
objectStorage: createMemoryObjectStorageAdapter({
kind: "memory",
maxObjectBytes: 1_024,
}),
secretStore: { delete: vi.fn(async () => undefined) },
});
await expect(
capabilities.deleteDerivedDataPage({
job: job({ deleteMode: "keep", targetType: "source" }),
limit: 7,
signal: new AbortController().signal,
}),
).rejects.toThrow("golden question cleanup failed");
expect(fenceChecks).toBe(1);
});
it(`source keep still drains live whole-space Research writers (${dialect})`, async () => {
const calls: DatabaseExecuteInput[] = [];
const execute = async (input: DatabaseExecuteInput): Promise<DatabaseExecuteResult> => {
@ -1004,6 +1082,14 @@ describe("database durable deletion target capabilities", () => {
expect(childUpdates[0]?.sql).toContain("deleting_at");
expect(childUpdates[1]?.sql).toContain("deletion_job_id");
expect(childUpdates[1]?.sql).toContain("IS NULL");
const logicalDocumentUpdate = calls.find(
(call) => call.operation === "update" && call.tableName === "logical_documents",
);
expect(logicalDocumentUpdate?.sql).toContain(
dialect === "postgres"
? `"provider_item_digest" = NULL`
: "`provider_item_digest` = NULL",
);
const childResidue = calls.find(
(call) => call.operation === "select" && call.tableName === "document_assets",
);
@ -1116,6 +1202,13 @@ describe("database durable deletion target capabilities", () => {
expect(select?.sql).toContain("knowledge_space_staged_commits");
expect(select?.sql).not.toContain('target_lease."document_asset_id"');
expect(select?.sql).not.toContain("target_lease.`document_asset_id`");
if (dialect === "postgres") {
expect(select?.sql).toContain('target_lease."target_id" = CAST(CAST($2 AS UUID) AS TEXT)');
expect(select?.sql).toContain('target_lease."target_id" = CAST(CAST($3 AS UUID) AS TEXT)');
expect(select?.sql).toContain(
"semantic_document_ref.document_asset_id = CAST(CAST($3 AS UUID) AS TEXT)",
);
}
expect(
calls.find(
(call) => call.operation === "delete" && call.tableName === "knowledge_fs_leases",
@ -1184,6 +1277,7 @@ describe("database durable deletion target capabilities", () => {
["document_multimodal_manifests", [{ id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2d25" }]],
["knowledge_paths", [{ id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2d26" }]],
["knowledge_space_staged_commits", [{ id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2d27" }]],
["parse_artifacts", [{ id: "018f0d60-7a49-7cc2-9c1b-5b36f18f2d28" }]],
]);
const calls: DatabaseExecuteInput[] = [];
const execute = async (input: DatabaseExecuteInput): Promise<DatabaseExecuteResult> => {
@ -1198,7 +1292,7 @@ describe("database durable deletion target capabilities", () => {
};
const capabilities = capabilitiesFor(dialect, execute);
for (let page = 0; page < 7; page += 1) {
for (let page = 0; page < 8; page += 1) {
await expect(
capabilities.deleteDerivedDataPage({
job: job(),
@ -1220,6 +1314,7 @@ describe("database durable deletion target capabilities", () => {
"document_outlines",
"document_multimodal_manifests",
"knowledge_space_staged_commits",
"parse_artifacts",
]);
const pathSelect = calls.find(
(call) => call.operation === "select" && call.tableName === "knowledge_paths",
@ -1229,6 +1324,45 @@ describe("database durable deletion target capabilities", () => {
expect(pathSelect?.sql).toContain("documentAssetIds");
expect(pathSelect?.sql).toContain("sourceSummaryNodeIds");
expect(pathSelect?.sql).toContain("communityId");
const parseArtifactSelect = calls.find(
(call) => call.operation === "select" && call.tableName === "parse_artifacts",
);
expect(parseArtifactSelect?.sql).toContain("document_assets");
expect(parseArtifactSelect?.sql).toContain("knowledge_space_id");
});
it(`scopes the final parse-artifact residue probe to the target knowledge space (${dialect})`, async () => {
const calls: DatabaseExecuteInput[] = [];
const execute = async (input: DatabaseExecuteInput): Promise<DatabaseExecuteResult> => {
calls.push(input);
if (input.operation === "select" && input.tableName === "deletion_jobs") {
return result([{ id: job().id }]);
}
if (input.operation === "select" && input.tableName === "knowledge_space_manifests") {
return result([{ object_key_prefix: `tenant-a/spaces/${spaceId}` }]);
}
return result([]);
};
const targetJob = job();
await expect(
capabilitiesFor(dialect, execute).deletePrimaryData({
job: targetJob,
leaseFence: {
deletionJobId: targetJob.id,
expectedRowVersion: targetJob.rowVersion,
leaseToken: targetJob.leaseToken as string,
},
signal: new AbortController().signal,
transaction: { execute },
}),
).resolves.toEqual({ clean: true });
const parseArtifactProbe = calls.find(
(call) => call.operation === "select" && call.tableName === "parse_artifacts",
);
expect(parseArtifactProbe?.sql).toContain("document_assets");
expect(parseArtifactProbe?.sql).toContain("knowledge_space_id");
});
it(`fails the space primary proof when any cascaded derived row survives (${dialect})`, async () => {

64
knowledge-fs/packages/api/src/database-durable-deletion-target-capabilities.ts
View File

@ -369,6 +369,7 @@ export function createDatabaseDurableDeletionTargetCapabilities({
});
throwIfAborted(signal);
const preservesDocuments = job.targetType === "source" && job.deleteMode === "keep";
let operationFailed = false;
try {
// Command logs, KnowledgeFS session metadata, Golden Question metadata, and Research inputs
// are opaque JSON. They cannot be attributed safely to one document/source, so every target
@ -544,10 +545,13 @@ export function createDatabaseDurableDeletionTargetCapabilities({
}
}
return { complete: true, deleted: 0 };
} catch (error) {
operationFailed = true;
throw error;
} finally {
// A cache/object adapter call can outlive the original lease. Recheck immediately before
// commit so every DB page rolls back when the worker fence expired mid-operation.
await assertJobFence(database, transaction, job);
if (!operationFailed) await assertJobFence(database, transaction, job);
}
});
},
@ -1980,7 +1984,7 @@ async function cleanupLogicalDocumentsForPrimaryDeletion(
maxRows: 0,
operation: "update",
params: [job.tenantId, job.knowledgeSpaceId, job.targetId, job.updatedAt],
sql: `UPDATE ${q("logical_documents")} SET ${q("source_id")} = NULL, ${q("provider_item_id")} = NULL, ${q("system_metadata")} = ${scrubSourceIdentityMetadataSql(database, q("system_metadata"), false)}, ${q("row_version")} = ${q("row_version")} + 1, ${q("updated_at")} = ${p(4)} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ${q("source_id")} = ${p(3)};`,
sql: `UPDATE ${q("logical_documents")} SET ${q("source_id")} = NULL, ${q("provider_item_id")} = NULL, ${q("provider_item_digest")} = NULL, ${q("system_metadata")} = ${scrubSourceIdentityMetadataSql(database, q("system_metadata"), false)}, ${q("row_version")} = ${q("row_version")} + 1, ${q("updated_at")} = ${p(4)} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ${q("source_id")} = ${p(3)};`,
tableName: "logical_documents",
});
return;
@ -2763,7 +2767,8 @@ async function nextTargetDocumentId(
if (job.targetType === "document_asset") return cursor ? undefined : job.targetId;
const q = (value: string) => quoteDatabaseIdentifier(database, value);
const p = (position: number) => databasePlaceholder(database, position);
const params: DatabaseQueryValue[] = [job.knowledgeSpaceId, cursor ?? ""];
const params: DatabaseQueryValue[] = [job.knowledgeSpaceId, cursor ?? null];
const after = nullableUuidCursorExpression(database, p(2));
let target = "";
if (job.targetType === "source") {
params.push(job.targetId);
@ -2776,7 +2781,7 @@ async function nextTargetDocumentId(
maxRows: 1,
operation: "select",
params,
sql: `SELECT ${q("id")} FROM ${q("document_assets")} WHERE ${q("knowledge_space_id")} = ${p(1)} AND ${q("id")} > ${p(2)}${target} ORDER BY ${q("id")} ASC LIMIT 1;`,
sql: `SELECT ${q("id")} FROM ${q("document_assets")} WHERE ${q("knowledge_space_id")} = ${p(1)} AND ${q("id")} > ${after}${target} ORDER BY ${q("id")} ASC LIMIT 1;`,
tableName: "document_assets",
});
return result.rows[0] ? stringColumn(result.rows[0], "id") : undefined;
@ -2801,12 +2806,12 @@ async function documentManifestObjectKeyPage(
const p = (position: number) => databasePlaceholder(database, position);
const activeId = state.manifestActiveId;
const params: DatabaseQueryValue[] = [job.tenantId, job.knowledgeSpaceId, documentId];
let cursorPredicate: string;
let cursorPredicate = "";
if (activeId) {
params.push(activeId);
cursorPredicate = ` AND manifest.${q("id")} = ${p(4)}`;
} else {
params.push(state.manifestAfter ?? "");
} else if (state.manifestAfter) {
params.push(state.manifestAfter);
cursorPredicate = ` AND manifest.${q("id")} > ${p(4)}`;
}
const result = await database.execute({
@ -2912,6 +2917,11 @@ interface SecretInventoryRef {
readonly rowId: string;
}
function nullableUuidCursorExpression(database: DatabaseAdapter, placeholder: string): string {
const value = `COALESCE(${placeholder}, '00000000-0000-0000-0000-000000000000')`;
return database.dialect === "postgres" ? `CAST(${value} AS UUID)` : value;
}
async function lifecycleSecretRefs(
database: DatabaseAdapter,
job: DurableDeletionTargetOperationInput["job"],
@ -2921,7 +2931,8 @@ async function lifecycleSecretRefs(
if (job.targetType === "document_asset" || job.targetType === "logical_document") return [];
const q = (value: string) => quoteDatabaseIdentifier(database, value);
const p = (position: number) => databasePlaceholder(database, position);
const params: DatabaseQueryValue[] = [job.tenantId, job.knowledgeSpaceId, cursor ?? "", limit];
const params: DatabaseQueryValue[] = [job.tenantId, job.knowledgeSpaceId, cursor ?? null, limit];
const after = nullableUuidCursorExpression(database, p(3));
let target = "";
if (job.targetType === "source") {
params.push(job.targetId);
@ -2931,7 +2942,7 @@ async function lifecycleSecretRefs(
maxRows: limit,
operation: "select",
params,
sql: `SELECT ${q("id")}, ${q("source_id")}, ${q("credential_ref")} FROM ${q("source_secret_lifecycle_refs")} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ${q("id")} > ${p(3)} AND ${q("state")} <> 'deleted'${target} ORDER BY ${q("id")} ASC LIMIT ${p(4)};`,
sql: `SELECT ${q("id")}, ${q("source_id")}, ${q("credential_ref")} FROM ${q("source_secret_lifecycle_refs")} WHERE ${q("tenant_id")} = ${p(1)} AND ${q("knowledge_space_id")} = ${p(2)} AND ${q("id")} > ${after} AND ${q("state")} <> 'deleted'${target} ORDER BY ${q("id")} ASC LIMIT ${p(4)};`,
tableName: "source_secret_lifecycle_refs",
});
return result.rows.map((row) => ({
@ -2965,7 +2976,8 @@ async function sourceSecretRefs(
if (job.targetType === "document_asset" || job.targetType === "logical_document") return [];
const q = (value: string) => quoteDatabaseIdentifier(database, value);
const p = (position: number) => databasePlaceholder(database, position);
const params: DatabaseQueryValue[] = [job.tenantId, job.knowledgeSpaceId, cursor ?? "", limit];
const params: DatabaseQueryValue[] = [job.tenantId, job.knowledgeSpaceId, cursor ?? null, limit];
const after = nullableUuidCursorExpression(database, p(3));
let target = "";
if (job.targetType === "source") {
params.push(job.targetId);
@ -2975,7 +2987,7 @@ async function sourceSecretRefs(
maxRows: limit,
operation: "select",
params,
sql: `SELECT s.${q("id")}, s.${q("credential_ref")} FROM ${q("sources")} s WHERE s.${q("knowledge_space_id")} = ${p(2)} AND s.${q("id")} > ${p(3)} AND s.${q("credential_ref")} IS NOT NULL${target} AND EXISTS (SELECT 1 FROM ${q("knowledge_spaces")} ks WHERE ks.${q("tenant_id")} = ${p(1)} AND ks.${q("id")} = ${p(2)}) AND NOT EXISTS (SELECT 1 FROM ${q("source_secret_lifecycle_refs")} lifecycle WHERE lifecycle.${q("tenant_id")} = ${p(1)} AND lifecycle.${q("knowledge_space_id")} = ${p(2)} AND lifecycle.${q("source_id")} = s.${q("id")} AND lifecycle.${q("credential_ref")} = s.${q("credential_ref")}) ORDER BY s.${q("id")} ASC LIMIT ${p(4)};`,
sql: `SELECT s.${q("id")}, s.${q("credential_ref")} FROM ${q("sources")} s WHERE s.${q("knowledge_space_id")} = ${p(2)} AND s.${q("id")} > ${after} AND s.${q("credential_ref")} IS NOT NULL AND s.${q("credential_ref")} <> ''${target} AND EXISTS (SELECT 1 FROM ${q("knowledge_spaces")} ks WHERE ks.${q("tenant_id")} = ${p(1)} AND ks.${q("id")} = ${p(2)}) AND NOT EXISTS (SELECT 1 FROM ${q("source_secret_lifecycle_refs")} lifecycle WHERE lifecycle.${q("tenant_id")} = ${p(1)} AND lifecycle.${q("knowledge_space_id")} = ${p(2)} AND lifecycle.${q("source_id")} = s.${q("id")} AND lifecycle.${q("credential_ref")} = s.${q("credential_ref")}) ORDER BY s.${q("id")} ASC LIMIT ${p(4)};`,
tableName: "sources",
});
return result.rows.map((row) => ({
@ -3128,7 +3140,7 @@ async function deleteGoldenQuestionPage(
const q = (value: string) => quoteDatabaseIdentifier(database, value);
const p = (position: number) => databasePlaceholder(database, position);
return database.transaction(async (transaction) => {
const params = targetDocumentQueryParams(job);
const params = targetGoldenQuestionQueryParams(job);
params.push(limit);
const alias = "target_golden_question";
const rows = await transaction.execute({
@ -3257,6 +3269,15 @@ function goldenMissingEvidencePredicateSql(
return `EXISTS (SELECT 1 FROM JSON_TABLE(${metadata}, '$.evidenceContext.missingEvidence[*]' COLUMNS (evidence_id VARCHAR(255) PATH '$.expectedEvidenceId')) AS golden_missing WHERE ${target})`;
}
function targetGoldenQuestionQueryParams(
job: DurableDeletionTargetOperationInput["job"],
): DatabaseQueryValue[] {
return job.targetType === "knowledge_space" ||
(job.targetType === "source" && job.deleteMode === "keep")
? [job.knowledgeSpaceId]
: targetDocumentQueryParams(job);
}
function targetDocumentQueryParams(
job: DurableDeletionTargetOperationInput["job"],
): DatabaseQueryValue[] {
@ -3354,7 +3375,12 @@ function targetDocumentMembershipAtSql(
const q = (value: string) => quoteDatabaseIdentifier(database, value);
const p = (position: number) => databasePlaceholder(database, position);
if (job.targetType === "document_asset") {
return `${documentIdExpression} = ${p(targetParamPosition)}`;
const targetParameter = textComparison
? database.dialect === "postgres"
? `CAST(CAST(${p(targetParamPosition)} AS UUID) AS TEXT)`
: `CAST(${p(targetParamPosition)} AS CHAR(36))`
: p(targetParamPosition);
return `${documentIdExpression} = ${targetParameter}`;
}
if (job.targetType === "logical_document") {
const selectedRevisionAsset = textComparison
@ -3589,7 +3615,7 @@ async function deleteDocumentDerivedResiduePage(
maxRows: limit,
operation: "select",
params,
sql: `SELECT artifact.${q("id")} FROM ${q("parse_artifacts")} AS artifact WHERE artifact.${q("document_asset_id")} ${documentPredicate} ORDER BY artifact.${q("id")} ASC LIMIT ${p(2)};`,
sql: `SELECT artifact.${q("id")} FROM ${q("parse_artifacts")} AS artifact WHERE artifact.${q("document_asset_id")} ${documentPredicate} AND EXISTS (SELECT 1 FROM ${q("document_assets")} AS target_document WHERE target_document.${q("id")} = artifact.${q("document_asset_id")} AND target_document.${q("knowledge_space_id")} = ${p(1)}) ORDER BY artifact.${q("id")} ASC LIMIT ${p(2)};`,
tableName: "parse_artifacts",
});
const parseArtifactIds = parseArtifacts.rows.map((row) => stringColumn(row, "id"));
@ -3797,7 +3823,7 @@ async function hasTargetGoldenQuestionResidue(
const result = await executor.execute({
maxRows: 1,
operation: "select",
params: targetDocumentQueryParams(job),
params: targetGoldenQuestionQueryParams(job),
sql: `SELECT ${alias}.${q("id")} FROM ${q("golden_questions")} AS ${alias} WHERE ${alias}.${q("knowledge_space_id")} = ${p(1)} AND ${targetGoldenQuestionPredicateSql(database, job, alias)} LIMIT 1;`,
tableName: "golden_questions",
});
@ -4145,7 +4171,7 @@ async function hasTargetDocumentForeignKeyResidue(
table: "artifact_segments",
},
{
sql: `SELECT artifact.${q("id")} FROM ${q("parse_artifacts")} AS artifact WHERE artifact.${q("document_asset_id")} ${documentPredicate} LIMIT 1;`,
sql: `SELECT artifact.${q("id")} FROM ${q("parse_artifacts")} AS artifact WHERE artifact.${q("document_asset_id")} ${documentPredicate} AND EXISTS (SELECT 1 FROM ${q("document_assets")} AS target_document WHERE target_document.${q("id")} = artifact.${q("document_asset_id")} AND target_document.${q("knowledge_space_id")} = ${p(1)}) LIMIT 1;`,
table: "parse_artifacts",
},
] as const;
@ -4373,6 +4399,10 @@ function targetKnowledgeFsLeasePredicateSql(
const q = (value: string) => quoteDatabaseIdentifier(database, value);
const p = (position: number) => databasePlaceholder(database, position);
const field = (column: string) => `${leaseAlias}.${q(column)}`;
const textParam = (position: number) =>
database.dialect === "postgres"
? `CAST(CAST(${p(position)} AS UUID) AS TEXT)`
: `CAST(${p(position)} AS CHAR(36))`;
const scope = `${field("tenant_id")} = ${p(1)} AND ${field("knowledge_space_id")} = ${p(2)}`;
if (job.targetType === "knowledge_space") return scope;
@ -4406,7 +4436,7 @@ function targetKnowledgeFsLeasePredicateSql(
const pathTarget = `${field("target_type")} = 'knowledge-path' AND EXISTS (SELECT 1 FROM ${q("knowledge_paths")} AS target_path WHERE target_path.${q("knowledge_space_id")} = ${p(2)} AND (${field("target_id")} = ${castId("target_path")} OR ${field("target_id")} = target_path.${q("target_id")} OR ${field("virtual_path")} = target_path.${q("virtual_path")}) AND ${targetSemanticPathPredicateSql(database, uuidDocumentPredicate, textDocumentPredicate, 2, "target_path")})`;
const stagedCommitTarget = `${field("target_type")} = 'staged-commit' AND EXISTS (SELECT 1 FROM ${q("knowledge_space_staged_commits")} AS target_commit WHERE target_commit.${q("tenant_id")} = ${p(1)} AND target_commit.${q("knowledge_space_id")} = ${p(2)} AND ${uuidDocumentMembership(`target_commit.${q("document_asset_id")}`)} AND (${field("target_id")} = ${castId("target_commit")} OR ${field("target_id")} = target_commit.${q("raw_object_key")} OR ${field("target_id")} = target_commit.${q("published_object_key")}))`;
return `${scope} AND ((${field("target_type")} = 'knowledge-space' AND ${field("target_id")} = ${p(2)}) OR (${field("target_type")} = 'document-asset' AND ${textDocumentMembership(field("target_id"))}) OR ${documentVirtualPath} OR ${textDocumentMembership(metadataDocumentId)} OR (${parseArtifactTarget}) OR (${projectionTarget}) OR (${pathTarget}) OR (${stagedCommitTarget}))`;
return `${scope} AND ((${field("target_type")} = 'knowledge-space' AND ${field("target_id")} = ${textParam(2)}) OR (${field("target_type")} = 'document-asset' AND ${textDocumentMembership(field("target_id"))}) OR ${documentVirtualPath} OR ${textDocumentMembership(metadataDocumentId)} OR (${parseArtifactTarget}) OR (${projectionTarget}) OR (${pathTarget}) OR (${stagedCommitTarget}))`;
}
async function hasActiveMutationLease(

9
knowledge-fs/packages/api/src/database-row-utils.test.ts
View File

@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import {
nonnegativeSafeIntegerColumn,
numberColumn,
optionalNumberColumn,
optionalStringColumn,
@ -17,6 +18,8 @@ describe("database-row-utils", () => {
it("reads required and optional number columns", () => {
expect(numberColumn({ count: 3 }, "count")).toBe(3);
expect(nonnegativeSafeIntegerColumn({ count: "3" }, "count")).toBe(3);
expect(nonnegativeSafeIntegerColumn({ count: 3 }, "count")).toBe(3);
expect(optionalNumberColumn({ count: null }, "count")).toBeUndefined();
expect(optionalNumberColumn({ count: undefined }, "count")).toBeUndefined();
expect(optionalNumberColumn({ count: 3 }, "count")).toBe(3);
@ -35,5 +38,11 @@ describe("database-row-utils", () => {
expect(() => optionalNumberColumn({ count: "3" }, "count")).toThrow(
"Database row column count must be a number",
);
expect(() => nonnegativeSafeIntegerColumn({ count: "-1" }, "count")).toThrow(
"Database row column count must be a nonnegative safe integer",
);
expect(() => nonnegativeSafeIntegerColumn({ count: "9007199254740992" }, "count")).toThrow(
"Database row column count must be a nonnegative safe integer",
);
});
});

11
knowledge-fs/packages/api/src/database-row-utils.ts
View File

@ -34,6 +34,17 @@ export function numberColumn(row: DatabaseRow, column: string): number {
return value;
}
export function nonnegativeSafeIntegerColumn(row: DatabaseRow, column: string): number {
const raw = row[column];
const value = typeof raw === "string" && /^\d+$/u.test(raw) ? Number(raw) : raw;
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) {
throw new Error(`Database row column ${column} must be a nonnegative safe integer`);
}
return value;
}
export function optionalNumberColumn(row: DatabaseRow, column: string): number | undefined {
const value = row[column];

6
knowledge-fs/packages/api/src/dify-capability-v2.test.ts
View File

@ -266,6 +266,12 @@ describe("Dify Capability v2 request guard", () => {
"sources.crawl",
"source",
],
createSourceSyncWorkflow: [
"POST",
"/knowledge-spaces/{id}/sources/{sourceId}/sync",
"source_workflows.sync.create",
"source",
],
getAnswerTrace: ["GET", "/queries/{traceId}", "queries.read", "query"],
getBulkOperation: ["GET", "/bulk-jobs/{id}", "bulk_jobs.read", "job"],
getDocument: [

145
knowledge-fs/packages/api/src/dify-capability-v2.ts
View File

@ -521,6 +521,25 @@ export const DIFY_CAPABILITY_V2_OPERATIONS: readonly DifyCapabilityV2Operation[]
resource: { pathParameter: "id" },
resourceType: "knowledge_space",
},
{
action: "logical_documents.list",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "listLogicalDocuments",
pathTemplate: "/knowledge-spaces/{id}/logical-documents",
resource: { pathParameter: "id" },
resourceType: "knowledge_space",
},
{
action: "logical_documents.read",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "getLogicalDocument",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/logical-documents/{documentId}",
resource: { pathParameter: "documentId" },
resourceType: "document",
},
{
action: "documents.create",
allowedCallerKinds: STANDARD_CALLERS,
@ -746,6 +765,132 @@ export const DIFY_CAPABILITY_V2_OPERATIONS: readonly DifyCapabilityV2Operation[]
resource: { pathParameter: "sourceId" },
resourceType: "source",
},
{
action: "source_workflows.sync.create",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "createSourceSyncWorkflow",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/sources/{sourceId}/sync",
resource: { pathParameter: "sourceId" },
resourceType: "source",
},
{
action: "source_providers.list",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "listSourceProviders",
pathTemplate: "/source-providers",
resource: { namespace: true },
resourceType: "namespace",
},
{
action: "source_connections.create",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "createSourceConnection",
pathTemplate: "/knowledge-spaces/{id}/source-connections",
resource: { pathParameter: "id" },
resourceType: "knowledge_space",
},
{
action: "source_connections.list",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "listSourceConnections",
pathTemplate: "/knowledge-spaces/{id}/source-connections",
resource: { pathParameter: "id" },
resourceType: "knowledge_space",
},
{
action: "source_connections.refresh",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "refreshSourceConnection",
pathTemplate: "/knowledge-spaces/{id}/source-connections/{connectionId}/refresh",
resource: { pathParameter: "id" },
resourceType: "knowledge_space",
},
{
action: "source_workflows.preview.create",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "createSourceCrawlPreviewWorkflow",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/sources/{sourceId}/crawl-preview",
resource: { pathParameter: "sourceId" },
resourceType: "source",
},
{
action: "source_sync_policies.read",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "getSourceSyncPolicy",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
resource: { pathParameter: "sourceId" },
resourceType: "source",
},
{
action: "source_sync_policies.update",
allowedCallerKinds: STANDARD_CALLERS,
method: "PUT",
operationId: "putSourceSyncPolicy",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
resource: { pathParameter: "sourceId" },
resourceType: "source",
},
{
action: "source_workflows.read",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "getSourceWorkflow",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/source-workflows/{runId}",
resource: { pathParameter: "runId" },
resourceType: "job",
},
{
action: "source_workflows.cancel",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "cancelSourceWorkflow",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/source-workflows/{runId}/cancel",
resource: { pathParameter: "runId" },
resourceType: "job",
},
{
action: "source_workflows.retry",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "retrySourceWorkflow",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/source-workflows/{runId}/retry",
resource: { pathParameter: "runId" },
resourceType: "job",
},
{
action: "source_workflows.pages.list",
allowedCallerKinds: STANDARD_CALLERS,
method: "GET",
operationId: "listCrawlPreviewPages",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/source-workflows/{runId}/pages",
resource: { pathParameter: "runId" },
resourceType: "job",
},
{
action: "source_workflows.selection.create",
allowedCallerKinds: STANDARD_CALLERS,
method: "POST",
operationId: "selectCrawlPreviewPages",
parentResource: { pathParameter: "id" },
pathTemplate: "/knowledge-spaces/{id}/source-workflows/{runId}/selection",
resource: { pathParameter: "runId" },
resourceType: "job",
},
{
action: "sources.crawl",
allowedCallerKinds: STANDARD_CALLERS,

4
knowledge-fs/packages/api/src/document-compilation-attempt-repository.test.ts
View File

@ -1311,8 +1311,10 @@ describe("database document compilation attempt repository", () => {
knowledgeSpaceId,
candidatePublicationId,
candidateFingerprint,
"candidate",
2,
]);
expect(fake.calls[1]?.sql).toContain("'published'");
expect(fake.calls[1]?.sql).toContain("projection_set_publication_heads");
expect(fake.calls[1]?.sql).toContain("FOR UPDATE");
});

34
knowledge-fs/packages/api/src/document-compilation-attempt-repository.ts
View File

@ -3043,7 +3043,7 @@ function productIntentRestoreConflict(): DocumentCompilationAttemptTransitionErr
async function requireDatabaseCandidateBinding(
database: DatabaseAdapter,
transaction: DatabaseExecutor,
attempt: Pick<DocumentCompilationAttempt, "knowledgeSpaceId" | "tenantId">,
attempt: Pick<DocumentCompilationAttempt, "baseHeadRevision" | "knowledgeSpaceId" | "tenantId">,
candidate: { readonly candidateFingerprint: string; readonly candidatePublicationId: string },
): Promise<void> {
const result = await transaction.execute({
@ -3054,7 +3054,7 @@ async function requireDatabaseCandidateBinding(
uuid(attempt.knowledgeSpaceId, "knowledgeSpaceId"),
uuid(candidate.candidatePublicationId, "candidatePublicationId"),
ProjectionSetFingerprintSchema.parse(candidate.candidateFingerprint),
"candidate",
nonnegativeInteger(attempt.baseHeadRevision, "baseHeadRevision"),
],
sql: `SELECT ${quoteDatabaseIdentifier(database, "id")} FROM ${quoteDatabaseIdentifier(
database,
@ -3071,10 +3071,34 @@ async function requireDatabaseCandidateBinding(
)} AND ${quoteDatabaseIdentifier(database, "fingerprint")} = ${databasePlaceholder(
database,
4,
)} AND ${quoteDatabaseIdentifier(database, "status")} = ${databasePlaceholder(
)} AND (${quoteDatabaseIdentifier(database, "status")} = 'candidate' OR (${quoteDatabaseIdentifier(
database,
5,
)} LIMIT 1 FOR UPDATE;`,
"status",
)} = 'published' AND EXISTS (SELECT 1 FROM ${quoteDatabaseIdentifier(
database,
"projection_set_publication_heads",
)} AS publication_head WHERE publication_head.${quoteDatabaseIdentifier(
database,
"tenant_id",
)} = ${quoteDatabaseIdentifier(database, publicationTableName)}.${quoteDatabaseIdentifier(
database,
"tenant_id",
)} AND publication_head.${quoteDatabaseIdentifier(
database,
"knowledge_space_id",
)} = ${quoteDatabaseIdentifier(database, publicationTableName)}.${quoteDatabaseIdentifier(
database,
"knowledge_space_id",
)} AND publication_head.${quoteDatabaseIdentifier(
database,
"publication_id",
)} = ${quoteDatabaseIdentifier(database, publicationTableName)}.${quoteDatabaseIdentifier(
database,
"id",
)} AND publication_head.${quoteDatabaseIdentifier(
database,
"head_revision",
)} = ${databasePlaceholder(database, 5)}))) LIMIT 1 FOR UPDATE;`,
tableName: publicationTableName,
});
if (!result.rows[0]) {

84
knowledge-fs/packages/api/src/document-compilation-publication-coordinator.test.ts
View File

@ -288,6 +288,90 @@ describe("document compilation publication coordinator", () => {
expect(compose).not.toHaveBeenCalled();
});
it("completes a rebuild as a no-op when it resolves to the current published snapshot", async () => {
const publications = createInMemoryProjectionSetPublicationRepository({ maxPublications: 10 });
const execution = fakeExecution(attempt());
const members = createInMemoryProjectionSetPublicationMemberRepository({
attempts: {
get: async (id) => (id === execution.context.attempt.id ? execution.context.attempt : null),
},
maxListLimit: 10,
maxMembers: 10,
publications,
});
const compose = vi.spyOn(members, "composeDocumentCandidate");
const material = fingerprintMaterial();
const fingerprint = await buildProjectionSetFingerprint(material);
await publications.createCandidate({
createdAt: now,
fingerprint,
id: conflictingPublicationId,
knowledgeSpaceId,
metadata: {
[DocumentCompilationCandidateMetadataKey]: {
attemptId: conflictingPublicationId,
},
},
projectionVersion: 3,
tenantId,
});
await publications.publish({
expectedHeadRevision: 0,
fingerprint,
knowledgeSpaceId,
tenantId,
updatedAt: now,
});
const coordinator = createDocumentCompilationPublicationCoordinator({
maxComponents: 100,
members,
publications,
validator: allowingValidator(),
});
await expect(
coordinator.composeCandidate({
candidateId: candidatePublicationId,
componentReceipt: replacementReceipt(),
createdAt: now,
execution: execution.context,
fingerprintMaterial: material,
projectionVersion: 3,
}),
).resolves.toMatchObject({
attempt: {
candidateFingerprint: fingerprint,
candidatePublicationId: conflictingPublicationId,
checkpoint: "projection_built",
},
candidate: { id: conflictingPublicationId, status: "published" },
inheritedMemberCount: 0,
replacedMemberCount: 0,
});
expect(compose).not.toHaveBeenCalled();
await expect(
coordinator.evaluateAndPublishCandidate({
evaluator: {
evaluate: async () => {
throw new Error("an identical published snapshot must not be evaluated again");
},
},
execution: execution.context,
updatedAt: now,
}),
).resolves.toMatchObject({
attempt: { checkpoint: "smoke_eval_passed" },
evaluation: "previously-passed",
publication: { headRevision: 1, published: { id: conflictingPublicationId } },
});
await expect(publications.getPublished({ knowledgeSpaceId, tenantId })).resolves.toMatchObject({
fingerprint,
headRevision: 1,
id: conflictingPublicationId,
});
});
it("requires server-side component validation before candidate creation or member mutation", async () => {
const publications = createInMemoryProjectionSetPublicationRepository({ maxPublications: 10 });
const members = createInMemoryProjectionSetPublicationMemberRepository({

84
knowledge-fs/packages/api/src/document-compilation-publication-coordinator.ts
View File

@ -210,7 +210,10 @@ export function createDocumentCompilationPublicationCoordinator({
const initialAttempt = validateAttempt(input.execution.attempt);
const deletionToken = await captureCompilationDeletionFence(deletionFence, initialAttempt);
const assertWritable = () => assertCompilationDeletionFence(deletionFence, deletionToken);
const candidateId = normalizeUuid(input.candidateId);
const proposedCandidateId = normalizeUuid(input.candidateId);
const candidateId = initialAttempt.candidatePublicationId
? normalizeUuid(initialAttempt.candidatePublicationId)
: proposedCandidateId;
const createdAt = DateTimeSchema.parse(input.createdAt);
const projectionVersion = positiveInteger(input.projectionVersion, "projectionVersion");
const fingerprintMaterial = ProjectionSetFingerprintMaterialSchema.parse(
@ -249,20 +252,48 @@ export function createDocumentCompilationPublicationCoordinator({
projectionVersion,
tenantId: initialAttempt.tenantId,
});
assertCandidateIdentity(candidate, {
const reusesCurrentPublication = await isCurrentPublishedSnapshot(publications, candidate, {
attempt: initialAttempt,
candidateId,
fingerprint,
projectionVersion,
});
if (!reusesCurrentPublication) {
assertCandidateIdentity(candidate, {
attempt: initialAttempt,
candidateId: proposedCandidateId,
fingerprint,
projectionVersion,
});
}
let attempt = validateAttempt(input.execution.attempt);
assertSameAttemptScope(attempt, initialAttempt);
if (!attempt.candidatePublicationId) {
await assertWritable();
attempt = await bindCandidate(input.execution, attempt, candidateId, fingerprint);
attempt = await bindCandidate(input.execution, attempt, candidate.id, fingerprint);
} else {
assertAttemptCandidateBinding(attempt, candidateId, fingerprint);
assertAttemptCandidateBinding(attempt, candidate.id, fingerprint);
}
if (reusesCurrentPublication) {
assertExecutionFence(input.execution);
await assertWritable();
attempt = validateAttempt(await input.execution.heartbeat());
assertSameAttemptScope(attempt, initialAttempt);
assertAttemptCandidateBinding(attempt, candidate.id, fingerprint);
if (attempt.checkpoint === "nodes_generated") {
attempt = await input.execution.advance({
candidateFingerprint: fingerprint,
candidatePublicationId: candidate.id,
checkpoint: "projection_built",
});
}
return {
attempt,
candidate,
inheritedMemberCount: 0,
replacedMemberCount: 0,
};
}
assertExecutionFence(input.execution);
@ -345,13 +376,20 @@ export function createDocumentCompilationPublicationCoordinator({
"Document compilation candidate publication was not found",
);
}
assertCandidateIdentity(candidate, {
allowedStatuses: ["candidate", "published"],
const reusesCurrentPublication = await isCurrentPublishedSnapshot(publications, candidate, {
attempt: initialAttempt,
candidateId: candidatePublicationId,
fingerprint: candidateFingerprint,
projectionVersion: candidate.projectionVersion,
});
if (!reusesCurrentPublication) {
assertCandidateIdentity(candidate, {
allowedStatuses: ["candidate", "published"],
attempt: initialAttempt,
candidateId: candidatePublicationId,
fingerprint: candidateFingerprint,
projectionVersion: candidate.projectionVersion,
});
}
try {
if (candidate.status === "published") {
@ -360,7 +398,8 @@ export function createDocumentCompilationPublicationCoordinator({
!published ||
published.id !== candidatePublicationId ||
published.fingerprint !== candidateFingerprint ||
published.headRevision !== initialAttempt.baseHeadRevision + 1
published.headRevision !==
initialAttempt.baseHeadRevision + (reusesCurrentPublication ? 0 : 1)
) {
throw new DocumentCompilationCandidateIdentityConflictError(
"Published document compilation candidate is not the expected publication head",
@ -514,6 +553,33 @@ export function createDocumentCompilationPublicationCoordinator({
};
}
async function isCurrentPublishedSnapshot(
publications: Pick<ProjectionSetPublicationRepository, "getPublished">,
candidate: ProjectionSetPublication,
expected: {
readonly attempt: DocumentCompilationAttempt;
readonly fingerprint: string;
readonly projectionVersion: number;
},
): Promise<boolean> {
if (
candidate.status !== "published" ||
candidate.fingerprint !== expected.fingerprint ||
candidate.projectionVersion !== expected.projectionVersion
) {
return false;
}
const published = await publications.getPublished({
knowledgeSpaceId: expected.attempt.knowledgeSpaceId,
tenantId: expected.attempt.tenantId,
});
return (
published?.id === candidate.id &&
published.fingerprint === expected.fingerprint &&
published.headRevision === expected.attempt.baseHeadRevision
);
}
async function ensureExclusiveCandidate(
publications: Pick<ProjectionSetPublicationRepository, "createCandidate" | "getByFingerprint">,
input: Parameters<ProjectionSetPublicationRepository["createCandidate"]>[0],

89
knowledge-fs/packages/api/src/document-write-handlers.ts
View File

@ -92,7 +92,11 @@ import {
LegacySpacePublicationBootstrapSnapshotConflictError,
withKnowledgeSpaceDocumentMutationLease,
} from "./legacy-space-publication-bootstrap";
import type { DocumentRevision, LogicalDocumentRepository } from "./logical-document-repository";
import type {
DocumentRevision,
LogicalDocumentRepository,
LogicalDocumentWithActiveRevision,
} from "./logical-document-repository";
import {
LogicalDocumentConflictError,
LogicalDocumentNotFoundError,
@ -368,7 +372,10 @@ export function registerDocumentWriteHandlers({
const bulkJobId = generateBulkUploadId();
const items = [];
const bulkItems: BulkOperationItem[] = [];
const enqueueAsset = async (asset: DocumentAsset) => {
const enqueueAsset = async (
asset: DocumentAsset,
logicalDocument?: LogicalDocumentWithActiveRevision,
) => {
const compilationJob = await traceAsync(
traces,
traceId,
@ -387,7 +394,7 @@ export function registerDocumentWriteHandlers({
);
bulkItems.push({
compilationJobId: compilationJob.id,
documentId: asset.id,
documentId: logicalDocument?.id ?? asset.id,
requiredPermissionScope: requiredPermissionScopeForAsset(asset),
status: "queued",
});
@ -399,17 +406,34 @@ export function registerDocumentWriteHandlers({
stage: "queued" as const,
},
status: "queued" as const,
statusUrl: createDocumentAssetStatusUrl({ documentAssetId: asset.id, knowledgeSpaceId }),
statusUrl: logicalDocument
? createLogicalDocumentTaskStatusUrl({
documentId: logicalDocument.id,
knowledgeSpaceId,
taskId: compilationJob.id,
})
: createDocumentAssetStatusUrl({ documentAssetId: asset.id, knowledgeSpaceId }),
};
};
for (const documentId of requestedDocumentIds ?? []) {
const asset = await traceAsync(traces, traceId, "ingestion.bulk_reindex_asset_lookup", () =>
assets.get({
id: documentId,
knowledgeSpaceId,
}),
);
const logicalDocument = logicalDocuments
? await logicalDocuments.get({
documentId,
knowledgeSpaceId,
tenantId: subject.tenantId,
})
: null;
const assetId = logicalDocument?.active?.documentAssetId ?? documentId;
const asset =
logicalDocument && !logicalDocument.active
? null
: await traceAsync(traces, traceId, "ingestion.bulk_reindex_asset_lookup", () =>
assets.get({
id: assetId,
knowledgeSpaceId,
}),
);
if (!asset || !candidatePermissionAllowsAsset(asset, candidateGrants)) {
items.push({
@ -423,7 +447,7 @@ export function registerDocumentWriteHandlers({
continue;
}
items.push(await enqueueAsset(asset));
items.push(await enqueueAsset(asset, logicalDocument ?? undefined));
}
for (const asset of selectedAssets?.items ?? []) {
@ -592,7 +616,6 @@ export function registerDocumentWriteHandlers({
let asset: DocumentAsset | undefined;
let logicalRevision: DocumentRevision | undefined;
let compilationJobId: string | undefined;
let pathCreated = false;
try {
await assertWritable();
@ -640,17 +663,6 @@ export function registerDocumentWriteHandlers({
);
asset = createdAsset;
createdAssets.push(createdAsset);
await assertWritable();
await traceAsync(traces, traceId, "ingestion.bulk_document_path_upsert", () =>
knowledgePaths.upsertMany([
buildDocumentKnowledgePath({
asset: createdAsset,
id: generateKnowledgePathId(),
tenantId: subject.tenantId,
}),
]),
);
pathCreated = true;
logicalRevision = (
await traceAsync(traces, traceId, "ingestion.bulk_logical_revision_create", () =>
@ -787,15 +799,6 @@ export function registerDocumentWriteHandlers({
// Once a revision exists, retain its raw asset as a failed, inspectable revision; only
// this item is failed and previously accepted jobs keep their objects and queue state.
if (!logicalRevision) {
if (pathCreated) {
await knowledgePaths
.deleteByDocumentAsset({
documentAssetId: id,
knowledgeSpaceId,
maxPaths: 1,
})
.catch(() => undefined);
}
await scrubStaleDocumentUploadWithRetry(
// The durable stale-write scrubber is intentionally deletion-fence-only. This
// branch has already proved deletion did not win, so compensate the unpublished
@ -1261,16 +1264,18 @@ export function registerDocumentWriteHandlers({
asset = scopedAsset;
metadataAsset = asset;
}
await traceAsync(traces, traceId, "ingestion.document_path_upsert", () =>
knowledgePaths.upsertMany([
buildDocumentKnowledgePath({
asset,
id: generateKnowledgePathId(),
tenantId: subject.tenantId,
}),
]),
);
metadataPathCreated = true;
if (!compilationAuthorization) {
await traceAsync(traces, traceId, "ingestion.document_path_upsert", () =>
knowledgePaths.upsertMany([
buildDocumentKnowledgePath({
asset,
id: generateKnowledgePathId(),
tenantId: subject.tenantId,
}),
]),
);
metadataPathCreated = true;
}
await assertWritable();
await traceAsync(traces, traceId, "ingestion.staged_commit_metadata_prepared", () =>
stagedCommits.transition({

19
knowledge-fs/packages/api/src/durable-deletion-repository.test.ts
View File

@ -1430,7 +1430,10 @@ describe.each(["postgres", "tidb"] as const)(
it("retries and then completes a fenced external item with redaction", async () => {
const running = runningJobRow({ checkpoint: "deleting_objects" });
const itemId = "del-item-1";
const pending = itemRow({ id: itemId });
const pending = itemRow({
id: itemId,
...(dialect === "postgres" ? { ordinal: "1" } : {}),
});
const retryAt = "2026-07-14T12:01:00.000Z";
const retrying = itemRow({
attempts: 1,
@ -1467,7 +1470,7 @@ describe.each(["postgres", "tidb"] as const)(
now: createdAt,
}),
).resolves.toMatchObject([
{ attempts: 0, id: itemId, objectKey: pending.object_key, status: "pending" },
{ attempts: 0, id: itemId, objectKey: pending.object_key, ordinal: 1, status: "pending" },
]);
claimScript.expectDone();
@ -1515,11 +1518,13 @@ describe.each(["postgres", "tidb"] as const)(
redactedAt: completedAt,
status: "completed",
});
expect(
completeScript.calls.find(
(call) => call.operation === "update" && call.tableName === "deletion_job_items",
)?.sql,
).toContain("redacted_at");
const completeItemSql = completeScript.calls.find(
(call) => call.operation === "update" && call.tableName === "deletion_job_items",
)?.sql;
expect(completeItemSql).toContain("redacted_at");
if (dialect === "postgres") {
expect(completeItemSql).toContain("THEN CAST($3 AS TIMESTAMPTZ)");
}
completeScript.expectDone();
});

7
knowledge-fs/packages/api/src/durable-deletion-repository.ts
View File

@ -14,6 +14,7 @@ import {
resolveCapabilityJobPublicationGrant,
} from "./capability-job-fence";
import {
nonnegativeSafeIntegerColumn,
numberColumn,
optionalNumberColumn,
optionalStringColumn,
@ -1299,11 +1300,13 @@ async function completeDeletionItem(
) {
return null;
}
const completionTimestamp =
database.dialect === "postgres" ? `CAST(${p(database, 3)} AS TIMESTAMPTZ)` : p(database, 3);
const updated = await transaction.execute({
maxRows: 0,
operation: "update",
params: [item.attempts + 1, item.rowVersion + 1, input.now, item.id, job.id, item.rowVersion],
sql: `UPDATE ${q(database, itemTable)} SET ${q(database, "status")} = 'completed', ${q(database, "attempts")} = ${p(database, 1)}, ${q(database, "next_attempt_at")} = NULL, ${q(database, "object_key")} = NULL, ${q(database, "credential_ref")} = NULL, ${q(database, "cache_key")} = NULL, ${q(database, "last_error_code")} = NULL, ${q(database, "last_error_message")} = NULL, ${q(database, "row_version")} = ${p(database, 2)}, ${q(database, "updated_at")} = ${p(database, 3)}, ${q(database, "completed_at")} = ${p(database, 3)}, ${q(database, "redacted_at")} = CASE WHEN ${q(database, "kind")} IN ('object', 'secret_ref', 'cache_key') THEN ${p(database, 3)} ELSE NULL END WHERE ${q(database, "id")} = ${p(database, 4)} AND ${q(database, "deletion_job_id")} = ${p(database, 5)} AND ${q(database, "row_version")} = ${p(database, 6)};`,
sql: `UPDATE ${q(database, itemTable)} SET ${q(database, "status")} = 'completed', ${q(database, "attempts")} = ${p(database, 1)}, ${q(database, "next_attempt_at")} = NULL, ${q(database, "object_key")} = NULL, ${q(database, "credential_ref")} = NULL, ${q(database, "cache_key")} = NULL, ${q(database, "last_error_code")} = NULL, ${q(database, "last_error_message")} = NULL, ${q(database, "row_version")} = ${p(database, 2)}, ${q(database, "updated_at")} = ${completionTimestamp}, ${q(database, "completed_at")} = ${completionTimestamp}, ${q(database, "redacted_at")} = CASE WHEN ${q(database, "kind")} IN ('object', 'secret_ref', 'cache_key') THEN ${completionTimestamp} ELSE NULL END WHERE ${q(database, "id")} = ${p(database, 4)} AND ${q(database, "deletion_job_id")} = ${p(database, 5)} AND ${q(database, "row_version")} = ${p(database, 6)};`,
tableName: itemTable,
});
return updated.rowsAffected === 1
@ -3165,7 +3168,7 @@ function mapItem(row: DatabaseRow): DurableDeletionJobItem {
...(optionalStringColumn(row, "object_key")
? { objectKey: optionalStringColumn(row, "object_key") }
: {}),
ordinal: numberColumn(row, "ordinal"),
ordinal: nonnegativeSafeIntegerColumn(row, "ordinal"),
payloadDigest: stringColumn(row, "payload_digest"),
...(optionalStringColumn(row, "redacted_at")
? { redactedAt: optionalStringColumn(row, "redacted_at") }

83
knowledge-fs/packages/api/src/gateway-document-write.test.ts
View File

@ -3473,6 +3473,89 @@ describe("document write gateway integration", () => {
).toThrow("Bulk document reindex maxBulkReindexDocuments must be at least 1");
});
it("resolves a logical document id to its active asset for reindexing", async () => {
const knowledgeSpaceId = "018f0d60-7a49-7cc2-9c1b-5b36f18f2c42";
const logicalDocumentId = "018f0d60-7a49-7cc2-9c1b-5b36f18f3d01";
const assetId = "018f0d60-7a49-7cc2-9c1b-5b36f18f3a01";
const adapter = createNodePlatformAdapter({ env: {} });
const assets = createInMemoryDocumentAssetRepository({ maxAssets: 10 });
const asset = await assets.create({
filename: "Logical.md",
id: assetId,
knowledgeSpaceId,
mimeType: "text/markdown",
objectKey: "tenant-1/spaces/space/documents/logical.md",
sha256: "a".repeat(64),
sizeBytes: 1,
});
const logicalDocuments = createInMemoryLogicalDocumentRepository({
canReadDocument: ({ candidateGrants }) => candidateGrants.includes("document:read"),
canReadRevision: ({ candidateGrants }) => candidateGrants.includes("document:read"),
generateDocumentId: () => logicalDocumentId,
maxDocuments: 10,
maxRevisionsPerDocument: 2,
});
const candidate = await logicalDocuments.createCandidateRevision({
contentHash: "a".repeat(64),
documentAssetId: asset.id,
documentAssetVersion: asset.version,
knowledgeSpaceId,
mimeType: asset.mimeType,
now: "2026-07-27T00:00:00.000Z",
sizeBytes: asset.sizeBytes,
systemMetadata: {},
tenantId: "tenant-1",
title: asset.filename,
});
await logicalDocuments.activateRevision({
documentId: logicalDocumentId,
expectedActiveRevision: null,
expectedRowVersion: 0,
knowledgeSpaceId,
now: "2026-07-27T00:01:00.000Z",
revision: candidate.revision.revision,
tenantId: "tenant-1",
});
const compilationJobs = createDocumentCompilationJobStateMachine({
generateId: () => "logical-reindex-job-1",
jobs: adapter.jobs,
repository: createInMemoryDocumentCompilationJobRepository({ maxJobs: 10 }),
});
const app = createKnowledgeGateway({
adapter,
auth: createTestAuthVerifier(),
documentAssets: assets,
documentCompilationJobs: compilationJobs,
generateBulkUploadId: () => "logical-reindex-bulk-1",
knowledgeSpaces: createInMemoryKnowledgeSpaceRepository({
generateId: () => knowledgeSpaceId,
maxListLimit: 10,
maxSpaces: 10,
}),
logicalDocuments,
});
await app.request("/knowledge-spaces", {
body: JSON.stringify({ name: "Logical reindex", slug: "logical-reindex" }),
headers: { ...bearer(writeToken), "content-type": "application/json" },
method: "POST",
});
const response = await app.request(
`/knowledge-spaces/${knowledgeSpaceId}/documents/bulk/reindex`,
{
body: JSON.stringify({ documentIds: [logicalDocumentId] }),
headers: { ...bearer(writeToken), "content-type": "application/json" },
method: "POST",
},
);
expect(response.status).toBe(202);
await expect(response.json()).resolves.toMatchObject({
items: [{ asset: { id: assetId }, status: "queued" }],
total: 1,
});
});
it("reports tenant-scoped bulk job progress across queued and completed operations", async () => {
const adapter = createNodePlatformAdapter({ env: {} });
const assets = createInMemoryDocumentAssetRepository({

21
knowledge-fs/packages/api/src/gateway.test.ts
View File

@ -9874,27 +9874,6 @@ describe("createKnowledgeGateway", () => {
nodes,
}),
).toThrow("Incremental reindexer maxNodes must be at least 1");
await expect(
createIncrementalReindexer({
artifacts,
compute,
denseBuilder: {
build: async () => [],
},
maxNodes: 4,
nodes,
}).reindex({
knowledgeSpaceId,
parseArtifact: ParseArtifactSchema.parse({
...changedArtifact,
artifactHash: "c".repeat(64),
}),
projectionVersion: 2,
}),
).rejects.toThrow(
"Incremental reindexer denseModel is required when denseBuilder is configured",
);
await nodes.deleteByDocumentAsset({ documentAssetId, knowledgeSpaceId, maxNodes: 4 });
const denseBuilds: unknown[] = [];
await expect(

15
knowledge-fs/packages/api/src/index-reindexer.test.ts
View File

@ -623,7 +623,7 @@ describe("incremental reindexer", () => {
).rejects.toThrow("inconsistent text embedding space");
});
it("validates bounded configuration, dense model requirements, and max node output", async () => {
it("validates bounded configuration, optional dense indexing, and max node output", async () => {
const artifacts = createInMemoryParseArtifactRepository({ maxArtifacts: 4 });
const nodes = createInMemoryKnowledgeNodeRepository({
maxBatchSize: 4,
@ -641,11 +641,17 @@ describe("incremental reindexer", () => {
}),
).toThrow("Incremental reindexer maxNodes must be at least 1");
let denseBuilds = 0;
await expect(
createIncrementalReindexer({
artifacts,
compute,
denseBuilder: { build: async () => [] },
denseBuilder: {
build: async () => {
denseBuilds += 1;
return [];
},
},
maxNodes: 4,
nodes,
}).reindex({
@ -653,9 +659,8 @@ describe("incremental reindexer", () => {
parseArtifact: parseArtifact({ artifactHash: "c".repeat(64) }),
projectionVersion: 1,
}),
).rejects.toThrow(
"Incremental reindexer denseModel is required when denseBuilder is configured",
);
).resolves.toMatchObject({ status: "rebuilt" });
expect(denseBuilds).toBe(0);
await expect(
createIncrementalReindexer({

11
knowledge-fs/packages/api/src/index-reindexer.ts
View File

@ -161,7 +161,7 @@ export function createIncrementalReindexer({
}
: {}),
reindex: async (input) => {
validateIncrementalReindexInput(input, { denseBuilder, visualBuilder });
validateIncrementalReindexInput(input, { visualBuilder });
const parseArtifact = cloneParseArtifact(ParseArtifactSchema.parse(input.parseArtifact));
const publicationGenerationId =
input.publicationGenerationId === undefined
@ -375,10 +375,7 @@ function validateReindexProjectionDimensions(
function validateIncrementalReindexInput(
input: IncrementalReindexInput,
{
denseBuilder,
visualBuilder,
}: Pick<IncrementalReindexerOptions, "denseBuilder" | "visualBuilder">,
{ visualBuilder }: Pick<IncrementalReindexerOptions, "visualBuilder">,
): void {
if (!input.knowledgeSpaceId.trim()) {
throw new Error("Incremental reindexer knowledgeSpaceId is required");
@ -396,10 +393,6 @@ function validateIncrementalReindexInput(
PublicationGenerationIdSchema.parse(input.publicationGenerationId);
}
if (denseBuilder && !input.denseModel?.trim()) {
throw new Error("Incremental reindexer denseModel is required when denseBuilder is configured");
}
if (visualBuilder && !input.visualModel?.trim()) {
throw new Error(
"Incremental reindexer visualModel is required when visualBuilder is configured",

3
knowledge-fs/packages/api/src/logical-document-repository.ts
View File

@ -7,6 +7,7 @@ import {
import { CapabilityPublicationFencedError } from "./capability-grant-provenance";
import { resolveCapabilityJobPublicationGrant } from "./capability-job-fence";
import {
nonnegativeSafeIntegerColumn,
numberColumn,
optionalNumberColumn,
optionalStringColumn,
@ -2218,7 +2219,7 @@ function mapRevision(row: DatabaseRow): DocumentRevision {
knowledgeSpaceId: stringColumn(row, "knowledge_space_id"),
mimeType: stringColumn(row, "mime_type"),
revision: numberColumn(row, "revision"),
sizeBytes: numberColumn(row, "size_bytes"),
sizeBytes: nonnegativeSafeIntegerColumn(row, "size_bytes"),
state,
systemMetadata: jsonObjectColumn(row, "system_metadata"),
tenantId: stringColumn(row, "tenant_id"),

6
knowledge-fs/packages/api/src/page-index-build-repository.test.ts
View File

@ -161,13 +161,13 @@ describe("flattened PageIndex build repository", () => {
rows: [
{
checksum: manifestParams[10],
actual_node_count: nodeRows.length,
actual_term_count: termRows.length,
actual_node_count: String(nodeRows.length),
actual_term_count: String(termRows.length),
document_asset_id: manifestParams[3],
document_outline_id: manifestParams[4],
document_version: manifestParams[5],
id: manifestParams[0],
invalid_term_count: 0,
invalid_term_count: "0",
knowledge_space_id: manifestParams[1],
node_count: manifestParams[8],
publication_generation_id: manifestParams[2],

13
knowledge-fs/packages/api/src/page-index-build-repository.ts
View File

@ -15,7 +15,12 @@ import {
} from "@knowledge/core";
import { deterministicChildId } from "./api-shared-utils";
import { numberColumn, optionalStringColumn, stringColumn } from "./database-row-utils";
import {
nonnegativeSafeIntegerColumn,
numberColumn,
optionalStringColumn,
stringColumn,
} from "./database-row-utils";
import { databasePlaceholder, quoteDatabaseIdentifier } from "./database-sql-utils";
import { jsonArrayColumn, jsonObjectColumn } from "./json-utils";
import {
@ -745,9 +750,9 @@ async function readLockedDatabasePageIndex(
) as typeof PageIndexTokenizerVersion,
};
return {
actualNodeCount: numberColumn(manifestRow, "actual_node_count"),
actualTermCount: numberColumn(manifestRow, "actual_term_count"),
invalidTermCount: numberColumn(manifestRow, "invalid_term_count"),
actualNodeCount: nonnegativeSafeIntegerColumn(manifestRow, "actual_node_count"),
actualTermCount: nonnegativeSafeIntegerColumn(manifestRow, "actual_term_count"),
invalidTermCount: nonnegativeSafeIntegerColumn(manifestRow, "invalid_term_count"),
manifest,
};
}

13
knowledge-fs/packages/api/src/source-product-routes.ts
View File

@ -71,6 +71,7 @@ const BulkWorkflowItem = z.object({
export const listSourceProvidersRoute = createRoute({
method: "get",
operationId: "listSourceProviders",
path: "/source-providers",
responses: {
200: {
@ -83,6 +84,7 @@ export const listSourceProvidersRoute = createRoute({
export const createSourceConnectionRoute = createRoute({
method: "post",
operationId: "createSourceConnection",
path: "/knowledge-spaces/{id}/source-connections",
request: {
params: SpaceParams,
@ -189,6 +191,7 @@ export const completeSourceOAuthRoute = createRoute({
export const listSourceConnectionsRoute = createRoute({
method: "get",
operationId: "listSourceConnections",
path: "/knowledge-spaces/{id}/source-connections",
request: {
params: SpaceParams,
@ -231,6 +234,7 @@ export const getSourceConnectionRoute = createRoute({
export const refreshSourceConnectionRoute = createRoute({
method: "post",
operationId: "refreshSourceConnection",
path: "/knowledge-spaces/{id}/source-connections/{connectionId}/refresh",
request: {
params: ConnectionParams,
@ -277,6 +281,7 @@ export const revokeSourceConnectionRoute = createRoute({
export const createSourceSyncWorkflowRoute = createRoute({
method: "post",
operationId: "createSourceSyncWorkflow",
path: "/knowledge-spaces/{id}/sources/{sourceId}/sync",
request: { params: SourceParams, headers: IdempotencyHeader },
responses: {
@ -294,6 +299,7 @@ export const createSourceSyncWorkflowRoute = createRoute({
export const createSourceCrawlPreviewWorkflowRoute = createRoute({
method: "post",
operationId: "createSourceCrawlPreviewWorkflow",
path: "/knowledge-spaces/{id}/sources/{sourceId}/crawl-preview",
request: { params: SourceParams, headers: IdempotencyHeader },
responses: {
@ -374,6 +380,7 @@ export const createSourceImportWorkflowRoute = createRoute({
export const getSourceSyncPolicyRoute = createRoute({
method: "get",
operationId: "getSourceSyncPolicy",
path: "/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
request: { params: SourceParams },
responses: {
@ -389,6 +396,7 @@ export const getSourceSyncPolicyRoute = createRoute({
export const putSourceSyncPolicyRoute = createRoute({
method: "put",
operationId: "putSourceSyncPolicy",
path: "/knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
request: {
params: SourceParams,
@ -487,6 +495,7 @@ export const listSourceWorkflowsRoute = createRoute({
export const getSourceWorkflowRoute = createRoute({
method: "get",
operationId: "getSourceWorkflow",
path: "/knowledge-spaces/{id}/source-workflows/{runId}",
request: { params: WorkflowParams },
responses: {
@ -532,6 +541,7 @@ export const listSourceBulkWorkflowItemsRoute = createRoute({
export const cancelSourceWorkflowRoute = createRoute({
method: "post",
operationId: "cancelSourceWorkflow",
path: "/knowledge-spaces/{id}/source-workflows/{runId}/cancel",
request: {
params: WorkflowParams,
@ -558,6 +568,7 @@ export const cancelSourceWorkflowRoute = createRoute({
export const retrySourceWorkflowRoute = createRoute({
method: "post",
operationId: "retrySourceWorkflow",
path: "/knowledge-spaces/{id}/source-workflows/{runId}/retry",
request: { params: WorkflowParams },
responses: {
@ -574,6 +585,7 @@ export const retrySourceWorkflowRoute = createRoute({
export const listCrawlPreviewPagesRoute = createRoute({
method: "get",
operationId: "listCrawlPreviewPages",
path: "/knowledge-spaces/{id}/source-workflows/{runId}/pages",
request: {
params: WorkflowParams,
@ -612,6 +624,7 @@ export const listCrawlPreviewPagesRoute = createRoute({
export const selectCrawlPreviewPagesRoute = createRoute({
method: "post",
operationId: "selectCrawlPreviewPages",
path: "/knowledge-spaces/{id}/source-workflows/{runId}/selection",
request: {
params: WorkflowParams,

148
knowledge-fs/packages/api/src/source-product-workflow-database-repository.test.ts
View File

@ -209,6 +209,127 @@ describe.each(["postgres", "tidb"] as const)(
);
});
it("attributes capability workflow completion to the admitted grant subject", async () => {
const calls: DatabaseExecuteInput[] = [];
const database = orderedMutationDatabase(
dialect,
calls,
runningCapabilitySourceRunRow(),
false,
);
const repository = createDatabaseSourceProductWorkflowRepository({ database });
await expect(
repository.complete({
fence: { leaseToken, rowVersion: 7, runId, workerId: "source-worker" },
now,
}),
).resolves.toMatchObject({ state: "completed" });
const activity = calls.find(
(call) =>
call.tableName === "knowledge_space_activity_events" && call.operation === "insert",
);
expect(activity?.params.slice(3, 5)).toEqual(["member", "editor-a"]);
});
it("attributes capability workflow failure to the admitted grant subject", async () => {
const calls: DatabaseExecuteInput[] = [];
const database = orderedMutationDatabase(
dialect,
calls,
runningCapabilitySourceRunRow(),
false,
);
const repository = createDatabaseSourceProductWorkflowRepository({ database });
await expect(
repository.fail({
errorCode: "SOURCE_IMPORT_FAILED",
errorMessage: "provider unavailable",
fence: { leaseToken, rowVersion: 7, runId, workerId: "source-worker" },
now,
}),
).resolves.toMatchObject({ state: "failed" });
const activity = calls.find(
(call) =>
call.tableName === "knowledge_space_activity_events" && call.operation === "insert",
);
expect(activity?.params.slice(3, 5)).toEqual(["member", "editor-a"]);
});
it("keeps capability workflow update columns aligned with their parameters", async () => {
const calls: DatabaseExecuteInput[] = [];
const database = orderedMutationDatabase(dialect, calls, capabilitySourceRunRow(), false);
const repository = createDatabaseSourceProductWorkflowRepository({ database });
await repository.cancel({
capabilityGrantId,
now,
reason: "stop",
runId,
});
const update = calls.find(
(call) => call.tableName === "source_workflow_runs" && call.operation === "update",
);
expect(update?.sql).toContain("capability_grant_id");
expect(update?.params[11]).toBe(capabilityGrantId);
const placeholders = update?.sql.match(dialect === "postgres" ? /\$\d+/gu : /\?/gu) ?? [];
expect(placeholders).toHaveLength(update?.params.length ?? 0);
});
it("terminalizes a fenced worker failure while durable deletion is active", async () => {
const calls: DatabaseExecuteInput[] = [];
const database = orderedMutationDatabase(dialect, calls, runRow(), false, true);
const repository = createDatabaseSourceProductWorkflowRepository({ database });
await expect(
repository.fail({
errorCode: "SOURCE_IMPORT_PARTIAL_FAILURE",
errorMessage: "Import compensation requested deletion",
fence: { leaseToken, rowVersion: 7, runId, workerId: "source-worker" },
now,
}),
).resolves.toMatchObject({
activeSlot: undefined,
state: "failed",
});
expect(
calls.some(
(call) => call.tableName === "source_workflow_runs" && call.operation === "update",
),
).toBe(true);
});
it("cancels an authorized source workflow while durable deletion is active", async () => {
const calls: DatabaseExecuteInput[] = [];
const database = orderedMutationDatabase(
dialect,
calls,
sourceRunRow("running"),
false,
true,
);
const repository = createDatabaseSourceProductWorkflowRepository({ database });
await expect(
repository.cancel({
accessChannel: "interactive",
now,
permissionSnapshotId,
permissionSnapshotRevision: 1,
reason: "stop",
requestedBySubjectId: "editor-a",
runId,
}),
).resolves.toMatchObject({
activeSlot: undefined,
state: "canceled",
});
});
it("revalidates capability source workflows at restart and terminals revoked work", async () => {
const build = (active: boolean) => {
const calls: DatabaseExecuteInput[] = [];
@ -1863,6 +1984,7 @@ function claimDatabase(
if (input.tableName === "knowledge_space_permission_snapshots") {
return { rows: [permissionRow()], rowsAffected: 1 };
}
if (input.tableName === "capability_grants") return activeCapabilityGrant();
if (isAccessLock(input.tableName)) return oneRow(input.tableName);
if (input.tableName === "sources") return { rows: [sourceRow([])], rowsAffected: 1 };
if (input.tableName === "source_workflow_outbox" && input.operation === "select") {
@ -2138,6 +2260,18 @@ function capabilitySourceRunRow(): DatabaseRow {
};
}
function runningCapabilitySourceRunRow(): DatabaseRow {
return {
...sourceRunRow("running"),
access_channel: null,
capability_grant_id: capabilityGrantId,
permission_snapshot_id: null,
permission_snapshot_revision: null,
requested_by_subject_id: null,
required_permission_scope: null,
};
}
function newBulkRun(): NewSourceWorkflowRun {
return {
accessChannel: "interactive",
@ -2262,14 +2396,19 @@ function orderedMutationDatabase(
calls: DatabaseExecuteInput[],
row: DatabaseRow,
idempotencyMiss: boolean,
activeDeletion = false,
): DatabaseAdapter {
let storedActivity: DatabaseRow | undefined;
return testDatabase(dialect, async (input) => {
calls.push(input);
if (input.tableName === "knowledge_spaces") return activeSpace();
if (input.tableName === "deletion_jobs") return empty();
if (input.tableName === "deletion_jobs") {
return activeDeletion ? oneRow("deletion_jobs") : empty();
}
if (input.tableName === "knowledge_space_permission_snapshots") {
return { rows: [permissionRow()], rowsAffected: 1 };
}
if (input.tableName === "capability_grants") return activeCapabilityGrant();
if (isAccessLock(input.tableName)) return oneRow(input.tableName);
if (input.tableName === "sources") {
return { rows: [sourceRow([])], rowsAffected: 1 };
@ -2286,6 +2425,13 @@ function orderedMutationDatabase(
if (input.tableName === "source_crawl_preview_pages" && input.operation === "select") {
return { rows: [{ page_id: "page-a" }], rowsAffected: 1 };
}
if (input.tableName === "knowledge_space_activity_events") {
if (input.operation === "insert") {
storedActivity = activityRow(input.params);
return { rows: [], rowsAffected: 1 };
}
return storedActivity ? { rows: [storedActivity], rowsAffected: 1 } : empty();
}
return { rows: [], rowsAffected: 1 };
});
}

185
knowledge-fs/packages/api/src/source-product-workflow-database-repository.ts
View File

@ -609,7 +609,7 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
}),
complete: ({ fence, now, state = "completed" }) =>
database.transaction(async (tx) => {
const { run: current } = await requireFenced(database, tx, fence, now);
const { permission, run: current } = await requireFenced(database, tx, fence, now);
if (state === "preview_ready" && current.kind !== "crawl-preview") invalidState();
const terminal = state === "completed" || state === "zero_results";
const next = await writeFenced(database, tx, current, {
@ -631,7 +631,15 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
});
await finishOutbox(database, tx, current.id, "completed", now);
if (terminal && current.kind === "sync" && current.sourceId) {
await appendSourceWorkflowActivity(database, tx, next, "source.synced", "success", now);
await appendSourceWorkflowActivity(
database,
tx,
next,
"source.synced",
"success",
now,
permission?.actorSubjectId,
);
}
return next;
}),
@ -662,7 +670,16 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
}),
fail: ({ errorCode, errorMessage, fence, now }) =>
database.transaction(async (tx) => {
const { run: current } = await requireFenced(database, tx, fence, now, [], true);
const { permission, run: current } = await requireFenced(
database,
tx,
fence,
now,
[],
true,
true,
true,
);
const next = await writeTerminal(database, tx, current, {
errorCode,
errorMessage,
@ -671,7 +688,15 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
});
await finishOutbox(database, tx, current.id, "completed", now);
if (current.sourceId) {
await appendSourceWorkflowActivity(database, tx, next, "source.failed", "failure", now);
await appendSourceWorkflowActivity(
database,
tx,
next,
"source.failed",
"failure",
now,
permission?.actorSubjectId,
);
}
return next;
}),
@ -686,13 +711,22 @@ export function createDatabaseSourceProductWorkflowRepository(input: {
runId,
}) =>
database.transaction(async (tx) => {
const admitted = await getRunForMutationAdmission(database, tx, runId, now, {
accessChannel,
capabilityGrantId,
permissionSnapshotId,
permissionSnapshotRevision,
requestedBySubjectId,
});
const admitted = await getRunForMutationAdmission(
database,
tx,
runId,
now,
{
accessChannel,
capabilityGrantId,
permissionSnapshotId,
permissionSnapshotRevision,
requestedBySubjectId,
},
[],
false,
true,
);
if (!admitted) return null;
const current = admitted.run;
if (["completed", "zero_results", "canceled"].includes(current.state)) return current;
@ -1444,6 +1478,7 @@ async function updateRun(
"progress_completed",
"progress_skipped",
"progress_failed",
"capability_grant_id",
"permission_snapshot_id",
"permission_snapshot_revision",
"requested_by_subject_id",
@ -1465,8 +1500,8 @@ async function updateRun(
"required_permission_scope",
] as const;
const allParams = runParams(next);
// Immutable id/tenant/space occupy 0..2 and created_at is immutable at index 29.
const sourceParams = [...allParams.slice(3, 29), ...allParams.slice(30)];
// Immutable id/tenant/space occupy 0..2 and created_at is immutable at index 30.
const sourceParams = [...allParams.slice(3, 30), ...allParams.slice(31)];
const updateParams = [...sourceParams, next.id, next.rowVersion - 1, ...extraParams];
const idPosition = mutableColumns.length + 1;
const versionPosition = idPosition + 1;
@ -1489,6 +1524,8 @@ async function requireFenced(
now: string,
additionalSourceIds: readonly string[] = [],
allowInvalidPermission = false,
allowDeletionFencedTerminalization = false,
bypassAuthorizationThroughDeletionFence = false,
) {
const admitted = await getRunForMutationAdmission(
database,
@ -1498,6 +1535,8 @@ async function requireFenced(
undefined,
additionalSourceIds,
allowInvalidPermission,
allowDeletionFencedTerminalization,
bypassAuthorizationThroughDeletionFence,
);
const run = admitted?.run;
if (
@ -1526,6 +1565,8 @@ async function getRunForMutationAdmission(
>,
additionalSourceIds: readonly string[] = [],
allowInvalidPermission = false,
allowDeletionFencedTerminalization = false,
bypassAuthorizationThroughDeletionFence = false,
): Promise<{
readonly permission: SourceWorkflowAuthorization | undefined;
readonly run: SourceWorkflowRun;
@ -1533,7 +1574,12 @@ async function getRunForMutationAdmission(
} | null> {
const candidate = await getRun(database, tx, runId, false);
if (!candidate) return null;
if (!(await lockKnowledgeSpaceForDeletionAdmission(database, tx, candidate))) {
const writable = await lockKnowledgeSpaceForDeletionAdmission(database, tx, candidate);
const terminalizingThroughDeletionFence = !writable && allowDeletionFencedTerminalization;
if (
!writable &&
(!terminalizingThroughDeletionFence || !(await knowledgeSpaceExists(database, tx, candidate)))
) {
throw new SourceWorkflowError(
"SOURCE_WORKFLOW_SPACE_NOT_WRITABLE",
"Knowledge space is missing or deletion-fenced",
@ -1543,43 +1589,53 @@ async function getRunForMutationAdmission(
? { ...candidate, ...authorizationOverride }
: candidate;
let permission: SourceWorkflowAuthorization | undefined;
try {
permission = await assertSourceWorkflowPermissionFence(database, tx, authorizationBinding, now);
} catch (error) {
if (
!allowInvalidPermission ||
!(error instanceof SourceWorkflowError) ||
error.code !== "SOURCE_WORKFLOW_PERMISSION_INVALID"
) {
throw error;
}
}
let sourceScopes: ReadonlyMap<string, readonly string[]>;
try {
sourceScopes = await lockSourceWorkflowAdmissions(
database,
tx,
candidate.knowledgeSpaceId,
[candidate.sourceId, ...additionalSourceIds],
permission,
);
} catch (error) {
if (
!allowInvalidPermission ||
!(error instanceof SourceWorkflowError) ||
error.code !== "SOURCE_WORKFLOW_PERMISSION_INVALID"
) {
throw error;
}
if (terminalizingThroughDeletionFence && bypassAuthorizationThroughDeletionFence) {
permission = undefined;
sourceScopes = await lockSourceWorkflowAdmissions(
database,
tx,
candidate.knowledgeSpaceId,
[candidate.sourceId, ...additionalSourceIds],
undefined,
true,
);
sourceScopes = new Map();
} else {
try {
permission = await assertSourceWorkflowPermissionFence(
database,
tx,
authorizationBinding,
now,
);
} catch (error) {
if (
!allowInvalidPermission ||
!(error instanceof SourceWorkflowError) ||
error.code !== "SOURCE_WORKFLOW_PERMISSION_INVALID"
) {
throw error;
}
}
try {
sourceScopes = await lockSourceWorkflowAdmissions(
database,
tx,
candidate.knowledgeSpaceId,
[candidate.sourceId, ...additionalSourceIds],
permission,
);
} catch (error) {
if (
!allowInvalidPermission ||
!(error instanceof SourceWorkflowError) ||
error.code !== "SOURCE_WORKFLOW_PERMISSION_INVALID"
) {
throw error;
}
permission = undefined;
sourceScopes = await lockSourceWorkflowAdmissions(
database,
tx,
candidate.knowledgeSpaceId,
[candidate.sourceId, ...additionalSourceIds],
undefined,
true,
);
}
}
const current = await getRun(database, tx, runId, true);
if (!current) return null;
@ -1600,6 +1656,21 @@ async function getRunForMutationAdmission(
return { permission, run: current, sourceScopes };
}
async function knowledgeSpaceExists(
database: DatabaseAdapter,
tx: DatabaseExecutor,
input: Pick<SourceWorkflowRun, "knowledgeSpaceId" | "tenantId">,
): Promise<boolean> {
const result = await tx.execute({
maxRows: 1,
operation: "select",
params: [input.tenantId, input.knowledgeSpaceId],
sql: `SELECT ${q(database, "id")} FROM ${q(database, "knowledge_spaces")} WHERE ${q(database, "tenant_id")} = ${p(database, 1)} AND ${q(database, "id")} = ${p(database, 2)} LIMIT 1;`,
tableName: "knowledge_spaces",
});
return result.rows.length === 1;
}
async function lockSourceWorkflowAdmissions(
database: DatabaseAdapter,
tx: DatabaseExecutor,
@ -1701,7 +1772,10 @@ async function assertSourceWorkflowPermissionFence(
knowledgeSpaceId: binding.knowledgeSpaceId,
tenantId: binding.tenantId,
});
return { permissionScopes: grant.contentScopeIds };
return {
actorSubjectId: grant.subjectId,
permissionScopes: grant.contentScopeIds,
};
} catch {
throw new SourceWorkflowError(
"SOURCE_WORKFLOW_PERMISSION_INVALID",
@ -1745,10 +1819,15 @@ async function assertSourceWorkflowPermissionFence(
);
}
assertSourceWorkflowScopeAllowed(binding.requiredPermissionScope, permission.permissionScopes);
return permission;
return {
actorSubjectId: permission.subjectId,
permissionScopes: permission.permissionScopes,
};
}
interface SourceWorkflowAuthorization {
/** Subject resolved under the same durable permission or Capability fence as terminalization. */
readonly actorSubjectId: string;
readonly permissionScopes: readonly string[];
}
@ -1927,6 +2006,7 @@ async function appendSourceWorkflowActivity(
action: "source.failed" | "source.synced",
result: "failure" | "success",
now: string,
actorSubjectId?: string,
) {
if (!run.sourceId) return;
const source = await tx.execute({
@ -1945,12 +2025,13 @@ async function appendSourceWorkflowActivity(
const requiredPermissionScope = candidatePermissionScopeSnapshot(
jsonStringArrayColumn(row, "permission_scope"),
);
const memberActorId = actorSubjectId ?? run.requestedBySubjectId;
await appendKnowledgeSpaceActivityWithExecutor({
database,
executor: tx,
input: {
action,
actor: { id: run.requestedBySubjectId, type: "member" },
actor: memberActorId ? { id: memberActorId, type: "member" } : { type: "system" },
details: {
...(run.lastErrorCode ? { reasonCode: run.lastErrorCode } : {}),
count: run.progressCompleted,

25
knowledge-fs/packages/api/src/source-product-workflow-runtime.test.ts
View File

@ -174,7 +174,28 @@ describe("source-product workflow staged content store", () => {
describe("source-product workflow provider imports", () => {
it("stages a crawl preview, imports the frozen selection, and drains staged content", async () => {
const source = sourceRecord("crawl-preview-source", { type: "web" });
let source = sourceRecord("crawl-preview-source", {
metadata: { preview: true },
status: "disabled",
type: "web",
});
const sources = {
get: vi.fn(async () => source),
update: vi.fn(
async (input: {
readonly metadata?: Source["metadata"];
readonly status?: Source["status"];
}) => {
source = {
...source,
...(input.metadata ? { metadata: input.metadata } : {}),
...(input.status ? { status: input.status } : {}),
version: source.version + 1,
};
return source;
},
),
};
const pages = [
{
content: "First page body",
@ -219,6 +240,7 @@ describe("source-product workflow provider imports", () => {
maxCleanupBatchesPerRun: 2,
run,
source,
sources: sources as never,
websiteCrawl: { crawl: vi.fn(async () => ({ pages })) },
});
@ -255,6 +277,7 @@ describe("source-product workflow provider imports", () => {
});
expect(fixture.publish).toHaveBeenCalledTimes(2);
expect(deleteRun).toHaveBeenCalledTimes(2);
expect(source).toMatchObject({ metadata: { preview: false }, status: "active", version: 2 });
});
it("imports online-document records with and without optional identity metadata", async () => {

27
knowledge-fs/packages/api/src/source-product-workflow-runtime.ts
View File

@ -271,6 +271,7 @@ export function createSourceProductWorkflowRuntime(input: {
) {
await cleanupStagedContent(input, execution, maxCleanupBatchesPerRun);
}
await activateImportedPreviewSource(input, execution, source);
await execution.assertActive();
await input.repository.complete({
fence: fence(execution.run()),
@ -597,6 +598,32 @@ async function cleanupStagedContent(
);
}
async function activateImportedPreviewSource(
input: Parameters<typeof createSourceProductWorkflowRuntime>[0],
execution: RuntimeExecution,
source: Source | null,
): Promise<void> {
const run = execution.run();
if (
!source ||
run.kind !== "crawl-preview" ||
selectedPageIds(run).length === 0 ||
source.status !== "disabled" ||
source.metadata.preview !== true
) {
return;
}
await execution.assertActive();
const activated = await input.sources.update({
expectedVersion: source.version,
id: source.id,
knowledgeSpaceId: run.knowledgeSpaceId,
metadata: { ...source.metadata, preview: false },
status: "active",
});
if (!activated) throw runtimeError("SOURCE_NOT_FOUND", "Source no longer exists");
}
async function processCrawlPreview(
input: Parameters<typeof createSourceProductWorkflowRuntime>[0],
execution: RuntimeExecution,

24
knowledge-fs/packages/generation/src/dify-model-runtime-llm.test.ts
View File

@ -116,6 +116,30 @@ describe("Dify model runtime LLM provider", () => {
});
});
it("keeps content from Dify stream frames with null usage", async () => {
const provider = createDifyModelRuntimeLlmProvider({
...BASE,
client: fakeClient(() => [
{
delta: {
finish_reason: null,
message: { content: "Reply OK." },
usage: null,
},
model: BASE.model,
},
]),
});
const result = await provider.generate({
messages: [{ content: "Reply OK.", role: "user" }],
model: BASE.model,
tenantId: "tenant-abc",
});
expect(result.text).toBe("Reply OK.");
});
it("requires a per-call tenantId and validates constructor options", async () => {
const provider = createDifyModelRuntimeLlmProvider({
...BASE,

2
knowledge-fs/packages/generation/src/index.ts
View File

@ -745,7 +745,7 @@ const DifyModelRuntimeLlmChunkSchema = z.object({
total_tokens: z.number(),
})
.partial()
.optional(),
.nullish(),
})
.partial()
.optional(),

41
knowledge-fs/scripts/export-capability-v2-operations.test.mjs
View File

@ -20,7 +20,46 @@ test("Capability v2 operation export is deterministic and includes internal life
);
const document = JSON.parse(readFileSync(output, "utf8"));
assert.equal(document.schemaVersion, 1);
assert.equal(new Set(document.operations.map((operation) => operation.operationId)).size, 63);
assert.equal(new Set(document.operations.map((operation) => operation.operationId)).size, 78);
assert.deepEqual(
document.operations.find((operation) => operation.operationId === "createSourceSyncWorkflow"),
{
action: "source_workflows.sync.create",
allowedCallerKinds: ["interactive", "service", "agent", "workflow"],
method: "POST",
operationId: "createSourceSyncWorkflow",
parentResourceBinding: { pathParameter: "id" },
path: "/knowledge-spaces/{id}/sources/{sourceId}/sync",
resourceBinding: { pathParameter: "sourceId" },
resourceType: "source",
},
);
assert.deepEqual(
document.operations.find((operation) => operation.operationId === "listSourceProviders"),
{
action: "source_providers.list",
allowedCallerKinds: ["interactive", "service", "agent", "workflow"],
method: "GET",
operationId: "listSourceProviders",
parentResourceBinding: null,
path: "/source-providers",
resourceBinding: { namespace: true },
resourceType: "namespace",
},
);
assert.deepEqual(
document.operations.find((operation) => operation.operationId === "getSourceWorkflow"),
{
action: "source_workflows.read",
allowedCallerKinds: ["interactive", "service", "agent", "workflow"],
method: "GET",
operationId: "getSourceWorkflow",
parentResourceBinding: { pathParameter: "id" },
path: "/knowledge-spaces/{id}/source-workflows/{runId}",
resourceBinding: { pathParameter: "runId" },
resourceType: "job",
},
);
assert.deepEqual(
document.operations.find((operation) => operation.operationId === "cancelBackgroundTask"),
{

16
knowledge-fs/scripts/export-openapi.mjs
View File

@ -16,7 +16,7 @@ function argumentValue(name) {
process.env.NODE_ENV = "test";
const [
{ createNodePlatformAdapter },
{ createKnowledgeGateway },
{ createKnowledgeGateway, registerSourceProductHandlers },
{ createInMemoryCapabilityGrantProvenanceRepository },
] = await Promise.all([
import("../packages/adapters/src/node.ts"),
@ -53,6 +53,20 @@ const app = createKnowledgeGateway({
putSmallFile: unavailableInContractExport,
},
});
const unavailableService = new Proxy(
{},
{
get: () => unavailableInContractExport,
},
);
registerSourceProductHandlers({
app,
authorization: unavailableService,
connections: unavailableService,
providers: unavailableService,
repository: unavailableService,
workflows: unavailableService,
});
const response = await app.request("/openapi.json");
if (!response.ok) {
throw new Error(`OpenAPI export failed with HTTP ${response.status}`);

17
knowledge-fs/scripts/export-openapi.test.mjs
View File

@ -54,6 +54,23 @@ test("OpenAPI export is hermetic when invoked from a production environment", ()
document.paths["/upload-sessions/{id}/abort"].post.operationId,
"abortUploadSession",
);
assert.equal(
document.paths["/knowledge-spaces/{id}/sources/{sourceId}/sync"].post.operationId,
"createSourceSyncWorkflow",
);
assert.equal(document.paths["/source-providers"].get.operationId, "listSourceProviders");
assert.equal(
document.paths["/knowledge-spaces/{id}/source-connections"].post.operationId,
"createSourceConnection",
);
assert.equal(
document.paths["/knowledge-spaces/{id}/sources/{sourceId}/crawl-preview"].post.operationId,
"createSourceCrawlPreviewWorkflow",
);
assert.equal(
document.paths["/knowledge-spaces/{id}/source-workflows/{runId}/selection"].post.operationId,
"selectCrawlPreviewPages",
);
for (const legacyPath of [
"/knowledge-spaces/{id}/access-policy",
"/knowledge-spaces/{id}/members",

8
packages/contracts/console.ts
View File

@ -1,7 +1 @@
import { consoleRouterContract as generatedConsoleRouterContract } from './generated/api/console/router.gen'
import { contract as knowledgeFsContract } from './generated/knowledge-fs/orpc.gen'
export const consoleRouterContract = {
...generatedConsoleRouterContract,
knowledgeFs: knowledgeFsContract,
}
export { consoleRouterContract } from './generated/api/console/router.gen'

698
packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts
View File

File diff suppressed because it is too large Load Diff

680
packages/contracts/generated/api/console/knowledge-fs/types.gen.ts
View File

@ -70,14 +70,52 @@ export type KnowledgeFsAppBindingResponse = {
status: KnowledgeFsAppSpaceJoinStatus
}
export type KnowledgeFsBackgroundTaskListResponse = {
data: Array<KnowledgeFsBackgroundTaskResponse>
next_cursor?: string | null
}
export type KnowledgeFsBackgroundTaskResponse = {
can_cancel: boolean
can_retry: boolean
completed_at?: string | null
created_at: string
document_id?: string | null
document_revision?: number | null
error_code?: string | null
error_message?: string | null
id: string
knowledge_space_id: string
operation:
| 'document_delete'
| 'document_processing'
| 'document_reindex'
| 'document_upload'
| 'source_bulk'
| 'source_crawl_import'
| 'source_crawl_preview'
| 'source_online_document_import'
| 'source_online_drive_import'
| 'source_sync'
progress_completed: number
progress_failed: number
progress_percent: number
progress_total: number
source_id?: string | null
state: 'canceled' | 'completed' | 'failed' | 'queued' | 'running'
task_kind: 'document' | 'document_bulk' | 'source'
updated_at: string
}
export type KnowledgeFsBulkJobResponse = {
canceled_items: number
completed_items: number
created_at: string
failed_item_ids: Array<string>
failed_items: number
id: string
knowledge_space_id: string
status: 'completed' | 'failed' | 'running'
status: 'canceled' | 'completed' | 'failed' | 'running'
total_items: number
type: 'document_delete' | 'document_reindex' | 'document_upload'
updated_at: string
@ -268,6 +306,11 @@ export type KnowledgeFsDocumentCompilationJobResponse = {
version: number
}
export type KnowledgeFsLogicalDocumentListResponse = {
data: Array<KnowledgeFsLogicalDocumentResponse>
next_cursor?: string | null
}
export type KnowledgeFsMembersReplacePayload = {
members: Array<KnowledgeFsMemberBindingPayload>
}
@ -276,6 +319,48 @@ export type KnowledgeFsPermissionListResponse = {
data: Array<KnowledgeFsPermissionResponse>
}
export type KnowledgeFsOverviewHealthResponse = {
components: KnowledgeFsOverviewHealthComponentsResponse
generated_at: string
knowledge_space_id: string
state: 'degraded' | 'healthy' | 'unavailable' | 'unknown'
}
export type KnowledgeFsOverviewInventoryResponse = {
generated_at: string
graph_entities: KnowledgeFsOverviewInventoryDeltaResponse
graph_relations: KnowledgeFsOverviewInventoryDeltaResponse
index_coverage: KnowledgeFsOverviewIndexCoverageResponse
knowledge_space_id: string
source_categories: KnowledgeFsOverviewSourceCategoriesResponse
}
export type KnowledgeFsOverviewQueryOutcomesResponse = {
buckets: Array<KnowledgeFsOverviewQueryOutcomeBucketResponse>
current: KnowledgeFsOverviewQueryOutcomeCountsResponse
generated_at: string
knowledge_space_id: string
previous: KnowledgeFsOverviewQueryOutcomeCountsResponse
previous_since: string
since: string
window: '24h' | '30d' | '7d'
}
export type KnowledgeFsOverviewStatsResponse = {
answer_rate: KnowledgeFsOverviewRateComparisonResponse
documents: number
fresh_source_count: number
freshness_seconds?: number | null
generated_at: string
knowledge_space_id: string
latest_source_sync_at?: string | null
linked_apps: number
queries: KnowledgeFsOverviewCountComparisonResponse
source_count: number
stale_source_count: number
window: '24h' | '30d' | '7d'
}
export type KnowledgeFsQueryCreatePayload = {
activeDocumentIds?: Array<string>
activeEntityIds?: Array<string>
@ -391,6 +476,83 @@ export type KnowledgeFsSettingsPayload = {
retrieval?: KnowledgeFsProductRetrievalProfile | null
}
export type KnowledgeFsSourceConnectionListResponse = {
data: Array<KnowledgeFsSourceConnectionResponse>
next_cursor?: string | null
}
export type KnowledgeFsSourceConnectionCreatePayload = {
authKind: 'api-key' | 'endpoint'
configuration?: {
[key: string]: boolean | number | string
}
credentials: {
[key: string]: unknown
}
name: string
providerId: string
}
export type KnowledgeFsSourceConnectionResponse = {
auth_kind: 'api-key' | 'endpoint' | 'oauth2'
configuration: {
[key: string]: boolean | number | string
}
created_at: string
error_code?: string | null
expires_at?: string | null
id: string
knowledge_space_id: string
name: string
provider_id: string
scopes: Array<string>
status: 'active' | 'error' | 'expired' | 'provisioning' | 'revoked'
updated_at: string
version: number
}
export type KnowledgeFsSourceConnectionRefreshPayload = {
expectedVersion: number
}
export type KnowledgeFsSourceProviderListResponse = {
data: Array<KnowledgeFsSourceProviderResponse>
}
export type KnowledgeFsSourceWorkflowResponse = {
canceled_at?: string | null
checkpoint: string
completed_at?: string | null
created_at: string
cursor?: string | null
execution_attempts: number
id: string
kind: string
knowledge_space_id: string
last_error_code?: string | null
max_execution_attempts: number
progress_completed: number
progress_failed: number
progress_skipped: number
progress_total?: number | null
source_id?: string | null
state: string
updated_at: string
}
export type KnowledgeFsSourceWorkflowCancelPayload = {
reason?: string | null
}
export type KnowledgeFsCrawlPreviewPageListResponse = {
data: Array<KnowledgeFsCrawlPreviewPageResponse>
next_cursor?: string | null
}
export type KnowledgeFsCrawlPreviewSelectionPayload = {
pageIds: Array<string>
}
export type KnowledgeFsSourceListResponse = {
data: Array<KnowledgeFsSourceResponse>
next_cursor?: string | null
@ -442,17 +604,6 @@ export type KnowledgeFsSourceUpdatePayload = {
status?: 'active' | 'disabled' | 'error' | 'syncing' | null
}
export type KnowledgeFsSourceCrawlResponse = {
completed?: number | null
failed?: number | null
imported?: number | null
pages: Array<KnowledgeFsCrawledPageResponse>
replaced?: number | null
skipped?: number | null
status?: string | null
total?: number | null
}
export type KnowledgeFsSourceFilesResponse = {
buckets: Array<KnowledgeFsSourceFileBucketResponse>
}
@ -476,6 +627,28 @@ export type KnowledgeFsSourcePagesResponse = {
workspaces: Array<KnowledgeFsSourceWorkspacePagesResponse>
}
export type KnowledgeFsSourceSyncPolicyResponse = {
created_at: string
custom_interval_seconds?: number | null
enabled: boolean
expected_source_version: number
id: string
knowledge_space_id: string
mode: 'custom' | 'interval' | 'manual' | 'provider'
next_run_at?: string | null
revision: number
source_id: string
updated_at: string
}
export type KnowledgeFsSourceSyncPolicyPayload = {
customIntervalSeconds?: number | null
enabled: boolean
expectedRevision: number
expectedSourceVersion: number
mode: 'custom' | 'interval' | 'manual' | 'provider'
}
export type KnowledgeFsSourceCredentialTestResponse = {
code?: string | null
error?: string | null
@ -552,6 +725,7 @@ export type KnowledgeFsjwkResponse = {
export type KnowledgeFsSpaceListItemResponse = {
control_space_id: string
created_at: string
knowledge_space_id: string | null
owner_account_id: string
permission_keys: Array<KnowledgeFsProductPermission>
@ -559,6 +733,7 @@ export type KnowledgeFsSpaceListItemResponse = {
state: KnowledgeFsControlSpaceState
technical_status: 'available' | 'not_ready' | 'unavailable'
technical_summary?: KnowledgeFsTechnicalSummary | null
updated_at: string
visibility: KnowledgeFsControlSpaceVisibility
}
@ -727,6 +902,62 @@ export type KnowledgeFsPermissionResponse = {
status: string
}
export type KnowledgeFsOverviewHealthComponentsResponse = {
index: KnowledgeFsOverviewHealthComponentResponse
ingestion: KnowledgeFsOverviewHealthComponentResponse
profile_publication: KnowledgeFsOverviewHealthComponentResponse
query_availability: KnowledgeFsOverviewHealthComponentResponse
source_freshness: KnowledgeFsOverviewHealthComponentResponse
worker_readiness: KnowledgeFsOverviewHealthComponentResponse
}
export type KnowledgeFsOverviewInventoryDeltaResponse = {
added_last_7d: number
total: number
}
export type KnowledgeFsOverviewIndexCoverageResponse = {
indexed: number
percentage: number
total: number
}
export type KnowledgeFsOverviewSourceCategoriesResponse = {
crawl: number
online_documents: number
online_drives: number
uploads: number
}
export type KnowledgeFsOverviewQueryOutcomeBucketResponse = {
answered: number
end_at: string
low_confidence: number
no_evidence: number
query_count: number
start_at: string
}
export type KnowledgeFsOverviewQueryOutcomeCountsResponse = {
answer_rate: number
answered: number
low_confidence: number
no_evidence: number
query_count: number
}
export type KnowledgeFsOverviewRateComparisonResponse = {
change_percentage_points: number
previous_value: number
value: number
}
export type KnowledgeFsOverviewCountComparisonResponse = {
change_rate: number | null
previous_value: number
value: number
}
export type KnowledgeFsAdmittedQueryRequest = {
activeDocumentIds?: Array<string>
activeEntityIds?: Array<string>
@ -802,9 +1033,20 @@ export type KnowledgeFsProductRetrievalProfile = {
topK: number
}
export type KnowledgeFsCrawledPageResponse = {
content: string
export type KnowledgeFsSourceProviderResponse = {
auth_kinds: Array<'api-key' | 'endpoint' | 'oauth2'>
available: boolean
capabilities: Array<'online-document' | 'online-drive' | 'website-crawl'>
configuration: Array<KnowledgeFsSourceProviderFieldResponse>
display_name: string
id: string
unavailable_reason?: string | null
}
export type KnowledgeFsCrawlPreviewPageResponse = {
description?: string | null
etag?: string | null
page_id: string
source_url: string
title?: string | null
}
@ -931,6 +1173,11 @@ export type KnowledgeFsDurableDeletionProgressResponse = {
export type KnowledgeFsControlSpacePermissionRole = 'editor' | 'owner' | 'viewer'
export type KnowledgeFsOverviewHealthComponentResponse = {
codes: Array<string>
state: 'degraded' | 'healthy' | 'unavailable' | 'unknown'
}
export type KnowledgeFsProductRerankProfile = {
enabled: boolean
model?: KnowledgeFsProfileModelSelection | null
@ -942,6 +1189,15 @@ export type KnowledgeFsProductScoreThreshold = {
value?: number | null
}
export type KnowledgeFsSourceProviderFieldResponse = {
description?: string | null
format?: 'password' | 'uri' | null
name: string
required: boolean
secret: boolean
type: 'boolean' | 'integer' | 'string'
}
export type KnowledgeFsSourceFileResponse = {
id: string
name: string
@ -1122,6 +1378,62 @@ export type DeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppI
export type DeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdResponse =
DeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdResponses[keyof DeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksData = {
body?: never
path: {
control_space_id: string
}
query?: {
cursor?: string
limit?: number
}
url: '/knowledge-fs/spaces/{control_space_id}/background-tasks'
}
export type GetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksResponses = {
200: KnowledgeFsBackgroundTaskListResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksResponse =
GetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelData = {
body?: never
path: {
control_space_id: string
task_id: string
task_kind: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/cancel'
}
export type PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponses =
{
200: KnowledgeFsBackgroundTaskResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse =
PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryData = {
body?: never
path: {
control_space_id: string
task_id: string
task_kind: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/retry'
}
export type PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponses = {
200: KnowledgeFsBackgroundTaskResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponse =
PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdBulkJobsByJobIdData = {
body?: never
path: {
@ -1224,6 +1536,9 @@ export type PostKnowledgeFsSpacesByControlSpaceIdDocumentsResponse =
export type DeleteKnowledgeFsSpacesByControlSpaceIdDocumentsBulkData = {
body: KnowledgeFsBulkDocumentDeletePayload
headers: {
'Idempotency-Key': string
}
path: {
control_space_id: string
}
@ -1256,6 +1571,9 @@ export type PostKnowledgeFsSpacesByControlSpaceIdDocumentsReindexResponse =
export type DeleteKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdData = {
body: KnowledgeFsDocumentDeletePayload
headers: {
'Idempotency-Key': string
}
path: {
control_space_id: string
document_id: string
@ -1468,6 +1786,41 @@ export type PostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponses = {
export type PostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponse =
PostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsData = {
body?: never
path: {
control_space_id: string
}
query?: {
cursor?: string
}
url: '/knowledge-fs/spaces/{control_space_id}/logical-documents'
}
export type GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsResponses = {
200: KnowledgeFsLogicalDocumentListResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsResponse =
GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdData = {
body?: never
path: {
control_space_id: string
document_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/logical-documents/{document_id}'
}
export type GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponses = {
200: KnowledgeFsLogicalDocumentResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponse =
GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponses]
export type PutKnowledgeFsSpacesByControlSpaceIdMembersData = {
body: KnowledgeFsMembersReplacePayload
path: {
@ -1484,6 +1837,74 @@ export type PutKnowledgeFsSpacesByControlSpaceIdMembersResponses = {
export type PutKnowledgeFsSpacesByControlSpaceIdMembersResponse =
PutKnowledgeFsSpacesByControlSpaceIdMembersResponses[keyof PutKnowledgeFsSpacesByControlSpaceIdMembersResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewHealthData = {
body?: never
path: {
control_space_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/overview/health'
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewHealthResponses = {
200: KnowledgeFsOverviewHealthResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewHealthResponse =
GetKnowledgeFsSpacesByControlSpaceIdOverviewHealthResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdOverviewHealthResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryData = {
body?: never
path: {
control_space_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/overview/inventory'
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryResponses = {
200: KnowledgeFsOverviewInventoryResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryResponse =
GetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesData = {
body?: never
path: {
control_space_id: string
}
query?: {
window?: '24h' | '30d' | '7d'
}
url: '/knowledge-fs/spaces/{control_space_id}/overview/query-outcomes'
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesResponses = {
200: KnowledgeFsOverviewQueryOutcomesResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesResponse =
GetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewStatsData = {
body?: never
path: {
control_space_id: string
}
query?: {
window?: '24h' | '30d' | '7d'
}
url: '/knowledge-fs/spaces/{control_space_id}/overview/stats'
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewStatsResponses = {
200: KnowledgeFsOverviewStatsResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdOverviewStatsResponse =
GetKnowledgeFsSpacesByControlSpaceIdOverviewStatsResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdOverviewStatsResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdPermissionsData = {
body?: never
path: {
@ -1684,6 +2105,165 @@ export type PatchKnowledgeFsSpacesByControlSpaceIdSettingsResponses = {
export type PatchKnowledgeFsSpacesByControlSpaceIdSettingsResponse =
PatchKnowledgeFsSpacesByControlSpaceIdSettingsResponses[keyof PatchKnowledgeFsSpacesByControlSpaceIdSettingsResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsData = {
body?: never
path: {
control_space_id: string
}
query?: {
cursor?: string
limit?: number
}
url: '/knowledge-fs/spaces/{control_space_id}/source-connections'
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponses = {
200: KnowledgeFsSourceConnectionListResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse =
GetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsData = {
body: KnowledgeFsSourceConnectionCreatePayload
path: {
control_space_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-connections'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponses = {
201: KnowledgeFsSourceConnectionResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshData = {
body: KnowledgeFsSourceConnectionRefreshPayload
path: {
connection_id: string
control_space_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-connections/{connection_id}/refresh'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponses = {
200: KnowledgeFsSourceConnectionResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourceProvidersData = {
body?: never
path: {
control_space_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-providers'
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceProvidersResponses = {
200: KnowledgeFsSourceProviderListResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceProvidersResponse =
GetKnowledgeFsSpacesByControlSpaceIdSourceProvidersResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdSourceProvidersResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdData = {
body?: never
path: {
control_space_id: string
run_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}'
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdResponses = {
200: KnowledgeFsSourceWorkflowResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdResponse =
GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelData = {
body: KnowledgeFsSourceWorkflowCancelPayload
path: {
control_space_id: string
run_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}/cancel'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponses = {
200: KnowledgeFsSourceWorkflowResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesData = {
body?: never
path: {
control_space_id: string
run_id: string
}
query?: {
cursor?: string
limit?: number
}
url: '/knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}/pages'
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesResponses = {
200: KnowledgeFsCrawlPreviewPageListResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesResponse =
GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryData = {
body?: never
path: {
control_space_id: string
run_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}/retry'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponses = {
200: KnowledgeFsSourceWorkflowResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionData = {
body: KnowledgeFsCrawlPreviewSelectionPayload
headers: {
'Idempotency-Key': string
}
path: {
control_space_id: string
run_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}/selection'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponses = {
202: KnowledgeFsSourceWorkflowResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourcesData = {
body?: never
path: {
@ -1720,6 +2300,9 @@ export type PostKnowledgeFsSpacesByControlSpaceIdSourcesResponse =
export type DeleteKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdData = {
body: KnowledgeFsSourceDeletePayload
headers: {
'Idempotency-Key': string
}
path: {
control_space_id: string
source_id: string
@ -1771,22 +2354,25 @@ export type PatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponses = {
export type PatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponse =
PatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponses[keyof PatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlData = {
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewData = {
body?: never
headers: {
'Idempotency-Key': string
}
path: {
control_space_id: string
source_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/crawl'
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/crawl-preview'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlResponses = {
200: KnowledgeFsSourceCrawlResponse
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponses = {
202: KnowledgeFsSourceWorkflowResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdFilesData = {
body?: never
@ -1864,6 +2450,60 @@ export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesResponses
export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesResponse =
GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncData = {
body?: never
headers: {
'Idempotency-Key': string
}
path: {
control_space_id: string
source_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/sync'
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponses = {
202: KnowledgeFsSourceWorkflowResponse
}
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponse =
PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponses[keyof PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponses]
export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyData = {
body?: never
path: {
control_space_id: string
source_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/sync-policy'
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponses = {
200: KnowledgeFsSourceSyncPolicyResponse
}
export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponse =
GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponses[keyof GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponses]
export type PutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyData = {
body: KnowledgeFsSourceSyncPolicyPayload
path: {
control_space_id: string
source_id: string
}
query?: never
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/sync-policy'
}
export type PutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponses = {
200: KnowledgeFsSourceSyncPolicyResponse
}
export type PutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponse =
PutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponses[keyof PutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponses]
export type PostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdTestData = {
body?: never
path: {

667
packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts
View File

@ -2,17 +2,62 @@
import * as z from 'zod'
/**
* KnowledgeFSBackgroundTaskResponse
*/
export const zKnowledgeFsBackgroundTaskResponse = z.object({
can_cancel: z.boolean(),
can_retry: z.boolean(),
completed_at: z.iso.datetime().nullish(),
created_at: z.iso.datetime(),
document_id: z.string().nullish(),
document_revision: z.int().gte(1).nullish(),
error_code: z.string().nullish(),
error_message: z.string().nullish(),
id: z.string(),
knowledge_space_id: z.string(),
operation: z.enum([
'document_delete',
'document_processing',
'document_reindex',
'document_upload',
'source_bulk',
'source_crawl_import',
'source_crawl_preview',
'source_online_document_import',
'source_online_drive_import',
'source_sync',
]),
progress_completed: z.int().gte(0),
progress_failed: z.int().gte(0),
progress_percent: z.int().gte(0).lte(100),
progress_total: z.int().gte(0),
source_id: z.string().nullish(),
state: z.enum(['canceled', 'completed', 'failed', 'queued', 'running']),
task_kind: z.enum(['document', 'document_bulk', 'source']),
updated_at: z.iso.datetime(),
})
/**
* KnowledgeFSBackgroundTaskListResponse
*/
export const zKnowledgeFsBackgroundTaskListResponse = z.object({
data: z.array(zKnowledgeFsBackgroundTaskResponse),
next_cursor: z.string().nullish(),
})
/**
* KnowledgeFSBulkJobResponse
*/
export const zKnowledgeFsBulkJobResponse = z.object({
canceled_items: z.int().gte(0),
completed_items: z.int().gte(0),
created_at: z.iso.datetime(),
failed_item_ids: z.array(z.string()),
failed_items: z.int().gte(0),
id: z.string(),
knowledge_space_id: z.string(),
status: z.enum(['completed', 'failed', 'running']),
status: z.enum(['canceled', 'completed', 'failed', 'running']),
total_items: z.int().gte(0),
type: z.enum(['document_delete', 'document_reindex', 'document_upload']),
updated_at: z.iso.datetime(),
@ -216,6 +261,89 @@ export const zKnowledgeFsResearchTaskPlanPayload = z.object({
topK: z.int().gte(1).lte(50).nullish(),
})
/**
* KnowledgeFSSourceConnectionCreatePayload
*/
export const zKnowledgeFsSourceConnectionCreatePayload = z.object({
authKind: z.enum(['api-key', 'endpoint']),
configuration: z.record(z.string(), z.union([z.boolean(), z.int(), z.string()])).optional(),
credentials: z.record(z.string(), z.unknown()),
name: z.string().min(1).max(160),
providerId: z.string().min(1).max(128),
})
/**
* KnowledgeFSSourceConnectionResponse
*/
export const zKnowledgeFsSourceConnectionResponse = z.object({
auth_kind: z.enum(['api-key', 'endpoint', 'oauth2']),
configuration: z.record(z.string(), z.union([z.boolean(), z.int(), z.string()])),
created_at: z.iso.datetime(),
error_code: z.string().nullish(),
expires_at: z.iso.datetime().nullish(),
id: z.string(),
knowledge_space_id: z.string(),
name: z.string(),
provider_id: z.string(),
scopes: z.array(z.string()),
status: z.enum(['active', 'error', 'expired', 'provisioning', 'revoked']),
updated_at: z.iso.datetime(),
version: z.int().gte(1),
})
/**
* KnowledgeFSSourceConnectionListResponse
*/
export const zKnowledgeFsSourceConnectionListResponse = z.object({
data: z.array(zKnowledgeFsSourceConnectionResponse),
next_cursor: z.string().nullish(),
})
/**
* KnowledgeFSSourceConnectionRefreshPayload
*/
export const zKnowledgeFsSourceConnectionRefreshPayload = z.object({
expectedVersion: z.int().gte(1),
})
/**
* KnowledgeFSSourceWorkflowResponse
*/
export const zKnowledgeFsSourceWorkflowResponse = z.object({
canceled_at: z.iso.datetime().nullish(),
checkpoint: z.string(),
completed_at: z.iso.datetime().nullish(),
created_at: z.iso.datetime(),
cursor: z.string().nullish(),
execution_attempts: z.int().gte(0),
id: z.string(),
kind: z.string(),
knowledge_space_id: z.string(),
last_error_code: z.string().nullish(),
max_execution_attempts: z.int().gte(1),
progress_completed: z.int().gte(0),
progress_failed: z.int().gte(0),
progress_skipped: z.int().gte(0),
progress_total: z.int().gte(0).nullish(),
source_id: z.string().nullish(),
state: z.string(),
updated_at: z.iso.datetime(),
})
/**
* KnowledgeFSSourceWorkflowCancelPayload
*/
export const zKnowledgeFsSourceWorkflowCancelPayload = z.object({
reason: z.string().max(1000).nullish(),
})
/**
* KnowledgeFSCrawlPreviewSelectionPayload
*/
export const zKnowledgeFsCrawlPreviewSelectionPayload = z.object({
pageIds: z.array(z.string()).min(1).max(200),
})
/**
* KnowledgeFSSourceCreatePayload
*/
@ -274,6 +402,34 @@ export const zKnowledgeFsSourceUpdatePayload = z.object({
status: z.enum(['active', 'disabled', 'error', 'syncing']).nullish(),
})
/**
* KnowledgeFSSourceSyncPolicyResponse
*/
export const zKnowledgeFsSourceSyncPolicyResponse = z.object({
created_at: z.iso.datetime(),
custom_interval_seconds: z.int().nullish(),
enabled: z.boolean(),
expected_source_version: z.int().gte(1),
id: z.string(),
knowledge_space_id: z.string(),
mode: z.enum(['custom', 'interval', 'manual', 'provider']),
next_run_at: z.iso.datetime().nullish(),
revision: z.int().gte(1),
source_id: z.string(),
updated_at: z.iso.datetime(),
})
/**
* KnowledgeFSSourceSyncPolicyPayload
*/
export const zKnowledgeFsSourceSyncPolicyPayload = z.object({
customIntervalSeconds: z.int().gte(3600).lte(2592000).nullish(),
enabled: z.boolean(),
expectedRevision: z.int().gte(0),
expectedSourceVersion: z.int().gte(1),
mode: z.enum(['custom', 'interval', 'manual', 'provider']),
})
/**
* KnowledgeFSSourceCredentialTestResponse
*/
@ -447,6 +603,7 @@ export const zKnowledgeFsSpaceDetailResponse = z.object({
*/
export const zKnowledgeFsSpaceListItemResponse = z.object({
control_space_id: z.string(),
created_at: z.iso.datetime(),
knowledge_space_id: z.string().nullable(),
owner_account_id: z.string(),
permission_keys: z.array(zKnowledgeFsProductPermission),
@ -454,6 +611,7 @@ export const zKnowledgeFsSpaceListItemResponse = z.object({
state: zKnowledgeFsControlSpaceState,
technical_status: z.enum(['available', 'not_ready', 'unavailable']),
technical_summary: zKnowledgeFsTechnicalSummary.nullish(),
updated_at: z.iso.datetime(),
visibility: zKnowledgeFsControlSpaceVisibility,
})
@ -603,6 +761,14 @@ export const zKnowledgeFsDocumentRevisionListResponse = z.object({
next_cursor: z.string().nullish(),
})
/**
* KnowledgeFSLogicalDocumentListResponse
*/
export const zKnowledgeFsLogicalDocumentListResponse = z.object({
data: z.array(zKnowledgeFsLogicalDocumentResponse),
next_cursor: z.string().nullish(),
})
/**
* KnowledgeFSDocumentOutlineNodeResponse
*/
@ -642,6 +808,118 @@ export const zKnowledgeFsDocumentOutlineResponse = z.object({
version: z.int().gte(1),
})
/**
* KnowledgeFSOverviewInventoryDeltaResponse
*/
export const zKnowledgeFsOverviewInventoryDeltaResponse = z.object({
added_last_7d: z.int().gte(0),
total: z.int().gte(0),
})
/**
* KnowledgeFSOverviewIndexCoverageResponse
*/
export const zKnowledgeFsOverviewIndexCoverageResponse = z.object({
indexed: z.int().gte(0),
percentage: z.number().gte(0).lte(100),
total: z.int().gte(0),
})
/**
* KnowledgeFSOverviewSourceCategoriesResponse
*/
export const zKnowledgeFsOverviewSourceCategoriesResponse = z.object({
crawl: z.int().gte(0),
online_documents: z.int().gte(0),
online_drives: z.int().gte(0),
uploads: z.int().gte(0),
})
/**
* KnowledgeFSOverviewInventoryResponse
*/
export const zKnowledgeFsOverviewInventoryResponse = z.object({
generated_at: z.iso.datetime(),
graph_entities: zKnowledgeFsOverviewInventoryDeltaResponse,
graph_relations: zKnowledgeFsOverviewInventoryDeltaResponse,
index_coverage: zKnowledgeFsOverviewIndexCoverageResponse,
knowledge_space_id: z.string(),
source_categories: zKnowledgeFsOverviewSourceCategoriesResponse,
})
/**
* KnowledgeFSOverviewQueryOutcomeBucketResponse
*/
export const zKnowledgeFsOverviewQueryOutcomeBucketResponse = z.object({
answered: z.int().gte(0),
end_at: z.iso.datetime(),
low_confidence: z.int().gte(0),
no_evidence: z.int().gte(0),
query_count: z.int().gte(0),
start_at: z.iso.datetime(),
})
/**
* KnowledgeFSOverviewQueryOutcomeCountsResponse
*/
export const zKnowledgeFsOverviewQueryOutcomeCountsResponse = z.object({
answer_rate: z.number().gte(0).lte(1),
answered: z.int().gte(0),
low_confidence: z.int().gte(0),
no_evidence: z.int().gte(0),
query_count: z.int().gte(0),
})
/**
* KnowledgeFSOverviewQueryOutcomesResponse
*/
export const zKnowledgeFsOverviewQueryOutcomesResponse = z.object({
buckets: z.array(zKnowledgeFsOverviewQueryOutcomeBucketResponse),
current: zKnowledgeFsOverviewQueryOutcomeCountsResponse,
generated_at: z.iso.datetime(),
knowledge_space_id: z.string(),
previous: zKnowledgeFsOverviewQueryOutcomeCountsResponse,
previous_since: z.iso.datetime(),
since: z.iso.datetime(),
window: z.enum(['24h', '30d', '7d']),
})
/**
* KnowledgeFSOverviewRateComparisonResponse
*/
export const zKnowledgeFsOverviewRateComparisonResponse = z.object({
change_percentage_points: z.number(),
previous_value: z.number().gte(0).lte(1),
value: z.number().gte(0).lte(1),
})
/**
* KnowledgeFSOverviewCountComparisonResponse
*/
export const zKnowledgeFsOverviewCountComparisonResponse = z.object({
change_rate: z.number().nullable(),
previous_value: z.int().gte(0),
value: z.int().gte(0),
})
/**
* KnowledgeFSOverviewStatsResponse
*/
export const zKnowledgeFsOverviewStatsResponse = z.object({
answer_rate: zKnowledgeFsOverviewRateComparisonResponse,
documents: z.int().gte(0),
fresh_source_count: z.int().gte(0),
freshness_seconds: z.int().gte(0).nullish(),
generated_at: z.iso.datetime(),
knowledge_space_id: z.string(),
latest_source_sync_at: z.iso.datetime().nullish(),
linked_apps: z.int().gte(0),
queries: zKnowledgeFsOverviewCountComparisonResponse,
source_count: z.int().gte(0),
stale_source_count: z.int().gte(0),
window: z.enum(['24h', '30d', '7d']),
})
/**
* KnowledgeFSAdmittedQueryRequest
*/
@ -802,27 +1080,22 @@ export const zKnowledgeFsProfileModelSelection = z.object({
})
/**
* KnowledgeFSCrawledPageResponse
* KnowledgeFSCrawlPreviewPageResponse
*/
export const zKnowledgeFsCrawledPageResponse = z.object({
content: z.string(),
export const zKnowledgeFsCrawlPreviewPageResponse = z.object({
description: z.string().nullish(),
etag: z.string().nullish(),
page_id: z.string(),
source_url: z.string(),
title: z.string().nullish(),
})
/**
* KnowledgeFSSourceCrawlResponse
* KnowledgeFSCrawlPreviewPageListResponse
*/
export const zKnowledgeFsSourceCrawlResponse = z.object({
completed: z.int().gte(0).nullish(),
failed: z.int().gte(0).nullish(),
imported: z.int().gte(0).nullish(),
pages: z.array(zKnowledgeFsCrawledPageResponse),
replaced: z.int().gte(0).nullish(),
skipped: z.int().gte(0).nullish(),
status: z.string().nullish(),
total: z.int().gte(0).nullish(),
export const zKnowledgeFsCrawlPreviewPageListResponse = z.object({
data: z.array(zKnowledgeFsCrawlPreviewPageResponse),
next_cursor: z.string().nullish(),
})
/**
@ -1127,6 +1400,36 @@ export const zKnowledgeFsPermissionListResponse = z.object({
data: z.array(zKnowledgeFsPermissionResponse),
})
/**
* KnowledgeFSOverviewHealthComponentResponse
*/
export const zKnowledgeFsOverviewHealthComponentResponse = z.object({
codes: z.array(z.string()),
state: z.enum(['degraded', 'healthy', 'unavailable', 'unknown']),
})
/**
* KnowledgeFSOverviewHealthComponentsResponse
*/
export const zKnowledgeFsOverviewHealthComponentsResponse = z.object({
index: zKnowledgeFsOverviewHealthComponentResponse,
ingestion: zKnowledgeFsOverviewHealthComponentResponse,
profile_publication: zKnowledgeFsOverviewHealthComponentResponse,
query_availability: zKnowledgeFsOverviewHealthComponentResponse,
source_freshness: zKnowledgeFsOverviewHealthComponentResponse,
worker_readiness: zKnowledgeFsOverviewHealthComponentResponse,
})
/**
* KnowledgeFSOverviewHealthResponse
*/
export const zKnowledgeFsOverviewHealthResponse = z.object({
components: zKnowledgeFsOverviewHealthComponentsResponse,
generated_at: z.iso.datetime(),
knowledge_space_id: z.string(),
state: z.enum(['degraded', 'healthy', 'unavailable', 'unknown']),
})
/**
* KnowledgeFSProductRerankProfile
*/
@ -1191,6 +1494,38 @@ export const zKnowledgeFsSettingsPayload = z.object({
retrieval: zKnowledgeFsProductRetrievalProfile.nullish(),
})
/**
* KnowledgeFSSourceProviderFieldResponse
*/
export const zKnowledgeFsSourceProviderFieldResponse = z.object({
description: z.string().nullish(),
format: z.enum(['password', 'uri']).nullish(),
name: z.string(),
required: z.boolean(),
secret: z.boolean(),
type: z.enum(['boolean', 'integer', 'string']),
})
/**
* KnowledgeFSSourceProviderResponse
*/
export const zKnowledgeFsSourceProviderResponse = z.object({
auth_kinds: z.array(z.enum(['api-key', 'endpoint', 'oauth2'])),
available: z.boolean(),
capabilities: z.array(z.enum(['online-document', 'online-drive', 'website-crawl'])),
configuration: z.array(zKnowledgeFsSourceProviderFieldResponse),
display_name: z.string(),
id: z.string(),
unavailable_reason: z.string().nullish(),
})
/**
* KnowledgeFSSourceProviderListResponse
*/
export const zKnowledgeFsSourceProviderListResponse = z.object({
data: z.array(zKnowledgeFsSourceProviderResponse),
})
/**
* KnowledgeFSSourceFileResponse
*/
@ -1388,6 +1723,47 @@ export const zDeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAp
export const zDeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdResponse =
z.void()
export const zGetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksPath = z.object({
control_space_id: z.string(),
})
export const zGetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksQuery = z.object({
cursor: z.string().min(1).max(8192).optional(),
limit: z.int().gte(1).lte(100).optional().default(50),
})
/**
* KnowledgeFS background tasks
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksResponse =
zKnowledgeFsBackgroundTaskListResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelPath =
z.object({
control_space_id: z.string(),
task_id: z.string(),
task_kind: z.string(),
})
/**
* KnowledgeFS background task canceled
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse =
zKnowledgeFsBackgroundTaskResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryPath =
z.object({
control_space_id: z.string(),
task_id: z.string(),
task_kind: z.string(),
})
/**
* KnowledgeFS background task retried
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponse =
zKnowledgeFsBackgroundTaskResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdBulkJobsByJobIdPath = z.object({
control_space_id: z.string(),
job_id: z.string(),
@ -1460,6 +1836,10 @@ export const zPostKnowledgeFsSpacesByControlSpaceIdDocumentsResponse = zKnowledg
export const zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsBulkBody =
zKnowledgeFsBulkDocumentDeletePayload
export const zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsBulkHeaders = z.object({
'Idempotency-Key': z.string(),
})
export const zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsBulkPath = z.object({
control_space_id: z.string(),
})
@ -1486,6 +1866,10 @@ export const zPostKnowledgeFsSpacesByControlSpaceIdDocumentsReindexResponse =
export const zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdBody =
zKnowledgeFsDocumentDeletePayload
export const zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdHeaders = z.object({
'Idempotency-Key': z.string(),
})
export const zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdPath = z.object({
control_space_id: z.string(),
document_id: z.string(),
@ -1637,6 +2021,31 @@ export const zPostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryPath = z.obje
export const zPostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponse =
zKnowledgeFsDocumentCompilationJobResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsPath = z.object({
control_space_id: z.string(),
})
export const zGetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsQuery = z.object({
cursor: z.string().min(1).max(1000).optional(),
})
/**
* KnowledgeFS logical documents
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsResponse =
zKnowledgeFsLogicalDocumentListResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdPath = z.object({
control_space_id: z.string(),
document_id: z.string(),
})
/**
* KnowledgeFS logical document
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponse =
zKnowledgeFsLogicalDocumentResponse
export const zPutKnowledgeFsSpacesByControlSpaceIdMembersBody = zKnowledgeFsMembersReplacePayload
export const zPutKnowledgeFsSpacesByControlSpaceIdMembersPath = z.object({
@ -1649,6 +2058,54 @@ export const zPutKnowledgeFsSpacesByControlSpaceIdMembersPath = z.object({
export const zPutKnowledgeFsSpacesByControlSpaceIdMembersResponse =
zKnowledgeFsPermissionListResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewHealthPath = z.object({
control_space_id: z.string(),
})
/**
* KnowledgeFS health
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewHealthResponse =
zKnowledgeFsOverviewHealthResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryPath = z.object({
control_space_id: z.string(),
})
/**
* KnowledgeFS inventory
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryResponse =
zKnowledgeFsOverviewInventoryResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesPath = z.object({
control_space_id: z.string(),
})
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesQuery = z.object({
window: z.enum(['24h', '30d', '7d']).optional().default('24h'),
})
/**
* KnowledgeFS query outcomes
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesResponse =
zKnowledgeFsOverviewQueryOutcomesResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewStatsPath = z.object({
control_space_id: z.string(),
})
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewStatsQuery = z.object({
window: z.enum(['24h', '30d', '7d']).optional().default('24h'),
})
/**
* KnowledgeFS Overview statistics
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdOverviewStatsResponse =
zKnowledgeFsOverviewStatsResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdPermissionsPath = z.object({
control_space_id: z.string(),
})
@ -1791,6 +2248,130 @@ export const zPatchKnowledgeFsSpacesByControlSpaceIdSettingsPath = z.object({
*/
export const zPatchKnowledgeFsSpacesByControlSpaceIdSettingsResponse = zKnowledgeFsSettingsResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsPath = z.object({
control_space_id: z.string(),
})
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsQuery = z.object({
cursor: z.string().min(1).max(4096).optional(),
limit: z.int().gte(1).lte(200).optional().default(50),
})
/**
* KnowledgeFS source connections
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse =
zKnowledgeFsSourceConnectionListResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsBody =
zKnowledgeFsSourceConnectionCreatePayload
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsPath = z.object({
control_space_id: z.string(),
})
/**
* KnowledgeFS source connection created
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse =
zKnowledgeFsSourceConnectionResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshBody =
zKnowledgeFsSourceConnectionRefreshPayload
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshPath =
z.object({
connection_id: z.string(),
control_space_id: z.string(),
})
/**
* KnowledgeFS source connection refreshed
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponse =
zKnowledgeFsSourceConnectionResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceProvidersPath = z.object({
control_space_id: z.string(),
})
/**
* KnowledgeFS source providers
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceProvidersResponse =
zKnowledgeFsSourceProviderListResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPath = z.object({
control_space_id: z.string(),
run_id: z.string(),
})
/**
* KnowledgeFS source workflow
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdResponse =
zKnowledgeFsSourceWorkflowResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelBody =
zKnowledgeFsSourceWorkflowCancelPayload
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelPath = z.object({
control_space_id: z.string(),
run_id: z.string(),
})
/**
* KnowledgeFS source workflow canceled
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponse =
zKnowledgeFsSourceWorkflowResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesPath = z.object({
control_space_id: z.string(),
run_id: z.string(),
})
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesQuery = z.object({
cursor: z.string().min(1).max(4096).optional(),
limit: z.int().gte(1).lte(200).optional().default(50),
})
/**
* KnowledgeFS crawl preview pages
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesResponse =
zKnowledgeFsCrawlPreviewPageListResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryPath = z.object({
control_space_id: z.string(),
run_id: z.string(),
})
/**
* KnowledgeFS source workflow retried
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponse =
zKnowledgeFsSourceWorkflowResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionBody =
zKnowledgeFsCrawlPreviewSelectionPayload
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionHeaders =
z.object({
'Idempotency-Key': z.string(),
})
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionPath = z.object({
control_space_id: z.string(),
run_id: z.string(),
})
/**
* KnowledgeFS crawl preview selection accepted
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponse =
zKnowledgeFsSourceWorkflowResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesPath = z.object({
control_space_id: z.string(),
})
@ -1818,6 +2399,10 @@ export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesResponse = zKnowledgeF
export const zDeleteKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdBody =
zKnowledgeFsSourceDeletePayload
export const zDeleteKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdHeaders = z.object({
'Idempotency-Key': z.string(),
})
export const zDeleteKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPath = z.object({
control_space_id: z.string(),
source_id: z.string(),
@ -1858,16 +2443,20 @@ export const zPatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPath = z.ob
export const zPatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponse =
zKnowledgeFsSourceResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPath = z.object({
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewHeaders = z.object({
'Idempotency-Key': z.string(),
})
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewPath = z.object({
control_space_id: z.string(),
source_id: z.string(),
})
/**
* KnowledgeFS source crawl
* KnowledgeFS source crawl preview accepted
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlResponse =
zKnowledgeFsSourceCrawlResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponse =
zKnowledgeFsSourceWorkflowResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdFilesPath = z.object({
control_space_id: z.string(),
@ -1931,6 +2520,46 @@ export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesQuery =
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesResponse =
zKnowledgeFsSourcePagesResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncHeaders = z.object({
'Idempotency-Key': z.string(),
})
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPath = z.object({
control_space_id: z.string(),
source_id: z.string(),
})
/**
* KnowledgeFS source sync accepted
*/
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponse =
zKnowledgeFsSourceWorkflowResponse
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyPath = z.object({
control_space_id: z.string(),
source_id: z.string(),
})
/**
* KnowledgeFS source sync policy
*/
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponse =
zKnowledgeFsSourceSyncPolicyResponse
export const zPutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyBody =
zKnowledgeFsSourceSyncPolicyPayload
export const zPutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyPath = z.object({
control_space_id: z.string(),
source_id: z.string(),
})
/**
* KnowledgeFS source sync policy updated
*/
export const zPutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponse =
zKnowledgeFsSourceSyncPolicyResponse
export const zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdTestPath = z.object({
control_space_id: z.string(),
source_id: z.string(),
@ -2029,7 +2658,7 @@ export const zPostKnowledgeFsSpacesByControlSpaceIdUploadCapabilitiesResponse =
export const zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdSmallFileBody =
z.object({
file: z.custom<Blob | File>(),
file: z.custom<Blob | File>((value) => value instanceof Blob || value instanceof File),
})
export const zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdSmallFilePath =

1
packages/contracts/generated/api/console/system-features/types.gen.ts
View File

@ -23,6 +23,7 @@ export type SystemFeatureModel = {
is_allow_register: boolean
is_email_setup: boolean
knowledge_fs_enabled: boolean
knowledge_fs_upload_enabled: boolean
license: LicenseStatusModel
max_plugin_package_size: number
plugin_installation_permission: PluginInstallationPermissionModel

1
packages/contracts/generated/api/console/system-features/zod.gen.ts
View File

@ -135,6 +135,7 @@ export const zSystemFeatureModel = z.object({
is_allow_register: z.boolean().default(false),
is_email_setup: z.boolean().default(false),
knowledge_fs_enabled: z.boolean().default(false),
knowledge_fs_upload_enabled: z.boolean().default(false),
license: zLicenseStatusModel.default({ status: 'none' }),
max_plugin_package_size: z.int().default(15728640),
plugin_installation_permission: zPluginInstallationPermissionModel.default({

1
packages/contracts/generated/api/web/types.gen.ts
View File

@ -520,6 +520,7 @@ export type SystemFeatureModel = {
is_allow_register: boolean
is_email_setup: boolean
knowledge_fs_enabled: boolean
knowledge_fs_upload_enabled: boolean
license: LicenseStatusModel
max_plugin_package_size: number
plugin_installation_permission: PluginInstallationPermissionModel

1
packages/contracts/generated/api/web/zod.gen.ts
View File

@ -782,6 +782,7 @@ export const zSystemFeatureModel = z.object({
is_allow_register: z.boolean().default(false),
is_email_setup: z.boolean().default(false),
knowledge_fs_enabled: z.boolean().default(false),
knowledge_fs_upload_enabled: z.boolean().default(false),
license: zLicenseStatusModel.default({ status: 'none' }),
max_plugin_package_size: z.int().default(15728640),
plugin_installation_permission: zPluginInstallationPermissionModel.default({

12
packages/contracts/generated/knowledge-fs/metadata.gen.ts
View File

@ -1,12 +0,0 @@
// This file is auto-generated by scripts/generate-knowledge-fs-contract.mjs.
// Do not edit it manually.
export const knowledgeFsSourceOpenapiSha256 =
'f18910e9c45a64f0855e0643a7a626fb2889021b4f943458de86c6bd2469facb'
export const knowledgeFsConsoleDeclarationsSha256 =
'8bd1924747fdd0d478ca085817cbe000eb7e8630b2c6a03f4f13a6a0fac07946'
export const knowledgeFsGeneratedArtifactSha256 = {
'orpc.gen.ts': 'e0d9954f817e97a4e95dd38c4522fb403c8659d741ce485fa671b1b4ce90a540',
'types.gen.ts': 'a558ab80f32a8555bb5b44b7a596ef4a4a7a8cb7904390993aabcc587915f530',
'zod.gen.ts': 'ca698a6fa64a0717e29da5d4678976b55355a2c0fe5ddfa7037325aa79ab4762',
} as const

1568
packages/contracts/generated/knowledge-fs/orpc.gen.ts
View File

File diff suppressed because it is too large Load Diff

3302
packages/contracts/generated/knowledge-fs/types.gen.ts
View File

File diff suppressed because it is too large Load Diff

2257
packages/contracts/generated/knowledge-fs/zod.gen.ts
View File

File diff suppressed because it is too large Load Diff

47
packages/contracts/knowledge-fs-contract.test.mjs
View File

@ -1,47 +0,0 @@
import { createHash } from 'node:crypto'
import { readFile } from 'node:fs/promises'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { describe, expect, it } from 'vitest'
import {
knowledgeFsGeneratedArtifactSha256,
knowledgeFsSourceOpenapiSha256,
} from './generated/knowledge-fs/metadata.gen'
import { getStreamingOperationIds } from './scripts/knowledge-fs-contract-utils.mjs'
const packageRoot = dirname(fileURLToPath(import.meta.url))
describe('KnowledgeFS contract generation', () => {
it.each(['200', '2XX'])('detects an SSE response declared with %s', (status) => {
expect(
getStreamingOperationIds({
paths: {
'/tasks/{id}/events': {
get: {
operationId: 'streamTaskEvents',
responses: {
[status]: {
content: {
'text/event-stream': {},
},
},
},
},
},
},
}),
).toEqual(['streamTaskEvents'])
})
it('matches the pinned source contract and committed generated artifacts', async () => {
const lock = JSON.parse(
await readFile(join(packageRoot, '../../api/knowledge-fs-contract.lock.json'), 'utf8'),
)
expect(knowledgeFsSourceOpenapiSha256).toBe(lock.openapiSha256)
for (const [fileName, expectedSha256] of Object.entries(knowledgeFsGeneratedArtifactSha256)) {
const content = await readFile(join(packageRoot, 'generated/knowledge-fs', fileName))
expect(createHash('sha256').update(content).digest('hex'), fileName).toBe(expectedSha256)
}
})
})

61
packages/contracts/openapi-ts.knowledge-fs.config.ts
View File

@ -1,61 +0,0 @@
import { $, defineConfig } from '@hey-api/openapi-ts'
const input = process.env.KNOWLEDGE_FS_OPENAPI
const outputPath = process.env.KNOWLEDGE_FS_OUTPUT ?? 'generated/knowledge-fs'
if (!input) throw new Error('KNOWLEDGE_FS_OPENAPI must point to the filtered pinned export')
export default defineConfig({
input,
logs: {
file: false,
},
output: {
clean: true,
entryFile: false,
fileName: {
suffix: '.gen',
},
path: outputPath,
},
parser: {
patch: {
input: (spec) => {
const paths = spec.paths as Record<string, unknown> | undefined
if (!paths) return
for (const [path, pathItem] of Object.entries(paths)) {
delete paths[path]
paths[`/knowledge-fs${path}`] = pathItem
}
},
},
},
plugins: [
{
comments: false,
name: '@hey-api/typescript',
},
{
name: 'zod',
'~resolvers': {
string: (ctx) => {
if (ctx.schema.format === 'binary')
return $(ctx.symbols.z)
.attr('custom')
.call()
.generic($.type.or($.type('Blob'), $.type('File')))
return undefined
},
},
},
{
contracts: {
strategy: 'single',
},
name: 'orpc',
validator: 'zod',
},
],
})

5
packages/contracts/package.json
View File

@ -19,16 +19,11 @@
"./enterprise/*": {
"types": "./generated/enterprise/*.ts",
"import": "./generated/enterprise/*.ts"
},
"./knowledge-fs/*": {
"types": "./generated/knowledge-fs/*.ts",
"import": "./generated/knowledge-fs/*.ts"
}
},
"scripts": {
"gen-api-contract": "uv run --project ../../api ../../api/dev/generate_swagger_specs.py --output-dir openapi && uv run --project ../../api ../../api/dev/generate_fastopenapi_specs.py --output-dir openapi && node -e \"fs.rmSync('generated/api', { recursive: true, force: true })\" && openapi-ts -f openapi-ts.api.config.ts && vp fmt generated/api",
"gen-enterprise-contract": "openapi-ts -f openapi-ts.enterprise.config.ts",
"gen-knowledge-fs-contract": "node scripts/generate-knowledge-fs-contract.mjs",
"test": "vp test",
"type-check": "tsc"
},

119
packages/contracts/scripts/generate-knowledge-fs-contract.mjs
View File

@ -1,119 +0,0 @@
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
import { getStreamingOperationIds } from './knowledge-fs-contract-utils.mjs'
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const workspaceRoot = resolve(packageRoot, '../..')
const repository = resolve(
process.env.KNOWLEDGE_FS_REPO ?? resolve(workspaceRoot, '../knowledge-fs'),
)
const temporaryDirectory = await mkdtemp(join(tmpdir(), 'dify-knowledge-fs-types-'))
try {
const openapiPath = join(temporaryDirectory, 'knowledge-fs.console.json')
run(
'uv',
[
'run',
'--project',
resolve(workspaceRoot, 'api'),
resolve(workspaceRoot, 'api/dev/generate_knowledge_fs_contract.py'),
'--repository',
repository,
'--check',
'--output-openapi',
openapiPath,
],
workspaceRoot,
)
run('pnpm', ['exec', 'openapi-ts', '-f', 'openapi-ts.knowledge-fs.config.ts'], packageRoot, {
KNOWLEDGE_FS_OPENAPI: openapiPath,
})
await patchStreamingContracts(openapiPath)
run('pnpm', ['exec', 'vp', 'fmt', 'generated/knowledge-fs'], packageRoot)
await writeContractMetadata(openapiPath, await generatedArtifactSha256())
run('pnpm', ['exec', 'vp', 'fmt', 'generated/knowledge-fs/metadata.gen.ts'], packageRoot)
} finally {
await rm(temporaryDirectory, { force: true, recursive: true })
}
async function patchStreamingContracts(openapiPath) {
const document = JSON.parse(await readFile(openapiPath, 'utf8'))
const streamingOperationIds = getStreamingOperationIds(document)
if (streamingOperationIds.length === 0) return
const outputPath = join(packageRoot, 'generated/knowledge-fs/orpc.gen.ts')
let source = await readFile(outputPath, 'utf8')
source = replaceOnce(
source,
"import { oc } from '@orpc/contract'",
"import { eventIterator, oc } from '@orpc/contract'",
)
for (const operationId of streamingOperationIds) {
const responseSchema = `z${capitalize(operationId)}Response`
source = replaceOnce(
source,
`.output(${responseSchema})`,
`.output(eventIterator(${responseSchema}))`,
)
}
await writeFile(outputPath, source)
}
function capitalize(value) {
return value.charAt(0).toUpperCase() + value.slice(1)
}
function replaceOnce(source, target, replacement) {
const firstIndex = source.indexOf(target)
if (firstIndex === -1 || source.indexOf(target, firstIndex + target.length) !== -1)
throw new Error(`Expected exactly one generated occurrence of ${target}`)
return source.slice(0, firstIndex) + replacement + source.slice(firstIndex + target.length)
}
async function generatedArtifactSha256() {
const generatedDirectory = join(packageRoot, 'generated/knowledge-fs')
const fileNames = (await readdir(generatedDirectory))
.filter((fileName) => fileName.endsWith('.gen.ts') && fileName !== 'metadata.gen.ts')
.sort()
return Object.fromEntries(
await Promise.all(
fileNames.map(async (fileName) => [
fileName,
createHash('sha256')
.update(await readFile(join(generatedDirectory, fileName)))
.digest('hex'),
]),
),
)
}
async function writeContractMetadata(openapiPath, artifactSha256) {
const document = JSON.parse(await readFile(openapiPath, 'utf8'))
const source = [
'// This file is auto-generated by scripts/generate-knowledge-fs-contract.mjs.',
'// Do not edit it manually.',
'',
`export const knowledgeFsSourceOpenapiSha256 = ${JSON.stringify(document['x-dify-source-openapi-sha256'])}`,
`export const knowledgeFsConsoleDeclarationsSha256 = ${JSON.stringify(document['x-dify-console-declarations-sha256'])}`,
`export const knowledgeFsGeneratedArtifactSha256 = ${JSON.stringify(artifactSha256, null, 2)} as const`,
'',
].join('\n')
await writeFile(join(packageRoot, 'generated/knowledge-fs/metadata.gen.ts'), source)
}
function run(command, args, cwd, extraEnv = {}) {
execFileSync(command, args, {
cwd,
env: { ...process.env, ...extraEnv },
stdio: 'inherit',
})
}

19
packages/contracts/scripts/knowledge-fs-contract-utils.mjs
View File

@ -1,19 +0,0 @@
export function getStreamingOperationIds(document) {
return Object.values(document.paths ?? {})
.flatMap((pathItem) =>
Object.values(pathItem).flatMap((operation) => {
if (typeof operation !== 'object' || operation === null) return []
const isEventStream = Object.entries(operation.responses ?? {}).some(
([status, response]) =>
(status === '2XX' || /^2\d\d$/.test(status)) &&
typeof response === 'object' &&
response !== null &&
'text/event-stream' in (response.content ?? {}),
)
return isEventStream && typeof operation.operationId === 'string'
? [operation.operationId]
: []
}),
)
.sort()
}

4
web/context/system-features-state.ts
View File

@ -17,3 +17,7 @@ export const deploymentEditionAtom = atom((get) => {
export const brandingEnabledAtom = atom((get) => {
return get(systemFeaturesAtom).branding.enabled
})
export const knowledgeFsUploadEnabledAtom = atom((get) => {
return get(systemFeaturesAtom).knowledge_fs_upload_enabled
})

306
web/features/new-rag/__tests__/add-source-page.spec.tsx
View File

@ -1,7 +1,5 @@
import type {
GetKnowledgeSpacesByIdSourceConnectionsResponse,
GetSourceProvidersResponse,
} from '@dify/contracts/knowledge-fs/types.gen'
import type { DatasourceProviderAuthListResponse } from '@dify/contracts/api/console/auth/types.gen'
import type { SourceConnection, SourceProvider } from '../source-models'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { StrictMode } from 'react'
@ -9,14 +7,53 @@ import { render } from '@/test/console/render'
import { AddSourcePage } from '../add-source-page'
import { newKnowledgeSourceDraftStorageKey } from '../routes'
type GetKnowledgeSpacesByIdSourceConnectionsResponse = {
items: SourceConnection[]
nextCursor?: string
}
type GetSourceProvidersResponse = { items: SourceProvider[] }
const routerMock = vi.hoisted(() => ({
push: vi.fn(),
replace: vi.fn(),
}))
const connectFirecrawlButtonName = 'dataset.newKnowledge.connectProvider:{"provider":"Firecrawl"}'
vi.mock('@/next/navigation', () => ({ useRouter: () => routerMock }))
const toastInfoMock = vi.hoisted(() => vi.fn())
const providerApiResponse = vi.hoisted(() => (provider: SourceProvider) => ({
auth_kinds: provider.authKinds,
available: provider.available,
capabilities: provider.capabilities,
configuration: provider.configuration.map((field) => ({
description: field.description ?? null,
format: field.format ?? null,
name: field.name,
required: field.required,
secret: field.secret,
type: field.type,
})),
display_name: provider.displayName,
id: provider.id,
unavailable_reason: provider.unavailableReason ?? null,
}))
const connectionApiResponse = vi.hoisted(() => (connection: SourceConnection) => ({
auth_kind: connection.authKind,
configuration: connection.configuration,
created_at: connection.createdAt,
error_code: connection.errorCode ?? null,
expires_at: connection.expiresAt ?? null,
id: connection.id,
knowledge_space_id: connection.knowledgeSpaceId,
name: connection.name,
provider_id: connection.providerId,
scopes: connection.scopes,
status: connection.status,
updated_at: connection.updatedAt,
version: connection.version,
}))
vi.mock('@langgenius/dify-ui/toast', () => ({
toast: { info: toastInfoMock },
@ -25,12 +62,19 @@ vi.mock('@langgenius/dify-ui/toast', () => ({
type ConnectionsInfiniteData = {
pages: GetKnowledgeSpacesByIdSourceConnectionsResponse[]
}
const connectionInfiniteDataApiResponse = vi.hoisted(() => (data: ConnectionsInfiniteData) => ({
pages: data.pages.map((page) => ({
data: page.items.map(connectionApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}))
type ConnectionsInfiniteOptions = {
enabled?: boolean
getNextPageParam: (
lastPage: GetKnowledgeSpacesByIdSourceConnectionsResponse,
) => string | undefined
getNextPageParam: (lastPage: {
data: ReturnType<typeof connectionApiResponse>[]
next_cursor?: string | null
}) => string | null | undefined
input: (pageParam: string | null) => unknown
initialPageParam: string | null
}
@ -52,6 +96,12 @@ const queryState = vi.hoisted(() => ({
isPending: false,
refetch: vi.fn(),
},
datasourceAuth: {
data: { result: [] } as DatasourceProviderAuthListResponse | undefined,
error: null as unknown,
isPending: false,
refetch: vi.fn(),
},
}))
const clientMock = vi.hoisted(() => ({
@ -64,9 +114,10 @@ const queryClientMock = vi.hoisted(() => ({
}))
const providerQueryOptionsMock = vi.hoisted(() =>
vi.fn((options: { enabled?: boolean }) => ({
vi.fn((options: { enabled?: boolean; select?: (data: unknown) => unknown }) => ({
enabled: options.enabled,
queryKey: ['source-providers'],
select: options.select,
})),
)
const connectionInfiniteOptionsMock = vi.hoisted(() =>
@ -76,6 +127,12 @@ const connectionInfiniteOptionsMock = vi.hoisted(() =>
})),
)
const providerHookOptionsMock = vi.hoisted(() => vi.fn())
const datasourceAuthQueryOptionsMock = vi.hoisted(() =>
vi.fn((options: { enabled?: boolean }) => ({
enabled: options.enabled,
queryKey: ['datasource-auth'],
})),
)
const connectionHookOptionsMock = vi.hoisted(() => vi.fn())
vi.mock('@tanstack/react-query', async (importOriginal) => {
@ -84,11 +141,33 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
...original,
useInfiniteQuery: (options: unknown) => {
connectionHookOptionsMock(options)
return queryState.connections
return {
...queryState.connections,
data: queryState.connections.data
? connectionInfiniteDataApiResponse(queryState.connections.data)
: undefined,
refetch: async () => {
const result = (await queryState.connections.refetch()) as
| { data?: ConnectionsInfiniteData; error?: unknown }
| undefined
if (!result?.data) return result
return {
...result,
data: connectionInfiniteDataApiResponse(result.data),
}
},
}
},
useQuery: (options: unknown) => {
useQuery: (options: { queryKey?: string[]; select?: (data: unknown) => unknown }) => {
providerHookOptionsMock(options)
return queryState.providers
if (options.queryKey?.[0] === 'datasource-auth') return queryState.datasourceAuth
const raw = queryState.providers.data
? { data: queryState.providers.data.items.map(providerApiResponse) }
: undefined
return {
...queryState.providers,
data: raw && options.select ? options.select(raw) : raw,
}
},
useQueryClient: () => queryClientMock,
}
@ -97,18 +176,49 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
postKnowledgeSpacesByIdSourceConnections: clientMock.createConnection,
postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh: clientMock.refreshConnection,
spaces: {
byControlSpaceId: {
sourceConnections: {
byConnectionId: {
refresh: {
post: async (input: unknown) =>
connectionApiResponse(await clientMock.refreshConnection(input)),
},
},
post: async (input: unknown) =>
connectionApiResponse(await clientMock.createConnection(input)),
},
},
},
},
},
consoleQuery: {
knowledgeFs: {
getSourceProviders: {
queryOptions: providerQueryOptionsMock,
auth: {
plugin: {
datasource: {
defaultList: {
get: {
queryOptions: datasourceAuthQueryOptionsMock,
},
},
},
},
getKnowledgeSpacesByIdSourceConnections: {
infiniteOptions: connectionInfiniteOptionsMock,
key: vi.fn(() => ['source-connections']),
},
knowledgeFs: {
spaces: {
byControlSpaceId: {
sourceConnections: {
get: {
infiniteOptions: connectionInfiniteOptionsMock,
key: vi.fn(() => ['source-connections']),
},
},
sourceProviders: {
get: {
queryOptions: providerQueryOptionsMock,
},
},
},
},
},
},
@ -161,6 +271,69 @@ const firecrawlProvider: GetSourceProvidersResponse['items'][number] = {
id: 'plugin-daemon-website',
}
const difyManagedFirecrawlProvider: GetSourceProvidersResponse['items'][number] = {
authKinds: ['endpoint'],
available: true,
capabilities: ['website-crawl'],
configuration: [
{
name: 'credentialId',
required: true,
secret: false,
type: 'string',
},
{
name: 'pluginId',
required: true,
secret: false,
type: 'string',
},
{
name: 'provider',
required: true,
secret: false,
type: 'string',
},
{
name: 'datasource',
required: true,
secret: false,
type: 'string',
},
{
name: 'providerKind',
required: true,
secret: false,
type: 'string',
},
],
displayName: 'Dify website crawl',
id: 'plugin-daemon-website',
}
const firecrawlDatasourceAuth: DatasourceProviderAuthListResponse['result'][number] = {
author: 'langgenius',
credential_schema: [],
credentials_list: [
{
avatar_url: null,
credential: {},
id: 'firecrawl-credential-1',
is_default: true,
name: 'Default Firecrawl',
type: 'api-key',
},
],
description: { en_US: 'Firecrawl' },
icon: 'icon.svg',
label: { en_US: 'Firecrawl' },
name: 'firecrawl',
oauth_schema: null,
plugin_id: 'langgenius/firecrawl_datasource',
plugin_unique_identifier: 'langgenius/firecrawl_datasource:1.0.0@local',
provider: 'firecrawl',
}
const connection = (
status: 'provisioning' | 'active' | 'expired' | 'error' | 'revoked',
version = 2,
@ -190,9 +363,13 @@ describe('AddSourcePage', () => {
clientMock.refreshConnection.mockReset()
queryState.connections.refetch.mockReset()
queryState.providers.refetch.mockReset()
queryState.datasourceAuth.refetch.mockReset()
queryState.providers.data = { items: [firecrawlProvider] }
queryState.providers.error = null
queryState.providers.isPending = false
queryState.datasourceAuth.data = { result: [] }
queryState.datasourceAuth.error = null
queryState.datasourceAuth.isPending = false
queryState.connections.data = { pages: [{ items: [] }] }
queryState.connections.error = null
queryState.connections.hasNextPage = false
@ -213,18 +390,22 @@ describe('AddSourcePage', () => {
expect(providerQueryOptionsMock).toHaveBeenCalledWith({
context: { silent: true },
enabled: true,
input: {},
input: { params: { control_space_id: 'space-1' } },
retry: false,
select: expect.any(Function),
})
const options = connectionInfiniteOptionsMock.mock.lastCall?.[0]
expect(options).toBeDefined()
if (!options) throw new Error('Expected connection infinite query options')
expect(options.input(null)).toEqual({ params: { id: 'space-1' }, query: { limit: 200 } })
expect(options.input(null)).toEqual({
params: { control_space_id: 'space-1' },
query: { limit: 200 },
})
expect(options.input('next')).toEqual({
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
query: { cursor: 'next', limit: 200 },
})
expect(options.getNextPageParam({ items: [], nextCursor: 'next' })).toBe('next')
expect(options.getNextPageParam({ data: [], next_cursor: 'next' })).toBe('next')
expect(options.initialPageParam).toBeNull()
expect(screen.getByRole('status')).toBeInTheDocument()
})
@ -534,7 +715,7 @@ describe('AddSourcePage', () => {
)
await user.type(screen.getByLabelText(/Api Key/), 'secret-value')
await user.type(screen.getByLabelText('Endpoint'), 'https://crawl.example.com')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.connectProvider' }))
await user.click(screen.getByRole('button', { name: connectFirecrawlButtonName }))
await waitFor(() =>
expect(clientMock.createConnection).toHaveBeenCalledWith({
@ -550,7 +731,7 @@ describe('AddSourcePage', () => {
name: 'Firecrawl',
providerId: 'plugin-daemon-website',
},
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
}),
)
await screen.findByRole('status', { name: 'appApi.loading' })
@ -562,6 +743,64 @@ describe('AddSourcePage', () => {
expect(screen.queryByDisplayValue('secret-value')).not.toBeInTheDocument()
})
it('binds the default Dify Firecrawl credential for the real KnowledgeFS provider', async () => {
const user = userEvent.setup()
queryState.providers.data = { items: [difyManagedFirecrawlProvider] }
queryState.datasourceAuth.data = { result: [firecrawlDatasourceAuth] }
clientMock.createConnection.mockResolvedValue({
...connection('active'),
authKind: 'endpoint',
configuration: {
credentialId: 'firecrawl-credential-1',
datasource: 'crawl',
pluginId: 'langgenius/firecrawl_datasource',
provider: 'firecrawl',
providerKind: 'website',
},
})
render(<AddSourcePage knowledgeSpaceId="space-1" />)
await user.click(screen.getByRole('button', { name: connectFirecrawlButtonName }))
await waitFor(() =>
expect(clientMock.createConnection).toHaveBeenCalledWith({
body: {
authKind: 'endpoint',
configuration: {
credentialId: 'firecrawl-credential-1',
datasource: 'crawl',
pluginId: 'langgenius/firecrawl_datasource',
provider: 'firecrawl',
providerKind: 'website',
},
credentials: {},
name: 'Firecrawl',
providerId: 'plugin-daemon-website',
},
params: { control_space_id: 'space-1' },
}),
)
expect(screen.queryByLabelText(/Api Key/)).not.toBeInTheDocument()
})
it('opens Data Source settings when Dify has no Firecrawl credential', async () => {
const user = userEvent.setup()
queryState.providers.data = { items: [difyManagedFirecrawlProvider] }
render(<AddSourcePage knowledgeSpaceId="space-1" />)
expect(screen.queryByLabelText(/Api Key/)).not.toBeInTheDocument()
expect(
screen.queryByRole('button', { name: connectFirecrawlButtonName }),
).not.toBeInTheDocument()
await user.click(
screen.getByRole('button', { name: 'dataset.newKnowledge.openDataSourceSettings' }),
)
expect(routerMock.replace).toHaveBeenCalledWith('/integrations/data-source')
expect(clientMock.createConnection).not.toHaveBeenCalled()
})
it('releases the parent history guard before the crawl preview owns navigation', async () => {
const user = userEvent.setup()
const historyBack = vi.spyOn(window.history, 'back').mockImplementation(() => undefined)
@ -572,7 +811,7 @@ describe('AddSourcePage', () => {
screen.getByRole('button', { name: /^dataset\.newKnowledge\.configureProvider/ }),
)
await user.type(screen.getByLabelText(/Api Key/), 'secret-value')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.connectProvider' }))
await user.click(screen.getByRole('button', { name: connectFirecrawlButtonName }))
await waitFor(() => expect(historyBack).toHaveBeenCalledOnce())
expect(screen.queryByText(/dataset\.newKnowledge\.providerConnected/)).not.toBeInTheDocument()
@ -677,7 +916,7 @@ describe('AddSourcePage', () => {
)
await user.type(screen.getByLabelText(/Api Key/), 'do-not-retain')
await user.type(screen.getByLabelText('Endpoint'), 'https://crawl.example.com')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.connectProvider' }))
await user.click(screen.getByRole('button', { name: connectFirecrawlButtonName }))
expect(await screen.findByText('dataset.newKnowledge.connectionFailed')).toBeInTheDocument()
expect(screen.getByLabelText(/Api Key/)).toHaveValue('')
@ -696,10 +935,9 @@ describe('AddSourcePage', () => {
screen.getByRole('button', { name: /^dataset\.newKnowledge\.configureProvider/ }),
)
await user.type(screen.getByLabelText(/Api Key/), 'secret-value')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.connectProvider' }))
await user.click(screen.getByRole('button', { name: connectFirecrawlButtonName }))
await waitFor(() => expect(clientMock.createConnection).toHaveBeenCalledOnce())
await screen.findByRole('status', { name: 'appApi.loading' })
act(() => window.dispatchEvent(new PopStateEvent('popstate')))
expect(await screen.findByText(/dataset\.newKnowledge\.providerConnected/)).toBeInTheDocument()
expect(screen.queryByText('dataset.newKnowledge.connectionFailed')).not.toBeInTheDocument()
@ -734,7 +972,7 @@ describe('AddSourcePage', () => {
await user.type(screen.getByLabelText(/Api Key/), 'must-not-be-sent')
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.authKind.endpoint' }))
await user.type(screen.getByLabelText('Endpoint'), 'https://crawl.example.com')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.connectProvider' }))
await user.click(screen.getByRole('button', { name: connectFirecrawlButtonName }))
await waitFor(() =>
expect(clientMock.createConnection).toHaveBeenCalledWith({
@ -750,7 +988,7 @@ describe('AddSourcePage', () => {
name: 'Firecrawl',
providerId: 'plugin-daemon-website',
},
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
}),
)
})
@ -792,7 +1030,7 @@ describe('AddSourcePage', () => {
await waitFor(() =>
expect(clientMock.refreshConnection).toHaveBeenCalledWith({
body: { expectedVersion: 2 },
params: { connectionId: 'connection-1', id: 'space-1' },
params: { connection_id: 'connection-1', control_space_id: 'space-1' },
}),
)
expect(queryClientMock.invalidateQueries).toHaveBeenCalled()
@ -830,7 +1068,7 @@ describe('AddSourcePage', () => {
await waitFor(() =>
expect(clientMock.refreshConnection).toHaveBeenLastCalledWith({
body: { expectedVersion: 3 },
params: { connectionId: 'connection-1', id: 'space-1' },
params: { connection_id: 'connection-1', control_space_id: 'space-1' },
}),
)
})
@ -853,7 +1091,7 @@ describe('AddSourcePage', () => {
await waitFor(() =>
expect(clientMock.refreshConnection).toHaveBeenLastCalledWith({
body: { expectedVersion: 3 },
params: { connectionId: 'connection-1', id: 'space-1' },
params: { connection_id: 'connection-1', control_space_id: 'space-1' },
}),
)
})

116
web/features/new-rag/__tests__/crawl-selection-form.spec.tsx
View File

@ -1,9 +1,9 @@
import type {
GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse,
GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse,
CrawlPreviewPageList,
Source,
SourceSyncPolicy,
SourceWorkflowRun,
} from '@dify/contracts/knowledge-fs/types.gen'
} from '../source-models'
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
import { act, screen, waitFor } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
@ -13,19 +13,55 @@ import datasetTranslations from '@/i18n/en-US/dataset.json'
import { render } from '@/test/console/render'
import { CrawlSelectionForm } from '../crawl-selection-form'
type GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse = SourceSyncPolicy
type GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse = CrawlPreviewPageList
const clientMock = vi.hoisted(() => ({
getPolicy: vi.fn(),
getWorkflow: vi.fn(),
selectPages: vi.fn(),
updatePolicy: vi.fn(),
}))
const policyApiResponse = vi.hoisted(() => (policy: SourceSyncPolicy) => ({
created_at: policy.createdAt,
custom_interval_seconds: policy.customIntervalSeconds ?? null,
enabled: policy.enabled,
expected_source_version: policy.expectedSourceVersion,
id: policy.id,
knowledge_space_id: policy.knowledgeSpaceId,
mode: policy.mode,
next_run_at: policy.nextRunAt ?? null,
revision: policy.revision,
source_id: policy.sourceId,
updated_at: policy.updatedAt,
}))
const workflowApiResponse = vi.hoisted(() => (workflow: SourceWorkflowRun) => ({
canceled_at: workflow.canceledAt ?? null,
checkpoint: workflow.checkpoint,
completed_at: workflow.completedAt ?? null,
created_at: workflow.createdAt,
cursor: workflow.cursor ?? null,
execution_attempts: workflow.executionAttempts,
id: workflow.id,
knowledge_space_id: workflow.knowledgeSpaceId,
kind: workflow.kind,
last_error_code: workflow.lastErrorCode ?? null,
max_execution_attempts: workflow.maxExecutionAttempts,
progress_completed: workflow.progressCompleted,
progress_failed: workflow.progressFailed,
progress_skipped: workflow.progressSkipped,
progress_total: workflow.progressTotal ?? null,
source_id: workflow.sourceId ?? null,
state: workflow.state,
updated_at: workflow.updatedAt,
}))
const routerMock = vi.hoisted(() => ({ push: vi.fn() }))
const queryClientMock = vi.hoisted(() => ({ invalidateQueries: vi.fn() }))
const policyQueryOptionsMock = vi.hoisted(() =>
vi.fn(({ input }) => ({
queryFn: () => clientMock.getPolicy(input),
queryKey: ['sync-policy', input.params.sourceId],
vi.fn(({ input, select }) => ({
queryFn: async () => select(policyApiResponse(await clientMock.getPolicy(input))),
queryKey: ['sync-policy', input.params.source_id],
retry: false,
})),
)
@ -40,27 +76,51 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy: clientMock.getPolicy,
getKnowledgeSpacesByIdSourceWorkflowsByRunId: clientMock.getWorkflow,
spaces: {
byControlSpaceId: {
sourceWorkflows: {
byRunId: {
get: async (input: unknown) =>
workflowApiResponse(await clientMock.getWorkflow(input)),
selection: {
post: async (input: unknown) =>
workflowApiResponse(await clientMock.selectPages(input)),
},
},
},
sources: {
bySourceId: {
syncPolicy: {
get: async (input: unknown) => policyApiResponse(await clientMock.getPolicy(input)),
put: async (input: unknown) =>
policyApiResponse(await clientMock.updatePolicy(input)),
},
},
get: {
key: vi.fn(() => ['knowledge-sources']),
},
},
},
},
},
},
consoleQuery: {
knowledgeFs: {
getKnowledgeSpacesByIdSources: {
key: vi.fn(() => ['knowledge-sources']),
},
getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy: {
queryOptions: policyQueryOptionsMock,
},
postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection: {
mutationOptions: vi.fn(() => ({
mutationFn: (input: unknown) => clientMock.selectPages(input),
})),
},
putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy: {
mutationOptions: vi.fn(() => ({
mutationFn: (input: unknown) => clientMock.updatePolicy(input),
})),
spaces: {
byControlSpaceId: {
sources: {
bySourceId: {
syncPolicy: {
get: {
queryOptions: policyQueryOptionsMock,
},
},
},
get: {
key: vi.fn(() => ['knowledge-sources']),
},
},
},
},
},
},
@ -346,13 +406,13 @@ describe('CrawlSelectionForm', () => {
expectedSourceVersion: 3,
mode: 'custom',
},
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
})
expect(clientMock.selectPages).toHaveBeenCalledOnce()
expect(clientMock.selectPages).toHaveBeenCalledWith({
body: { pageIds: ['page-1'] },
headers: { 'Idempotency-Key': expect.any(String) },
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
})
selectionRequest.resolve({ ...run, checkpoint: 'import', state: 'queued' })
@ -499,7 +559,7 @@ describe('CrawlSelectionForm', () => {
expectedSourceVersion: 3,
mode,
},
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
})
expect(clientMock.selectPages).toHaveBeenCalledOnce()
},
@ -587,7 +647,7 @@ describe('CrawlSelectionForm', () => {
expectedSourceVersion: 4,
mode: 'manual',
},
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
})
expect(clientMock.selectPages).toHaveBeenCalledOnce()
})
@ -628,7 +688,7 @@ describe('CrawlSelectionForm', () => {
expectedSourceVersion: 3,
mode: 'provider',
},
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
})
expect(clientMock.selectPages).toHaveBeenCalledOnce()
})

279
web/features/new-rag/__tests__/create-knowledge-page.spec.tsx
View File

@ -7,8 +7,7 @@ import { newKnowledgeSourceDraftStorageKey } from '../routes'
const serviceMock = vi.hoisted(() => ({
create: vi.fn(),
getPolicy: vi.fn(),
patchPolicy: vi.fn(),
getDefaultModel: vi.fn(),
upload: vi.fn(),
uploadBulk: vi.fn(),
listKey: vi.fn(() => ['console', 'knowledgeFs', 'listKnowledgeSpaces']),
@ -28,6 +27,11 @@ const permissionStateMock = vi.hoisted(() => ({
keys: ['dataset.create_and_management', 'dataset.acl.access_config'],
}))
const systemFeaturesStateMock = vi.hoisted(() => ({
atom: Symbol('knowledgeFsUploadEnabledAtom'),
uploadEnabled: true,
}))
vi.mock('@/next/navigation', () => ({
useRouter: () => routerMock,
useSearchParams: () => ({
@ -39,6 +43,10 @@ vi.mock('@/context/permission-state', () => ({
workspacePermissionKeysAtom: permissionStateMock.atom,
}))
vi.mock('@/context/system-features-state', () => ({
knowledgeFsUploadEnabledAtom: systemFeaturesStateMock.atom,
}))
vi.mock('jotai', async (importOriginal) => {
const original = await importOriginal<typeof import('jotai')>()
return {
@ -46,40 +54,62 @@ vi.mock('jotai', async (importOriginal) => {
useAtomValue: (atom: unknown) =>
atom === permissionStateMock.atom
? permissionStateMock.keys
: original.useAtomValue(atom as Parameters<typeof original.useAtomValue>[0]),
: atom === systemFeaturesStateMock.atom
? systemFeaturesStateMock.uploadEnabled
: original.useAtomValue(atom as Parameters<typeof original.useAtomValue>[0]),
}
})
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
createKnowledgeSpace: serviceMock.create,
getKnowledgeSpacesByIdAccessPolicy: serviceMock.getPolicy,
patchKnowledgeSpacesByIdAccessPolicy: serviceMock.patchPolicy,
postKnowledgeSpacesByIdDocuments: serviceMock.upload,
postKnowledgeSpacesByIdDocumentsBulk: serviceMock.uploadBulk,
spaces: {
post: serviceMock.create,
},
},
workspaces: {
current: {
defaultModel: {
get: serviceMock.getDefaultModel,
},
},
},
},
consoleQuery: {
knowledgeFs: {
listKnowledgeSpaces: {
key: serviceMock.listKey,
spaces: {
get: {
key: serviceMock.listKey,
},
},
},
},
}))
const createdKnowledge = {
configurationStatus: 'ready',
createdAt: '2026-07-20T00:00:00Z',
id: 'e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084',
name: 'Product handbook',
revision: 1,
slug: 'product-handbook',
tenantId: 'tenant-1',
updatedAt: '2026-07-20T00:00:00Z',
control_space_id: 'e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084',
operation_id: 'operation-1',
state: 'provisioning' as const,
}
vi.mock('../knowledge-fs-upload', () => ({
uploadKnowledgeFsDocuments: async (
knowledgeSpaceId: string,
uploads: Array<{ file: File; id: string }>,
) => {
const files = uploads.map(({ file }) => file)
if (files.length === 1)
return serviceMock.upload({
body: { file: files[0] },
params: { control_space_id: knowledgeSpaceId },
})
return serviceMock.uploadBulk({
body: { files },
params: { control_space_id: knowledgeSpaceId },
})
},
}))
function renderPage(
queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } }),
) {
@ -110,20 +140,20 @@ describe('CreateKnowledgePage', () => {
vi.clearAllMocks()
globalThis.sessionStorage.clear()
serviceMock.create.mockResolvedValue(createdKnowledge)
serviceMock.getPolicy.mockResolvedValue({
id: 'policy-1',
ownerSubjectId: 'user-1',
partialMemberSubjectIds: [],
revision: 4,
visibility: 'only_me',
})
serviceMock.patchPolicy.mockResolvedValue({
id: 'policy-1',
ownerSubjectId: 'user-1',
partialMemberSubjectIds: [],
revision: 5,
visibility: 'all_members',
})
serviceMock.getDefaultModel.mockImplementation(({ query }: { query: { model_type: string } }) =>
Promise.resolve({
data: {
model: query.model_type === 'llm' ? 'echo' : 'embed',
model_type: query.model_type,
provider: {
provider:
query.model_type === 'llm'
? 'kurokobo/fake_models/fake_models'
: 'langgenius/cohere/cohere',
},
},
}),
)
serviceMock.upload.mockResolvedValue({
id: 'document-1',
})
@ -133,6 +163,7 @@ describe('CreateKnowledgePage', () => {
items: [],
})
permissionStateMock.keys = ['dataset.create_and_management', 'dataset.acl.access_config']
systemFeaturesStateMock.uploadEnabled = true
navigationMock.startMode = null
vi.spyOn(globalThis.crypto, 'randomUUID').mockReturnValue(
'a9c36c57-2d84-44d6-a36d-841f0d92a179',
@ -174,12 +205,29 @@ describe('CreateKnowledgePage', () => {
expect(serviceMock.create).toHaveBeenCalledWith({
body: {
description: 'Internal answers',
idempotencyKey: 'a9c36c57-2d84-44d6-a36d-841f0d92a179',
embedding: {
model: 'embed',
plugin_id: 'langgenius/cohere',
provider: 'cohere',
},
idempotency_key: 'a9c36c57-2d84-44d6-a36d-841f0d92a179',
name: 'Product handbook',
retrieval: {
default_mode: 'fast',
reasoning_model: {
model: 'echo',
plugin_id: 'kurokobo/fake_models',
provider: 'fake_models',
},
rerank: { enabled: false },
score_threshold: { enabled: false, stage: 'mode-final' },
top_k: 10,
},
slug: 'product-handbook-a9c36c572d84',
visibility: 'only_me',
},
})
})
expect(serviceMock.getPolicy).not.toHaveBeenCalled()
expect(invalidate).toHaveBeenCalledWith({
queryKey: ['console', 'knowledgeFs', 'listKnowledgeSpaces'],
})
@ -188,7 +236,7 @@ describe('CreateKnowledgePage', () => {
)
})
it('defaults authorized users to the Figma all-members policy and updates its revision', async () => {
it('creates the default all-members visibility atomically', async () => {
const user = userEvent.setup()
renderPage()
await fillRequiredFields(user)
@ -199,13 +247,8 @@ describe('CreateKnowledgePage', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => {
expect(serviceMock.patchPolicy).toHaveBeenCalledWith({
body: {
expectedRevision: 4,
partialMemberSubjectIds: [],
visibility: 'all_members',
},
params: { id: createdKnowledge.id },
expect(serviceMock.create).toHaveBeenCalledWith({
body: expect.objectContaining({ visibility: 'all_team_members' }),
})
})
})
@ -226,7 +269,9 @@ describe('CreateKnowledgePage', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce())
expect(serviceMock.getPolicy).not.toHaveBeenCalled()
expect(serviceMock.create).toHaveBeenCalledWith({
body: expect.objectContaining({ visibility: 'only_me' }),
})
})
it('prevents duplicate pending submissions', async () => {
@ -263,11 +308,64 @@ describe('CreateKnowledgePage', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledTimes(2))
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotencyKey).toBe(
serviceMock.create.mock.calls[1]?.[0].body.idempotencyKey,
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotency_key).toBe(
serviceMock.create.mock.calls[1]?.[0].body.idempotency_key,
)
})
it('unlocks editable fields and rotates the idempotency key after model preflight fails', async () => {
const user = userEvent.setup()
vi.mocked(globalThis.crypto.randomUUID)
.mockReturnValueOnce('11111111-1111-4111-8111-111111111111')
.mockReturnValueOnce('22222222-2222-4222-8222-222222222222')
serviceMock.getDefaultModel.mockImplementation(({ query }: { query: { model_type: string } }) =>
Promise.resolve(
query.model_type === 'llm'
? {
data: {
model: 'echo',
provider: { provider: 'kurokobo/fake_models/fake_models' },
},
}
: { data: null },
),
)
renderPage()
await fillRequiredFields(user)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
expect(await screen.findByRole('alert')).toHaveTextContent('dataset.newKnowledge.createFailed')
const nameInput = screen.getByRole('textbox', { name: 'dataset.newKnowledge.name' })
expect(nameInput).toBeEnabled()
expect(serviceMock.create).not.toHaveBeenCalled()
serviceMock.getDefaultModel.mockImplementation(({ query }: { query: { model_type: string } }) =>
Promise.resolve({
data: {
model: query.model_type === 'llm' ? 'echo' : 'embed',
provider: {
provider:
query.model_type === 'llm'
? 'kurokobo/fake_models/fake_models'
: 'langgenius/cohere/cohere',
},
},
}),
)
await user.clear(nameInput)
await user.type(nameInput, 'Updated handbook')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce())
expect(serviceMock.create).toHaveBeenCalledWith({
body: expect.objectContaining({
idempotency_key: '22222222-2222-4222-8222-222222222222',
name: 'Updated handbook',
}),
})
})
it.each([400, 401, 403, 422])(
'unlocks editable fields and rotates the idempotency key after a definitive %s rejection',
async (status) => {
@ -290,11 +388,11 @@ describe('CreateKnowledgePage', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledTimes(2))
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotencyKey).toBe(
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotency_key).toBe(
'11111111-1111-4111-8111-111111111111',
)
expect(serviceMock.create.mock.calls[1]?.[0].body).toMatchObject({
idempotencyKey: '22222222-2222-4222-8222-222222222222',
idempotency_key: '22222222-2222-4222-8222-222222222222',
name: 'Updated handbook',
})
},
@ -319,23 +417,30 @@ describe('CreateKnowledgePage', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledTimes(2))
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotencyKey).toBe(
serviceMock.create.mock.calls[1]?.[0].body.idempotencyKey,
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotency_key).toBe(
serviceMock.create.mock.calls[1]?.[0].body.idempotency_key,
)
},
)
it('safely resumes the permission step after a partial failure', async () => {
it('safely resumes a downstream upload after the control space is created', async () => {
const user = userEvent.setup()
navigationMock.startMode = 'upload'
const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } })
const invalidate = vi.spyOn(queryClient, 'invalidateQueries')
serviceMock.patchPolicy.mockRejectedValueOnce(new Error('policy update unavailable'))
serviceMock.upload.mockRejectedValueOnce(new Error('upload unavailable'))
renderPage(queryClient)
await user.upload(
screen.getByLabelText('dataset.newKnowledge.uploadFiles', {
selector: 'input[type="file"]',
}),
new File(['content'], 'handbook.md', { type: 'text/markdown' }),
)
await fillRequiredFields(user)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
expect(await screen.findByRole('alert')).toHaveTextContent(
'dataset.newKnowledge.permissionUpdateFailed',
'dataset.newKnowledge.documentUploadFailed',
)
expect(invalidate).toHaveBeenCalledWith({
queryKey: ['console', 'knowledgeFs', 'listKnowledgeSpaces'],
@ -346,44 +451,28 @@ describe('CreateKnowledgePage', () => {
await user.type(nameInput, ' changed')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.patchPolicy).toHaveBeenCalledTimes(2))
await waitFor(() => expect(serviceMock.upload).toHaveBeenCalledTimes(2))
expect(serviceMock.create).toHaveBeenCalledOnce()
expect(routerMock.replace).toHaveBeenCalledWith(
'/datasets/new/e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084/sources',
'/datasets/new/e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084/documents',
)
})
it('converges after a permission update succeeds but its response is lost', async () => {
it('converges after an atomic creation response is lost', async () => {
const user = userEvent.setup()
serviceMock.getPolicy
.mockResolvedValueOnce({
id: 'policy-1',
ownerSubjectId: 'user-1',
partialMemberSubjectIds: [],
revision: 4,
visibility: 'only_me',
})
.mockResolvedValueOnce({
id: 'policy-1',
ownerSubjectId: 'user-1',
partialMemberSubjectIds: [],
revision: 5,
visibility: 'all_members',
})
serviceMock.patchPolicy.mockRejectedValueOnce(new Error('response lost'))
serviceMock.create.mockRejectedValueOnce(new Error('response lost'))
renderPage()
await fillRequiredFields(user)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
expect(await screen.findByRole('alert')).toHaveTextContent(
'dataset.newKnowledge.permissionUpdateFailed',
)
expect(await screen.findByRole('alert')).toHaveTextContent('dataset.newKnowledge.createFailed')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(routerMock.replace).toHaveBeenCalledOnce())
expect(serviceMock.create).toHaveBeenCalledOnce()
expect(serviceMock.getPolicy).toHaveBeenCalledTimes(2)
expect(serviceMock.patchPolicy).toHaveBeenCalledOnce()
expect(serviceMock.create).toHaveBeenCalledTimes(2)
expect(serviceMock.create.mock.calls[0]?.[0].body.idempotency_key).toBe(
serviceMock.create.mock.calls[1]?.[0].body.idempotency_key,
)
})
it('keeps every start mode interactive without simulating backend success', async () => {
@ -460,6 +549,21 @@ describe('CreateKnowledgePage', () => {
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' })).toBeDisabled()
})
it('disables upload before creating a space when direct upload is unavailable', () => {
navigationMock.startMode = 'upload'
systemFeaturesStateMock.uploadEnabled = false
renderPage()
expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.startEmpty' })).toBeChecked()
const uploadFiles = screen.getByRole('radio', { name: 'dataset.newKnowledge.uploadFiles' })
expect(uploadFiles).toBeDisabled()
expect(uploadFiles).toHaveAccessibleDescription(
'dataset.newKnowledge.uploadFilesDescription dataset.cornerLabel.unavailable',
)
expect(serviceMock.create).not.toHaveBeenCalled()
})
it('continues from the upload mode after real creation succeeds', async () => {
const user = userEvent.setup()
navigationMock.startMode = 'upload'
@ -483,7 +587,7 @@ describe('CreateKnowledgePage', () => {
)
expect(serviceMock.upload).toHaveBeenCalledWith({
body: { file: expect.objectContaining({ name: 'handbook.md' }) },
params: { id: createdKnowledge.id },
params: { control_space_id: createdKnowledge.control_space_id },
})
})
@ -533,6 +637,12 @@ describe('CreateKnowledgePage', () => {
const maxPages = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })
await user.clear(maxPages)
await user.type(maxPages, '25')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlOptions' }))
expect(
screen.getByText(
'dataset.newKnowledge.includeSubpages: dataset.newKnowledge.booleanFalse · dataset.newKnowledge.maxPages: 25',
),
).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() =>
@ -895,14 +1005,19 @@ describe('CreateKnowledgePage', () => {
it('warns before leaving a partially created knowledge space', async () => {
const user = userEvent.setup()
serviceMock.patchPolicy.mockRejectedValueOnce(new Error('policy update unavailable'))
navigationMock.startMode = 'upload'
serviceMock.upload.mockRejectedValueOnce(new Error('upload unavailable'))
renderPage()
await user.upload(
screen.getByLabelText('dataset.newKnowledge.uploadFiles', {
selector: 'input[type="file"]',
}),
new File(['content'], 'handbook.md', { type: 'text/markdown' }),
)
await fillRequiredFields(user)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
expect(
await screen.findByText('dataset.newKnowledge.permissionUpdateFailed'),
).toBeInTheDocument()
expect(await screen.findByText('dataset.newKnowledge.documentUploadFailed')).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'common.operation.cancel' }))

131
web/features/new-rag/__tests__/create-knowledge-workflow.spec.ts Normal file
View File

@ -0,0 +1,131 @@
import { createKnowledge, isDefinitiveCreationRejection } from '../create-knowledge-workflow'
const serviceMock = vi.hoisted(() => ({
createSpace: vi.fn(),
getDefaultModel: vi.fn(),
}))
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
spaces: {
post: serviceMock.createSpace,
},
},
workspaces: {
current: {
defaultModel: {
get: serviceMock.getDefaultModel,
},
},
},
},
}))
describe('createKnowledge', () => {
beforeEach(() => {
vi.clearAllMocks()
serviceMock.getDefaultModel.mockImplementation(
({ query }: { query: { model_type: 'llm' | 'text-embedding' } }) =>
Promise.resolve({
data: {
model: query.model_type === 'llm' ? 'reasoning-model' : 'embedding-model',
provider: {
provider:
query.model_type === 'llm'
? 'langgenius/openai/openai'
: 'langgenius/cohere/cohere',
},
},
}),
)
serviceMock.createSpace.mockResolvedValue({
control_space_id: 'control-space-1',
operation_id: 'operation-1',
state: 'provisioning',
})
})
it('creates a control space with the new model intent and visibility', async () => {
const onCreated = vi.fn()
await expect(
createKnowledge({
description: 'Product docs',
idempotencyKey: '11111111-1111-4111-8111-111111111111',
name: 'Dify Product Docs',
onCreated,
visibility: 'all_team_members',
}),
).resolves.toEqual({
control_space_id: 'control-space-1',
operation_id: 'operation-1',
state: 'provisioning',
})
expect(serviceMock.createSpace).toHaveBeenCalledWith({
body: {
description: 'Product docs',
embedding: {
model: 'embedding-model',
plugin_id: 'langgenius/cohere',
provider: 'cohere',
},
idempotency_key: '11111111-1111-4111-8111-111111111111',
name: 'Dify Product Docs',
retrieval: {
default_mode: 'fast',
reasoning_model: {
model: 'reasoning-model',
plugin_id: 'langgenius/openai',
provider: 'openai',
},
rerank: { enabled: false },
score_threshold: { enabled: false, stage: 'mode-final' },
top_k: 10,
},
slug: expect.stringMatching(/^dify-product-docs-[a-z0-9]+$/),
visibility: 'all_team_members',
},
})
expect(onCreated).toHaveBeenCalledWith({
control_space_id: 'control-space-1',
operation_id: 'operation-1',
state: 'provisioning',
})
})
it('requires both default models before creating the control space', async () => {
serviceMock.getDefaultModel.mockImplementation(
({ query }: { query: { model_type: 'llm' | 'text-embedding' } }) =>
Promise.resolve(
query.model_type === 'llm'
? {
data: {
model: 'reasoning-model',
provider: { provider: 'langgenius/openai/openai' },
},
}
: { data: null },
),
)
await expect(
createKnowledge({
description: '',
idempotencyKey: '22222222-2222-4222-8222-222222222222',
name: '知识库',
onCreated: vi.fn(),
visibility: 'only_me',
}),
).rejects.toMatchObject({ name: 'KnowledgeCreationError', stage: 'preflight' })
expect(serviceMock.createSpace).not.toHaveBeenCalled()
})
})
describe('isDefinitiveCreationRejection', () => {
it('only treats client authorization and validation failures as definitive', () => {
expect(isDefinitiveCreationRejection({ status: 422 })).toBe(true)
expect(isDefinitiveCreationRejection({ status: 503 })).toBe(false)
})
})

2
web/features/new-rag/__tests__/document-detail-model.spec.ts
View File

@ -2,7 +2,7 @@ import type {
DocumentRevisionChunk,
LogicalDocument,
LogicalDocumentRevision,
} from '@dify/contracts/knowledge-fs/types.gen'
} from '../document-models'
import {
buildDocumentChunkTree,
chunkCharacterCount,

482
web/features/new-rag/__tests__/document-detail-page.spec.tsx
View File

@ -1,19 +1,30 @@
import type {
BulkDocumentReindexResult,
DocumentProcessingTask,
DocumentRevisionChunk,
LogicalDocument,
LogicalDocumentRevision,
} from '@dify/contracts/knowledge-fs/types.gen'
} from '../document-models'
import { act, fireEvent, screen, waitFor, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import copy from 'copy-to-clipboard'
import { renderWithNuqs as render } from '@/test/nuqs-testing'
import { DocumentDetailPage } from '../document-detail-page'
type BulkDocumentReindexResult = {
bulkJobId: string
items: Array<{
asset?: unknown
compilationJob?: unknown
documentId?: string
status: 'not_found' | 'queued'
statusUrl?: string
}>
total: number
}
type InfiniteOptions = {
enabled?: boolean
getNextPageParam: (lastPage: { nextCursor?: string }) => string | undefined
getNextPageParam: (lastPage: { next_cursor?: string | null }) => string | null | undefined
input: (pageParam: string | null) => unknown
initialPageParam: string | null
queryKind: 'chunks' | 'revisions' | 'tasks'
@ -31,18 +42,6 @@ const documentQuery = vi.hoisted(() => ({
refetch: vi.fn(),
}))
const taskSnapshotQuery = vi.hoisted(() => ({
data: undefined as DocumentProcessingTask | undefined,
error: null as unknown,
refetch: vi.fn(),
}))
const submissionTasksQuery = vi.hoisted(() => ({
data: undefined as { items: DocumentProcessingTask[] } | undefined,
error: null as unknown,
refetch: vi.fn(),
}))
const revisionsQuery = vi.hoisted(() => ({
data: undefined as
| { pages: Array<{ items: LogicalDocumentRevision[]; nextCursor?: string }> }
@ -92,10 +91,61 @@ const reindexMutation = vi.hoisted(() => ({ mutateAsync: vi.fn() }))
const queryClient = vi.hoisted(() => ({
invalidateQueries: vi.fn(),
removeQueries: vi.fn(),
setQueryData: vi.fn(),
}))
const toastState = vi.hoisted(() => ({ error: vi.fn(), info: vi.fn(), success: vi.fn() }))
const virtualizerState = vi.hoisted(() => ({ scrollToIndex: vi.fn() }))
const revisionApiResponse = vi.hoisted(
() => (revision: Exclude<LogicalDocumentRevision, null>) => ({
activated_at: revision.activatedAt ?? null,
content_hash: revision.contentHash,
created_at: revision.createdAt,
document_asset_id: revision.documentAssetId,
document_asset_version: revision.documentAssetVersion,
document_id: revision.documentId,
knowledge_space_id: revision.knowledgeSpaceId,
mime_type: revision.mimeType,
revision: revision.revision,
size_bytes: revision.sizeBytes,
state: revision.state,
}),
)
const chunkApiResponse = vi.hoisted(() => (item: DocumentRevisionChunk) => ({
created_at: item.createdAt,
document_id: item.documentId,
document_revision: item.documentRevision,
enabled: item.enabled,
id: item.id,
knowledge_space_id: item.knowledgeSpaceId,
ordinal: item.ordinal,
parent_chunk_id: item.parentChunkId ?? null,
text: item.text,
token_count: item.tokenCount,
user_metadata: item.userMetadata,
}))
const taskApiResponse = vi.hoisted(() => (item: DocumentProcessingTask) => ({
can_cancel: item.canCancel ?? true,
can_retry: item.canRetry ?? item.state === 'failed',
completed_at: item.completedAt ?? null,
created_at: item.createdAt,
document_id: item.documentId,
document_revision: item.documentRevision,
error_code: item.errorCode ?? null,
error_message: item.errorMessage ?? null,
id: item.id,
knowledge_space_id: item.knowledgeSpaceId,
operation: item.operation ?? 'document_processing',
progress_percent: item.progressPercent,
state:
item.state === 'succeeded'
? 'completed'
: item.state === 'dispatch_pending'
? 'queued'
: item.state === 'superseded'
? 'canceled'
: item.state,
task_kind: item.taskKind ?? 'document',
updated_at: item.updatedAt,
}))
const documentOptions = vi.hoisted(() =>
vi.fn((options: object) => ({
...options,
@ -103,23 +153,6 @@ const documentOptions = vi.hoisted(() =>
queryKind: 'document',
})),
)
const taskSnapshotOptions = vi.hoisted(() =>
vi.fn((options: object) => ({ ...options, queryKind: 'task-snapshot' })),
)
const documentSubmissionTasksOptions = vi.hoisted(() =>
vi.fn((options: object) => ({
...options,
queryKey: ['knowledge-fs', 'submission-tasks', 'space-1', 'document-1'],
queryKind: 'submission-tasks',
})),
)
const workspaceSubmissionTasksOptions = vi.hoisted(() =>
vi.fn((options: object) => ({
...options,
queryKey: ['knowledge-fs', 'workspace-submission-tasks', 'space-1'],
queryKind: 'submission-tasks',
})),
)
const revisionsOptions = vi.hoisted(() =>
vi.fn((options: Omit<InfiniteOptions, 'queryKind'>) => ({
...options,
@ -141,13 +174,6 @@ const documentTasksOptions = vi.hoisted(() =>
queryKind: 'tasks',
})),
)
const workspaceTasksOptions = vi.hoisted(() =>
vi.fn((options: Omit<InfiniteOptions, 'queryKind'>) => ({
...options,
queryKey: ['knowledge-fs', 'workspace-tasks', 'space-1'],
queryKind: 'tasks',
})),
)
vi.mock('jotai', async (importOriginal) => {
const original = await importOriginal<typeof import('jotai')>()
@ -197,22 +223,51 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
return {
...original,
useInfiniteQuery: (options: InfiniteOptions) => {
if (options.queryKind === 'revisions') return revisionsQuery
if (options.queryKind === 'revisions')
return {
...revisionsQuery,
data: revisionsQuery.data
? {
pages: revisionsQuery.data.pages.map((page) => ({
data: page.items.flatMap((revision) =>
revision ? [revisionApiResponse(revision)] : [],
),
next_cursor: page.nextCursor ?? null,
})),
}
: undefined,
}
if (
options.queryKind === 'chunks' ||
('queryKey' in options &&
Array.isArray(options.queryKey) &&
options.queryKey.includes('chunks'))
)
return chunksQuery
return tasksQuery
return {
...chunksQuery,
data: chunksQuery.data
? {
pages: chunksQuery.data.pages.map((page) => ({
data: page.items.map(chunkApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}
: undefined,
}
return {
...tasksQuery,
data: tasksQuery.data
? {
pages: tasksQuery.data.pages.map((page) => ({
data: page.items.map(taskApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}
: undefined,
}
},
useMutation: () => reindexMutation,
useQuery: (options: { queryKind?: string }) => {
if (options.queryKind === 'task-snapshot') return taskSnapshotQuery
if (options.queryKind === 'submission-tasks') return submissionTasksQuery
return documentQuery
},
useQuery: () => documentQuery,
useQueryClient: () => queryClient,
}
})
@ -220,33 +275,46 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleQuery: {
knowledgeFs: {
getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions: {
infiniteOptions: revisionsOptions,
key: () => ['knowledge-fs', 'revisions'],
},
getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks: {
infiniteOptions: chunksOptions,
key: () => ['knowledge-fs', 'chunks'],
},
getKnowledgeSpacesByIdLogicalDocumentsByDocumentId: {
queryOptions: documentOptions,
key: () => ['knowledge-fs', 'document'],
},
getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId: {
queryOptions: taskSnapshotOptions,
},
getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasks: {
infiniteOptions: documentTasksOptions,
key: () => ['knowledge-fs', 'tasks'],
queryOptions: documentSubmissionTasksOptions,
},
getKnowledgeSpacesByIdProcessingTasks: {
infiniteOptions: workspaceTasksOptions,
key: () => ['knowledge-fs', 'workspace-tasks'],
queryOptions: workspaceSubmissionTasksOptions,
},
postKnowledgeSpacesByIdDocumentsBulkReindex: {
mutationOptions: () => ({}),
spaces: {
byControlSpaceId: {
backgroundTasks: {
get: {
infiniteOptions: documentTasksOptions,
key: () => ['knowledge-fs', 'tasks'],
},
},
documents: {
byDocumentId: {
revisions: {
byRevision: {
chunks: {
get: {
infiniteOptions: chunksOptions,
key: () => ['knowledge-fs', 'chunks'],
},
},
},
get: {
infiniteOptions: revisionsOptions,
key: () => ['knowledge-fs', 'revisions'],
},
},
},
reindex: {
post: {
mutationOptions: () => ({}),
},
},
},
logicalDocuments: {
byDocumentId: {
get: {
queryOptions: documentOptions,
key: () => ['knowledge-fs', 'document'],
},
},
},
},
},
},
},
@ -363,10 +431,6 @@ describe('DocumentDetailPage', () => {
tasksQuery.isFetchNextPageError = false
tasksQuery.isFetchingNextPage = false
tasksQuery.isPending = false
taskSnapshotQuery.data = undefined
taskSnapshotQuery.error = null
submissionTasksQuery.data = undefined
submissionTasksQuery.error = null
permissionState.refresh.mockResolvedValue({
data: { dataset: { default_permission_keys: ['dataset.acl.edit'] } },
error: null,
@ -380,24 +444,28 @@ describe('DocumentDetailPage', () => {
expect(documentOptions).toHaveBeenCalledWith(
expect.objectContaining({
input: { params: { documentId: 'document-1', id: 'space-1' } },
input: {
params: { control_space_id: 'space-1', document_id: 'document-1' },
},
retry: expect.any(Function),
}),
)
expect(infiniteInput(revisionsOptions.mock.lastCall?.[0])(null)).toEqual({
params: { documentId: 'document-1', id: 'space-1' },
query: { limit: 50 },
params: { control_space_id: 'space-1', document_id: 'document-1' },
query: {},
})
expect(infiniteInput(chunksOptions.mock.lastCall?.[0])('next')).toEqual({
params: { documentId: 'document-1', id: 'space-1', revision: 3 },
query: { cursor: 'next', limit: 100 },
params: {
control_space_id: 'space-1',
document_id: 'document-1',
revision: 3,
},
query: { cursor: 'next' },
})
expect(infiniteInput(documentTasksOptions.mock.lastCall?.[0])(null)).toEqual({
params: { documentId: 'document-1', id: 'space-1' },
params: { control_space_id: 'space-1' },
query: { limit: 100 },
})
expect(workspaceTasksOptions).not.toHaveBeenCalled()
expect(workspaceSubmissionTasksOptions).not.toHaveBeenCalled()
})
it('does not construct a chunks request while the document is loading', () => {
@ -602,8 +670,12 @@ describe('DocumentDetailPage', () => {
screen.getByRole('combobox', { name: 'dataset.newKnowledge.documentRevision' }),
).toHaveTextContent('v2')
expect(infiniteInput(chunksOptions.mock.lastCall?.[0])(null)).toEqual({
params: { documentId: 'document-1', id: 'space-1', revision: 2 },
query: { limit: 100 },
params: {
control_space_id: 'space-1',
document_id: 'document-1',
revision: 2,
},
query: {},
})
})
@ -684,19 +756,31 @@ describe('DocumentDetailPage', () => {
).toBeEnabled()
})
it('polls only the discovered active task through the single-task contract', () => {
it('polls active work through the unified background-task contract', () => {
tasksQuery.data = { pages: [{ items: [task({ state: 'running' })] }] }
render(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
expect(taskSnapshotOptions).toHaveBeenCalledWith(
expect.objectContaining({
enabled: true,
input: {
params: { documentId: 'document-1', id: 'space-1', taskId: 'task-1' },
const taskOptions = documentTasksOptions.mock.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: {
data?: {
pages: Array<{
data: Array<ReturnType<typeof taskApiResponse>>
next_cursor: string | null
}>
}
}
}) => number | false
}
expect(
taskOptions.refetchInterval({
state: {
data: {
pages: [{ data: [taskApiResponse(task({ state: 'running' }))], next_cursor: null }],
},
},
refetchInterval: expect.any(Function),
}),
)
).toBe(5000)
expect(tasksQuery.refetch).not.toHaveBeenCalled()
})
@ -705,7 +789,7 @@ describe('DocumentDetailPage', () => {
const rendered = render(
<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />,
)
taskSnapshotQuery.data = task({ state: 'succeeded' })
tasksQuery.data = { pages: [{ items: [task({ state: 'succeeded' })] }] }
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
await waitFor(() => expect(queryClient.invalidateQueries).toHaveBeenCalledTimes(4))
@ -849,7 +933,7 @@ describe('DocumentDetailPage', () => {
expect(reindexMutation.mutateAsync).toHaveBeenCalledWith({
body: { documentIds: ['document-1'] },
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
})
await waitFor(() => expect(queryClient.invalidateQueries).toHaveBeenCalled())
@ -995,25 +1079,48 @@ describe('DocumentDetailPage', () => {
await user.click(button)
expect(reindexMutation.mutateAsync).toHaveBeenCalledOnce()
const discoveryOptions = documentSubmissionTasksOptions.mock.lastCall?.[0] as unknown as {
const discoveryOptions = documentTasksOptions.mock.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: { data?: { items: DocumentProcessingTask[] }; error?: unknown }
state: {
data?: {
pages: Array<{
data: Array<ReturnType<typeof taskApiResponse>>
next_cursor: string | null
}>
}
}
}) => number | false
retry: (failureCount: number, error: unknown) => boolean
}
expect(discoveryOptions.refetchInterval({ state: { data: { items: [] } } })).toBe(2000)
expect(
discoveryOptions.refetchInterval({
state: { data: { items: [task({ documentRevision: 5 })] } },
state: {
data: {
pages: [
{
data: [
taskApiResponse(task({ documentRevision: 4, id: 'old-failed', state: 'failed' })),
],
next_cursor: null,
},
],
},
},
}),
).toBe(false)
).toBe(2000)
expect(
discoveryOptions.refetchInterval({
state: { data: { items: [] }, error: { status: 403 } },
state: {
data: {
pages: [
{
data: [taskApiResponse(task({ documentRevision: 5 }))],
next_cursor: null,
},
],
},
},
}),
).toBe(false)
expect(discoveryOptions.retry(0, { status: 403 })).toBe(false)
expect(discoveryOptions.retry(0, { status: 404 })).toBe(false)
).toBe(5000)
})
it('keeps submission protection while a delayed status recheck is unresolved', async () => {
@ -1028,9 +1135,7 @@ describe('DocumentDetailPage', () => {
}),
)
try {
const rendered = render(
<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />,
)
render(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
const reindexButton = screen.getByRole('button', {
name: 'dataset.newKnowledge.reindexDocument',
})
@ -1039,21 +1144,10 @@ describe('DocumentDetailPage', () => {
await Promise.resolve()
await Promise.resolve()
})
submissionTasksQuery.error = new Error('submission discovery failed')
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
await act(() => vi.advanceTimersByTimeAsync(30000))
const alert = screen.getByRole('alert')
expect(alert).toHaveTextContent('dataset.newKnowledge.documentReindexConfirmationDelayed')
const timedOutDiscoveryOptions = documentSubmissionTasksOptions.mock
.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: { data?: { items: DocumentProcessingTask[] }; error?: unknown }
}) => number | false
}
expect(timedOutDiscoveryOptions.refetchInterval({ state: { data: { items: [] } } })).toBe(
false,
)
fireEvent.click(
within(alert).getByRole('button', {
name: 'dataset.newKnowledge.checkReindexStatus',
@ -1061,7 +1155,7 @@ describe('DocumentDetailPage', () => {
)
expect(reindexButton).toHaveAttribute('data-disabled')
expect(reindexMutation.mutateAsync).toHaveBeenCalledOnce()
expect(submissionTasksQuery.refetch).toHaveBeenCalledOnce()
expect(tasksQuery.refetch).toHaveBeenCalledOnce()
await act(async () => {
finishTaskRefresh?.()
@ -1081,14 +1175,30 @@ describe('DocumentDetailPage', () => {
})
expect(reindexMutation.mutateAsync).toHaveBeenCalledTimes(2)
expect(screen.getByRole('heading', { level: 1 })).toHaveFocus()
const discoveryOptions = documentSubmissionTasksOptions.mock.lastCall?.[0] as unknown as {
const discoveryOptions = documentTasksOptions.mock.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: { data?: { items: DocumentProcessingTask[] } }
state: {
data?: {
pages: Array<{
data: Array<ReturnType<typeof taskApiResponse>>
next_cursor: string | null
}>
}
}
}) => number | false
}
expect(
discoveryOptions.refetchInterval({
state: { data: { items: [task({ documentRevision: 4, state: 'failed' })] } },
state: {
data: {
pages: [
{
data: [taskApiResponse(task({ documentRevision: 4, state: 'failed' }))],
next_cursor: null,
},
],
},
},
}),
).toBe(2000)
} finally {
@ -1134,22 +1244,29 @@ describe('DocumentDetailPage', () => {
await Promise.resolve()
})
const discoveryOptions = documentSubmissionTasksOptions.mock.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: { data?: { items: DocumentProcessingTask[] }; error?: unknown }
}) => number | false
tasksQuery.data = {
pages: [
{
items: [
task({
documentRevision: 5,
id: 'late-first',
state: 'succeeded',
}),
],
},
],
}
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
expect(
discoveryOptions.refetchInterval({
state: { data: { items: [task({ documentRevision: 5, id: 'late-first' })] } },
}),
).toBe(2000)
screen.getByRole('button', { name: 'dataset.newKnowledge.reindexDocument' }),
).toHaveAttribute('data-disabled')
} finally {
vi.useRealTimers()
}
})
it('stops first-page submission discovery when task history observes the new task', async () => {
it('uses active-task polling after the unified task list observes the new task', async () => {
const user = userEvent.setup()
const rendered = render(
<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />,
@ -1162,61 +1279,82 @@ describe('DocumentDetailPage', () => {
}
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
expect(documentSubmissionTasksOptions.mock.lastCall?.[0]).toMatchObject({ enabled: false })
const taskOptions = documentTasksOptions.mock.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: {
data?: {
pages: Array<{
data: Array<ReturnType<typeof taskApiResponse>>
next_cursor: string | null
}>
}
}
}) => number | false
}
expect(
taskOptions.refetchInterval({
state: {
data: {
pages: [
{
data: [taskApiResponse(task({ documentRevision: 4, state: 'running' }))],
next_cursor: null,
},
],
},
},
}),
).toBe(5000)
})
it('stops snapshot polling and distrusts stale active task data after 403 or 404', async () => {
tasksQuery.data = { pages: [{ items: [task({ state: 'running' })] }] }
taskSnapshotQuery.error = { status: 404 }
it('surfaces unified task-list authorization failures and blocks re-indexing', () => {
tasksQuery.data = undefined
tasksQuery.error = { status: 403 }
const rendered = render(
<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />,
)
expect(screen.queryByRole('status')).toBeNull()
const snapshotOptions = taskSnapshotOptions.mock.lastCall?.[0] as {
refetchInterval: (query: {
state: { data?: DocumentProcessingTask; error?: unknown }
}) => number | false
}
expect(snapshotOptions.refetchInterval({ state: { error: { status: 404 } } })).toBe(false)
await waitFor(() => expect(queryClient.invalidateQueries).toHaveBeenCalled())
queryClient.invalidateQueries.mockClear()
taskSnapshotQuery.error = { status: 403 }
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
expect(screen.getByRole('alert')).toHaveTextContent(
'dataset.newKnowledge.tasksErrorDescription',
)
expect(
screen.getByRole('button', { name: 'dataset.newKnowledge.reindexDocument' }),
).toHaveAttribute('data-disabled')
const taskOptions = documentTasksOptions.mock.lastCall?.[0] as unknown as {
refetchInterval: (query: {
state: {
data?: {
pages: Array<{
data: Array<ReturnType<typeof taskApiResponse>>
next_cursor: string | null
}>
}
error?: unknown
}
}) => number | false
}
expect(taskOptions.refetchInterval({ state: { error: { status: 403 } } })).toBe(false)
tasksQuery.error = { status: 404 }
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
expect(screen.getByRole('alert')).toHaveTextContent(
'dataset.newKnowledge.tasksErrorDescription',
)
})
it('clears both task caches when a submission-discovered task snapshot returns 404', async () => {
submissionTasksQuery.data = { items: [task({ state: 'running' })] }
taskSnapshotQuery.error = { status: 404 }
it('recovers task state directly from the unified task list', () => {
tasksQuery.data = undefined
tasksQuery.error = { status: 404 }
const rendered = render(
<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />,
)
await waitFor(() => expect(queryClient.invalidateQueries).toHaveBeenCalledTimes(2))
expect(queryClient.invalidateQueries).toHaveBeenCalledWith({
queryKey: ['knowledge-fs', 'tasks', 'space-1', 'document-1'],
})
expect(queryClient.invalidateQueries).toHaveBeenCalledWith({
queryKey: ['knowledge-fs', 'submission-tasks', 'space-1', 'document-1'],
})
expect(queryClient.setQueryData).toHaveBeenCalledTimes(2)
submissionTasksQuery.data = { items: [task({ id: 'missing-task-2', state: 'running' })] }
tasksQuery.error = null
tasksQuery.data = { pages: [{ items: [task({ state: 'running' })] }] }
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
await waitFor(() => expect(queryClient.invalidateQueries).toHaveBeenCalledTimes(4))
expect(queryClient.setQueryData).toHaveBeenCalledTimes(4)
submissionTasksQuery.data = { items: [task({ state: 'running' })] }
rendered.rerender(<DocumentDetailPage documentId="document-1" knowledgeSpaceId="space-1" />)
await waitFor(() => expect(queryClient.setQueryData).toHaveBeenCalledTimes(4))
expect(screen.getByRole('status')).toHaveTextContent(
'dataset.newKnowledge.documentReindexProgress:{"progress":"45"}',
)
})
it('refreshes stale detail and task-list caches for a newer terminal task on revisit', async () => {

5
web/features/new-rag/__tests__/document-model.spec.ts
View File

@ -1,7 +1,4 @@
import type {
DocumentProcessingTask,
LogicalDocument,
} from '@dify/contracts/knowledge-fs/types.gen'
import type { DocumentProcessingTask, LogicalDocument } from '../document-models'
import {
documentDisplayStatus,
newestTaskByDocument,

416
web/features/new-rag/__tests__/documents-page.spec.tsx
View File

@ -1,8 +1,5 @@
import type {
DocumentProcessingTask,
LogicalDocument,
Source,
} from '@dify/contracts/knowledge-fs/types.gen'
import type { DocumentProcessingTask, LogicalDocument } from '../document-models'
import type { Source } from '../source-models'
import { hashKey } from '@tanstack/react-query'
import { act, fireEvent, screen, waitFor, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
@ -12,7 +9,7 @@ import { TaskEventObserver } from '../task-event-observer'
type InfiniteOptions = {
enabled?: boolean
getNextPageParam: (lastPage: { nextCursor?: string }) => string | undefined
getNextPageParam: (lastPage: { next_cursor?: string | null }) => string | null | undefined
input: (pageParam: string | null) => unknown
initialPageParam: string | null
queryKind: 'documents' | 'sources' | 'tasks'
@ -93,6 +90,12 @@ const queryClient = vi.hoisted(() => ({
}))
const streamProcessingTaskEvents = vi.hoisted(() => vi.fn())
const getTaskSnapshot = vi.hoisted(() => vi.fn())
const taskSnapshotRequestState = vi.hoisted(() => ({ index: 0 }))
const rawQueryDataCache = vi.hoisted(() => ({
documents: new WeakMap<object, object>(),
sources: new WeakMap<object, object>(),
tasks: new WeakMap<object, object>(),
}))
const permissionStateMock = vi.hoisted(() => ({
datasetAtom: Symbol('datasetDefaultPermissionKeysAtom'),
datasetKeys: ['dataset.acl.edit'],
@ -107,12 +110,84 @@ const permissionStateMock = vi.hoisted(() => ({
refreshAfterDenial: vi.fn(),
refreshAfterDenialAtom: Symbol('refreshWorkspacePermissionKeysAfterMutationDenialAtom'),
}))
const systemFeaturesStateMock = vi.hoisted(() => ({
atom: Symbol('knowledgeFsUploadEnabledAtom'),
uploadEnabled: true,
}))
const toastMock = vi.hoisted(() => ({
error: vi.fn(),
info: vi.fn(),
success: vi.fn(),
warning: vi.fn(),
}))
const revisionApiResponse = vi.hoisted(
() => (revision: NonNullable<LogicalDocument['active']>) => ({
activated_at: revision.activatedAt ?? null,
content_hash: revision.contentHash,
created_at: revision.createdAt,
document_asset_id: revision.documentAssetId,
document_asset_version: revision.documentAssetVersion,
document_id: revision.documentId,
knowledge_space_id: revision.knowledgeSpaceId,
mime_type: revision.mimeType,
revision: revision.revision,
size_bytes: revision.sizeBytes,
state: revision.state,
}),
)
const documentApiResponse = vi.hoisted(() => (item: LogicalDocument) => ({
active: item.active ? revisionApiResponse(item.active) : null,
active_revision: item.activeRevision ?? null,
created_at: item.createdAt,
id: item.id,
knowledge_space_id: item.knowledgeSpaceId,
provider_item_id: item.providerItemId ?? null,
row_version: item.rowVersion,
source_id: item.sourceId ?? null,
status: item.status,
title: item.title,
updated_at: item.updatedAt,
user_metadata: item.userMetadata,
}))
const taskApiResponse = vi.hoisted(() => (item: DocumentProcessingTask) => ({
can_cancel: item.canCancel ?? true,
can_retry: item.canRetry ?? item.state === 'failed',
completed_at: item.completedAt ?? null,
created_at: item.createdAt,
document_id: item.documentId,
document_revision: item.documentRevision,
error_code: item.errorCode ?? null,
error_message: item.errorMessage ?? null,
id: item.id,
knowledge_space_id: item.knowledgeSpaceId,
operation: item.operation ?? 'document_processing',
progress_percent: item.progressPercent,
state:
item.state === 'succeeded'
? 'completed'
: item.state === 'dispatch_pending'
? 'queued'
: item.state === 'superseded'
? 'canceled'
: item.state,
task_kind: item.taskKind ?? 'document',
updated_at: item.updatedAt,
}))
const sourceApiResponse = vi.hoisted(() => (item: Source) => ({
connection_id: item.connectionId ?? null,
created_at: item.createdAt,
credential_configured: item.credentialConfigured ?? null,
id: item.id,
knowledge_space_id: item.knowledgeSpaceId,
metadata: item.metadata,
name: item.name,
permission_scope: item.permissionScope ?? [],
status: item.status,
type: item.type,
updated_at: item.updatedAt,
uri: item.uri,
version: item.version ?? 1,
}))
vi.mock('@/context/permission-state', () => ({
datasetDefaultPermissionKeysAtom: permissionStateMock.datasetAtom,
@ -123,6 +198,10 @@ vi.mock('@/context/permission-state', () => ({
workspacePermissionKeysLoadingAtom: permissionStateMock.loadingAtom,
}))
vi.mock('@/context/system-features-state', () => ({
knowledgeFsUploadEnabledAtom: systemFeaturesStateMock.atom,
}))
vi.mock('jotai', async (importOriginal) => {
const original = await importOriginal<typeof import('jotai')>()
return {
@ -132,6 +211,7 @@ vi.mock('jotai', async (importOriginal) => {
if (atom === permissionStateMock.errorAtom) return permissionStateMock.error
if (atom === permissionStateMock.fetchingAtom) return permissionStateMock.fetching
if (atom === permissionStateMock.loadingAtom) return permissionStateMock.loading
if (atom === systemFeaturesStateMock.atom) return systemFeaturesStateMock.uploadEnabled
return original.useAtomValue(atom as Parameters<typeof original.useAtomValue>[0])
},
useSetAtom: (atom: unknown) =>
@ -159,21 +239,100 @@ const sourcesInfiniteOptions = vi.hoisted(() =>
vi.fn((options: Omit<InfiniteOptions, 'queryKind'>) => ({ ...options, queryKind: 'sources' })),
)
function rawDocumentQueryData(data: NonNullable<typeof documentsQuery.data>): {
pages: Array<{ data: Array<ReturnType<typeof documentApiResponse>>; next_cursor: string | null }>
} {
const cached = rawQueryDataCache.documents.get(data)
if (cached) return cached as ReturnType<typeof rawDocumentQueryData>
const mapped = {
pages: data.pages.map((page) => ({
data: page.items.map(documentApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}
rawQueryDataCache.documents.set(data, mapped)
return mapped
}
function rawSourceQueryData(data: NonNullable<typeof sourcesQuery.data>): {
pages: Array<{ data: Array<ReturnType<typeof sourceApiResponse>>; next_cursor: string | null }>
} {
const cached = rawQueryDataCache.sources.get(data)
if (cached) return cached as ReturnType<typeof rawSourceQueryData>
const mapped = {
pages: data.pages.map((page) => ({
data: page.items.map(sourceApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}
rawQueryDataCache.sources.set(data, mapped)
return mapped
}
function rawTaskQueryData(data: NonNullable<typeof tasksQuery.data>): {
pages: Array<{ data: Array<ReturnType<typeof taskApiResponse>>; next_cursor: string | null }>
} {
const cached = rawQueryDataCache.tasks.get(data)
if (cached) return cached as ReturnType<typeof rawTaskQueryData>
const mapped = {
pages: data.pages.map((page) => ({
data: page.items.map(taskApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}
rawQueryDataCache.tasks.set(data, mapped)
return mapped
}
vi.mock('@tanstack/react-query', async (importOriginal) => {
const original = await importOriginal<typeof import('@tanstack/react-query')>()
return {
...original,
useInfiniteQuery: (options: InfiniteOptions) => {
if (options.queryKind === 'documents') return documentsQuery
if (options.queryKind === 'sources') return sourcesQuery
return tasksQuery
if (options.queryKind === 'documents')
return {
...documentsQuery,
data: documentsQuery.data ? rawDocumentQueryData(documentsQuery.data) : undefined,
}
if (options.queryKind === 'sources')
return {
...sourcesQuery,
data: sourcesQuery.data ? rawSourceQueryData(sourcesQuery.data) : undefined,
}
return {
...tasksQuery,
data: tasksQuery.data ? rawTaskQueryData(tasksQuery.data) : undefined,
}
},
useMutation: (options: {
mutationKind: 'bulk-upload' | 'cancel' | 'reindex' | 'retry' | 'upload'
mutationFn?: (input: DocumentProcessingTask) => Promise<DocumentProcessingTask>
mutationKind?: 'bulk-upload' | 'cancel' | 'reindex' | 'retry' | 'upload'
}) => {
if (options.mutationFn)
return {
mutateAsync: options.mutationFn,
}
if (options.mutationKind === 'cancel') return cancelMutation
if (options.mutationKind === 'retry') return retryMutation
if (options.mutationKind === 'reindex') return reindexMutation
if (options.mutationKind === 'reindex')
return {
mutateAsync: async (input: unknown) => {
const result = await reindexMutation.mutateAsync(input)
return {
...result,
items: result.items.map(
(item: {
documentId?: string
document_id?: string
status: 'not_found' | 'queued'
}) => ({
...item,
document_id: item.document_id ?? item.documentId ?? null,
}),
),
}
},
}
if (options.mutationKind === 'bulk-upload') return bulkUploadMutation
return uploadMutation
},
@ -184,43 +343,100 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId: getTaskSnapshot,
spaces: {
byControlSpaceId: {
backgroundTasks: {
byTaskKind: {
byTaskId: {
cancel: {
post: async (input: unknown) =>
taskApiResponse(await cancelMutation.mutateAsync(input)),
},
retry: {
post: async (input: unknown) =>
taskApiResponse(await retryMutation.mutateAsync(input)),
},
},
},
get: async (input: unknown, options?: unknown) => {
const allTasks = tasksQuery.data?.pages.flatMap((page) => page.items) ?? []
const requestedTask = allTasks[taskSnapshotRequestState.index % allTasks.length]
taskSnapshotRequestState.index += 1
const snapshot = await getTaskSnapshot(
requestedTask
? {
params: {
documentId: requestedTask.documentId,
id: requestedTask.knowledgeSpaceId,
taskId: requestedTask.id,
},
}
: input,
options,
)
return {
data: snapshot ? [taskApiResponse(snapshot)] : [],
next_cursor: null,
}
},
},
},
},
},
},
consoleQuery: {
knowledgeFs: {
deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId: {
mutationOptions: () => ({ mutationKind: 'cancel' }),
},
getKnowledgeSpacesByIdLogicalDocuments: {
infiniteOptions: documentsInfiniteOptions,
key: () => ['knowledge-fs', 'documents'],
},
getKnowledgeSpacesByIdProcessingTasks: {
infiniteOptions: tasksInfiniteOptions,
key: () => ['knowledge-fs', 'tasks'],
},
getKnowledgeSpacesByIdSources: {
infiniteOptions: sourcesInfiniteOptions,
key: () => ['knowledge-fs', 'sources'],
},
postKnowledgeSpacesByIdDocuments: {
mutationOptions: () => ({ mutationKind: 'upload' }),
},
postKnowledgeSpacesByIdDocumentsBulk: {
mutationOptions: () => ({ mutationKind: 'bulk-upload' }),
},
postKnowledgeSpacesByIdDocumentsBulkReindex: {
mutationOptions: () => ({ mutationKind: 'reindex' }),
},
postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry: {
mutationOptions: () => ({ mutationKind: 'retry' }),
spaces: {
byControlSpaceId: {
backgroundTasks: {
get: {
infiniteOptions: tasksInfiniteOptions,
key: () => ['knowledge-fs', 'tasks'],
},
},
documents: {
reindex: {
post: {
mutationOptions: () => ({ mutationKind: 'reindex' }),
},
},
},
logicalDocuments: {
get: {
infiniteOptions: documentsInfiniteOptions,
key: () => ['knowledge-fs', 'documents'],
},
},
sources: {
get: {
infiniteOptions: sourcesInfiniteOptions,
key: () => ['knowledge-fs', 'sources'],
},
},
},
},
},
},
}))
vi.mock('../services/processing-task-events', () => ({ streamProcessingTaskEvents }))
vi.mock('../knowledge-fs-upload', () => ({
uploadKnowledgeFsDocuments: async (
knowledgeSpaceId: string,
uploads: Array<{ file: File; id: string }>,
) => {
const files = uploads.map(({ file }) => file)
if (files.length === 1)
return uploadMutation.mutateAsync({
body: { file: files[0] },
params: { control_space_id: knowledgeSpaceId },
})
return bulkUploadMutation.mutateAsync({
body: { files },
params: { control_space_id: knowledgeSpaceId },
})
},
}))
const document = (overrides: Partial<LogicalDocument> = {}): LogicalDocument => ({
active: {
@ -313,7 +529,9 @@ const source = (overrides: Partial<Source> = {}): Source => ({
describe('DocumentsPage', () => {
beforeEach(() => {
vi.clearAllMocks()
systemFeaturesStateMock.uploadEnabled = true
queryCacheListeners.clear()
taskSnapshotRequestState.index = 0
queryClient.cancelQueries.mockResolvedValue(undefined)
queryClient.invalidateQueries.mockResolvedValue(undefined)
documentsQuery.data = { pages: [{ items: [] }] }
@ -424,24 +642,24 @@ describe('DocumentsPage', () => {
const taskOptions = tasksInfiniteOptions.mock.lastCall?.[0]
const sourceOptions = sourcesInfiniteOptions.mock.lastCall?.[0]
expect(documentOptions?.input(null)).toEqual({
params: { id: 'space-1' },
query: { limit: 50 },
params: { control_space_id: 'space-1' },
query: {},
})
expect(documentOptions?.input('next')).toEqual({
params: { id: 'space-1' },
query: { cursor: 'next', limit: 50 },
params: { control_space_id: 'space-1' },
query: { cursor: 'next' },
})
expect(documentOptions?.getNextPageParam({ nextCursor: 'next' })).toBe('next')
expect(documentOptions?.getNextPageParam({ next_cursor: 'next' })).toBe('next')
expect(taskOptions?.input(null)).toEqual({
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
query: { limit: 100 },
})
expect(taskOptions?.getNextPageParam({ nextCursor: 'next' })).toBe('next')
expect(taskOptions?.getNextPageParam({ next_cursor: 'next' })).toBe('next')
expect(sourceOptions?.input(null)).toEqual({
params: { id: 'space-1' },
query: { limit: 100 },
params: { control_space_id: 'space-1' },
query: {},
})
expect(sourceOptions?.getNextPageParam({ nextCursor: 'next' })).toBe('next')
expect(sourceOptions?.getNextPageParam({ next_cursor: 'next' })).toBe('next')
expect(screen.getByRole('status', { name: 'appApi.loading' })).toBeInTheDocument()
})
@ -588,6 +806,19 @@ describe('DocumentsPage', () => {
expect(dataTransfer.dropEffect).toBe('copy')
})
it('keeps direct-upload actions unavailable until the deployment is verified', () => {
systemFeaturesStateMock.uploadEnabled = false
render(<DocumentsPage knowledgeSpaceId="space-1" />)
expect(screen.queryByLabelText('dataset.newKnowledge.uploadDocuments')).not.toBeInTheDocument()
const addDocument = screen.getByRole('button', {
name: 'dataset.newKnowledge.addDocument',
})
expect(addDocument).toBeDisabled()
expect(addDocument).toHaveAccessibleDescription('dataset.cornerLabel.unavailable')
})
it('removes the empty-state drop affordance when uploads are unavailable', () => {
permissionStateMock.datasetKeys = ['dataset.acl.readonly']
@ -744,7 +975,7 @@ describe('DocumentsPage', () => {
await user.upload(input, new File(['one'], 'one.md', { type: 'text/markdown' }))
expect(uploadMutation.mutateAsync).toHaveBeenCalledWith({
body: { file: expect.any(File) },
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
})
await user.upload(input, [
@ -753,7 +984,7 @@ describe('DocumentsPage', () => {
])
expect(bulkUploadMutation.mutateAsync).toHaveBeenCalledWith({
body: { files: [expect.any(File), expect.any(File)] },
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
})
expect(queryClient.invalidateQueries).toHaveBeenCalled()
const documentInvalidation = queryClient.invalidateQueries.mock.calls.find(
@ -763,7 +994,7 @@ describe('DocumentsPage', () => {
documentInvalidation?.predicate({
queryKey: [
['console', 'knowledgeFs', 'getKnowledgeSpacesByIdLogicalDocuments'],
{ input: { params: { id: 'space-1' } }, type: 'infinite' },
{ input: { params: { control_space_id: 'space-1' } }, type: 'infinite' },
],
}),
).toBe(true)
@ -771,7 +1002,7 @@ describe('DocumentsPage', () => {
documentInvalidation?.predicate({
queryKey: [
['console', 'knowledgeFs', 'getKnowledgeSpacesByIdLogicalDocuments'],
{ input: { params: { id: 'space-2' } }, type: 'infinite' },
{ input: { params: { control_space_id: 'space-2' } }, type: 'infinite' },
],
}),
).toBe(false)
@ -791,7 +1022,7 @@ describe('DocumentsPage', () => {
await waitFor(() =>
expect(uploadMutation.mutateAsync).toHaveBeenCalledWith({
body: { file: validFile },
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
}),
)
expect(bulkUploadMutation.mutateAsync).not.toHaveBeenCalled()
@ -816,68 +1047,28 @@ describe('DocumentsPage', () => {
)
})
it('reports partial and fully excluded bulk uploads from the contract result', async () => {
it('reports local exclusions and direct upload failures', async () => {
const user = userEvent.setup()
bulkUploadMutation.mutateAsync
.mockResolvedValueOnce({
accepted: 1,
bulkJobId: 'upload-partial',
excluded: 1,
items: [
{
filename: 'too-large.pdf',
index: 1,
mimeType: 'application/pdf',
reason: 'file_too_large',
sizeBytes: 10_000,
status: 'excluded',
},
],
total: 2,
})
.mockResolvedValueOnce({
accepted: 0,
bulkJobId: 'upload-rejected',
excluded: 2,
items: [
{
filename: 'one.md',
index: 0,
mimeType: 'text/markdown',
reason: 'quota_exceeded',
sizeBytes: 3,
status: 'excluded',
},
{
filename: 'two.md',
index: 1,
mimeType: 'text/markdown',
reason: 'quota_exceeded',
sizeBytes: 3,
status: 'excluded',
},
],
total: 2,
})
render(<DocumentsPage knowledgeSpaceId="space-1" />)
const input = screen.getByLabelText('dataset.newKnowledge.uploadDocuments')
const oversizedFile = new File(['large'], 'too-large.pdf', { type: 'application/pdf' })
Object.defineProperty(oversizedFile, 'size', { value: 16 * 1024 * 1024 })
await user.upload(input, [
new File(['one'], 'one.md', { type: 'text/markdown' }),
new File(['large'], 'too-large.pdf', { type: 'application/pdf' }),
oversizedFile,
])
expect(toastMock.warning).toHaveBeenCalledWith(
'dataset.newKnowledge.documentUploadPartial:{"accepted":1,"details":"too-large.pdf (dataset.newKnowledge.documentUploadExclusion.fileSize)","excluded":1}',
)
queryClient.invalidateQueries.mockClear()
bulkUploadMutation.mutateAsync.mockRejectedValueOnce(new Error('quota exceeded'))
await user.upload(input, [
new File(['one'], 'one.md', { type: 'text/markdown' }),
new File(['two'], 'two.md', { type: 'text/markdown' }),
])
expect(toastMock.error).toHaveBeenCalledWith(
'dataset.newKnowledge.documentUploadRejected:{"details":"one.md (dataset.newKnowledge.documentUploadExclusion.quota); two.md (dataset.newKnowledge.documentUploadExclusion.quota)"}',
)
expect(toastMock.error).toHaveBeenCalledWith('dataset.newKnowledge.documentUploadFailed')
expect(queryClient.invalidateQueries).not.toHaveBeenCalled()
})
@ -1080,7 +1271,7 @@ describe('DocumentsPage', () => {
taskCancellation?.predicate({
queryKey: [
['console', 'knowledgeFs', 'getKnowledgeSpacesByIdProcessingTasks'],
{ input: { params: { id: 'space-1' } }, type: 'infinite' },
{ input: { params: { control_space_id: 'space-1' } }, type: 'infinite' },
],
}),
).toBe(true)
@ -1088,7 +1279,7 @@ describe('DocumentsPage', () => {
taskCancellation?.predicate({
queryKey: [
['console', 'knowledgeFs', 'getKnowledgeSpacesByIdProcessingTasks'],
{ input: { params: { id: 'space-2' } }, type: 'infinite' },
{ input: { params: { control_space_id: 'space-2' } }, type: 'infinite' },
],
}),
).toBe(false)
@ -1709,7 +1900,7 @@ describe('DocumentsPage', () => {
expect(reindexMutation.mutateAsync).toHaveBeenCalledOnce()
expect(reindexMutation.mutateAsync).toHaveBeenCalledWith({
body: { documentIds: ['one'] },
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
})
await waitFor(() =>
expect(screen.getByRole('heading', { name: 'dataset.newKnowledge.documents' })).toHaveFocus(),
@ -1941,7 +2132,11 @@ describe('DocumentsPage', () => {
expect(cancelMutation.mutateAsync).toHaveBeenCalledOnce()
expect(cancelMutation.mutateAsync).toHaveBeenCalledWith({
params: { documentId: 'document-1', id: 'space-1', taskId: 'running' },
params: {
control_space_id: 'space-1',
task_id: 'running',
task_kind: 'document',
},
})
await act(async () => resolveCancel?.(task({ id: 'running', state: 'canceled' })))
expect(queryClient.invalidateQueries).toHaveBeenCalled()
@ -3906,7 +4101,20 @@ describe('DocumentsPage', () => {
await act(async () => vi.advanceTimersByTime(5000))
expect(streamProcessingTaskEvents).toHaveBeenCalledTimes(12)
const taskOptions = tasksInfiniteOptions.mock.lastCall?.[0]
expect(taskOptions?.refetchInterval).toBeUndefined()
expect(
taskOptions?.refetchInterval?.({
state: {
data: {
pages: [
{
data: [taskApiResponse(task({ id: 'active-0' }))],
next_cursor: null,
},
],
},
},
}),
).toBe(5000)
expect(
screen.getByRole('button', {
name: 'dataset.newKnowledge.tasksWithAttention:{"count":20}',

154
web/features/new-rag/__tests__/knowledge-fs-upload.spec.ts Normal file
View File

@ -0,0 +1,154 @@
import { uploadKnowledgeFsDocuments } from '../knowledge-fs-upload'
const serviceMock = vi.hoisted(() => ({
getSpace: vi.fn(),
issueCapability: vi.fn(),
smallFile: vi.fn(),
}))
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
spaces: {
byControlSpaceId: {
get: serviceMock.getSpace,
uploadCapabilities: {
post: serviceMock.issueCapability,
},
uploadSessions: {
byUploadSessionId: {
smallFile: {
post: serviceMock.smallFile,
},
},
},
},
},
},
},
}))
describe('uploadKnowledgeFsDocuments', () => {
beforeEach(() => {
vi.clearAllMocks()
serviceMock.getSpace.mockResolvedValue({
knowledge_space_id: 'physical-space-1',
state: 'active',
})
serviceMock.issueCapability.mockResolvedValue({
direct_origin: 'https://knowledge-fs.example',
expires_at: '2026-07-27T12:00:00Z',
operation_id: 'createUploadSession',
token: 'capability-token',
})
serviceMock.smallFile.mockResolvedValue({
session: { id: 'session-1', mode: 'small_fallback', status: 'completed' },
})
vi.spyOn(globalThis.crypto.subtle, 'digest').mockResolvedValue(new Uint8Array(32).buffer)
})
afterEach(() => {
vi.restoreAllMocks()
vi.unstubAllGlobals()
})
it('creates a capability-bound session and uses the Dify small-file fallback', async () => {
const request = vi.fn().mockResolvedValue(
new Response(
JSON.stringify({
session: {
id: 'session-1',
mode: 'small_fallback',
status: 'ready',
},
}),
{
status: 201,
headers: { 'content-type': 'application/json' },
},
),
)
vi.stubGlobal('fetch', request)
const file = new File(['hello'], 'hello.txt', { type: 'text/plain' })
await uploadKnowledgeFsDocuments('control-space-1', [{ file, id: 'upload-1' }])
expect(serviceMock.issueCapability).toHaveBeenCalledWith({
body: { operation_id: 'createUploadSession' },
params: { control_space_id: 'control-space-1' },
})
expect(request).toHaveBeenCalledWith(
'https://knowledge-fs.example/knowledge-spaces/physical-space-1/upload-sessions',
expect.objectContaining({
headers: {
Authorization: 'Bearer capability-token',
'Content-Type': 'application/json',
},
method: 'POST',
}),
)
expect(serviceMock.smallFile).toHaveBeenCalledWith({
body: { file },
params: {
control_space_id: 'control-space-1',
upload_session_id: 'session-1',
},
})
})
it('resumes only the failed file after a partial multi-file upload', async () => {
const request = vi.fn(async (_url: string, init?: RequestInit) => {
const body = JSON.parse(String(init?.body)) as { fileName: string }
const sessionId = body.fileName === 'a.txt' ? 'session-a' : 'session-b'
return new Response(
JSON.stringify({
session: {
id: sessionId,
mode: 'small_fallback',
status: 'ready',
},
}),
{
status: 201,
headers: { 'content-type': 'application/json' },
},
)
})
vi.stubGlobal('fetch', request)
serviceMock.smallFile.mockImplementation(
({ params }: { params: { upload_session_id: string } }) => {
if (
params.upload_session_id === 'session-b' &&
serviceMock.smallFile.mock.calls.filter(
([call]) => call.params.upload_session_id === 'session-b',
).length === 1
)
return Promise.reject(new Error('response lost'))
return Promise.resolve({
session: {
id: params.upload_session_id,
mode: 'small_fallback',
status: 'completed',
},
})
},
)
const uploads = [
{ file: new File(['a'], 'a.txt', { type: 'text/plain' }), id: 'upload-a' },
{ file: new File(['b'], 'b.txt', { type: 'text/plain' }), id: 'upload-b' },
]
const progress = new Map()
await expect(uploadKnowledgeFsDocuments('control-space-1', uploads, progress)).rejects.toThrow(
'response lost',
)
await expect(
uploadKnowledgeFsDocuments('control-space-1', uploads, progress),
).resolves.toBeUndefined()
expect(request).toHaveBeenCalledTimes(2)
expect(serviceMock.smallFile.mock.calls.map(([call]) => call.params.upload_session_id)).toEqual(
['session-a', 'session-b', 'session-b'],
)
})
})

26
web/features/new-rag/__tests__/knowledge-space-shell.spec.tsx
View File

@ -6,8 +6,8 @@ import { KnowledgeSpaceShell } from '../knowledge-space-shell'
const queryMock = vi.hoisted(() => ({
data: undefined as
| {
id: string
name: string
control_space_id: string
technical_summary: { name: string }
}
| undefined,
error: null as unknown,
@ -37,8 +37,12 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleQuery: {
knowledgeFs: {
getKnowledgeSpacesById: {
queryOptions: queryOptionsMock,
spaces: {
byControlSpaceId: {
get: {
queryOptions: queryOptionsMock,
},
},
},
},
},
@ -60,12 +64,17 @@ describe('KnowledgeSpaceShell', () => {
render(<KnowledgeSpaceShell knowledgeSpaceId="space-1">content</KnowledgeSpaceShell>)
expect(queryOptionsMock).toHaveBeenCalledWith({ input: { params: { id: 'space-1' } } })
expect(queryOptionsMock).toHaveBeenCalledWith({
input: { params: { control_space_id: 'space-1' } },
})
expect(screen.getByRole('status')).toBeInTheDocument()
})
it('renders a refresh-safe header and route navigation when loaded', () => {
queryMock.data = { id: 'space-1', name: 'Support knowledge' }
queryMock.data = {
control_space_id: 'space-1',
technical_summary: { name: 'Support knowledge' },
}
render(<KnowledgeSpaceShell knowledgeSpaceId="space-1">source content</KnowledgeSpaceShell>)
@ -129,7 +138,10 @@ describe('KnowledgeSpaceShell', () => {
it('marks Documents as the only current detail route', () => {
pathnameMock.value = '/datasets/new/space-1/documents'
queryMock.data = { id: 'space-1', name: 'Support knowledge' }
queryMock.data = {
control_space_id: 'space-1',
technical_summary: { name: 'Support knowledge' },
}
render(<KnowledgeSpaceShell knowledgeSpaceId="space-1">document content</KnowledgeSpaceShell>)

38
web/features/new-rag/__tests__/knowledge-view-switcher.spec.tsx Normal file
View File

@ -0,0 +1,38 @@
import { render, screen, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { KnowledgeViewSwitcher } from '../components/knowledge-view-switcher'
const guideStorageMock = vi.hoisted(() => ({
dismissed: false,
setDismissed: vi.fn(),
}))
vi.mock('@/features/new-rag/storage', () => ({
useNewKnowledgeGuideDismissedValue: () => guideStorageMock.dismissed,
useSetNewKnowledgeGuideDismissed: () => guideStorageMock.setDismissed,
}))
describe('KnowledgeViewSwitcher', () => {
beforeEach(() => {
vi.clearAllMocks()
guideStorageMock.dismissed = false
})
it('restores focus to the guide trigger when Escape closes the popover', async () => {
const user = userEvent.setup()
render(<KnowledgeViewSwitcher value="new" onChange={vi.fn()} />)
const trigger = screen.getByRole('button', {
name: 'dataset.newKnowledge.guideTitle',
})
const guide = screen.getByRole('dialog', {
name: 'dataset.newKnowledge.guideTitle',
})
within(guide).getByRole('button', { name: 'dataset.newKnowledge.gotIt' }).focus()
await user.keyboard('{Escape}')
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
expect(trigger).toHaveFocus()
})
})

77
web/features/new-rag/__tests__/new-knowledge-list.spec.tsx
View File

@ -1,17 +1,31 @@
import type { KnowledgeSpaceList } from '@dify/contracts/knowledge-fs/types.gen'
import type { InfiniteData } from '@tanstack/react-query'
import { screen, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { renderWithNuqs } from '@/test/nuqs-testing'
import { NewKnowledgeList } from '../new-knowledge-list'
type KnowledgeSpaceList = {
items: Array<{
createdAt: string
description?: string
iconRef?: string
id: string
name: string
revision: number
slug: string
tenantId: string
updatedAt: string
}>
nextCursor?: string
}
type ListKnowledgeSpacesInfiniteOptions = {
getNextPageParam: (lastPage: KnowledgeSpaceList) => string | undefined
initialPageParam: string | null
getNextPageParam: (lastPage: { has_more: boolean; page: number }) => number | undefined
initialPageParam: number
input: (pageParam: unknown) => {
query: {
cursor?: string
limit: number
page: number
}
}
}
@ -21,6 +35,28 @@ const externalApiPanelMock = vi.hoisted(() => ({
setOpen: vi.fn(),
}))
const toastInfoMock = vi.hoisted(() => vi.fn())
const knowledgeSpaceApiResponse = vi.hoisted(
() => (space: KnowledgeSpaceList['items'][number]) => ({
control_space_id: space.id,
created_at: space.createdAt,
knowledge_space_id: space.id,
owner_account_id: 'account-1',
permission_keys: ['knowledge_space_read'],
resource_version: space.revision,
state: 'active',
technical_status: 'available',
technical_summary: {
description: space.description ?? null,
icon: space.iconRef ?? null,
knowledge_space_id: space.id,
name: space.name,
revision: space.revision,
slug: space.slug,
},
updated_at: space.updatedAt,
visibility: 'only_me',
}),
)
vi.mock('@langgenius/dify-ui/toast', () => ({
toast: { info: toastInfoMock },
@ -69,7 +105,20 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
const original = await importOriginal<typeof import('@tanstack/react-query')>()
return {
...original,
useInfiniteQuery: () => queryMock,
useInfiniteQuery: () => ({
...queryMock,
data: queryMock.data
? {
...queryMock.data,
pages: queryMock.data.pages.map((page, index) => ({
data: page.items.map(knowledgeSpaceApiResponse),
has_more: Boolean(page.nextCursor),
limit: 30,
page: index + 1,
})),
}
: undefined,
}),
}
})
@ -92,8 +141,10 @@ vi.mock('@/context/permission-state', () => ({
vi.mock('@/service/client', () => ({
consoleQuery: {
knowledgeFs: {
listKnowledgeSpaces: {
infiniteOptions: consoleQueryMock.infiniteOptions,
spaces: {
get: {
infiniteOptions: consoleQueryMock.infiniteOptions,
},
},
},
},
@ -137,13 +188,13 @@ describe('NewKnowledgeList', () => {
const options = consoleQueryMock.infiniteOptions.mock.calls.at(-1)?.[0]
expect(options).toBeDefined()
expect(options?.initialPageParam).toBeNull()
expect(options?.input(null)).toEqual({ query: { limit: 30 } })
expect(options?.input('next-page')).toEqual({
query: { cursor: 'next-page', limit: 30 },
expect(options?.initialPageParam).toBe(1)
expect(options?.input(1)).toEqual({ query: { limit: 30, page: 1 } })
expect(options?.input(2)).toEqual({
query: { limit: 30, page: 2 },
})
expect(options?.getNextPageParam({ items: [], nextCursor: 'next-page' })).toBe('next-page')
expect(options?.getNextPageParam({ items: [] })).toBeUndefined()
expect(options?.getNextPageParam({ has_more: true, page: 1 })).toBe(2)
expect(options?.getNextPageParam({ has_more: false, page: 1 })).toBeUndefined()
})
it('links real knowledge spaces to the new detail shell', () => {

191
web/features/new-rag/__tests__/processing-task-events.spec.ts
View File

@ -1,68 +1,67 @@
import type { DocumentProcessingTaskEvent } from '@dify/contracts/knowledge-fs/types.gen'
import { withEventMeta } from '@orpc/client'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { streamProcessingTaskEvents } from '../services/processing-task-events'
const { mockStreamEvents } = vi.hoisted(() => ({
mockStreamEvents: vi.fn(),
const { listBackgroundTasks } = vi.hoisted(() => ({
listBackgroundTasks: vi.fn(),
}))
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents: mockStreamEvents,
spaces: {
byControlSpaceId: {
backgroundTasks: {
get: listBackgroundTasks,
},
},
},
},
},
}))
async function* eventIterator(...events: DocumentProcessingTaskEvent[]) {
yield* events
}
const task = (
state: 'completed' | 'failed' | 'queued' | 'running',
overrides: { documentId?: string; id?: string } = {},
) => ({
can_cancel: state === 'queued' || state === 'running',
can_retry: state === 'failed',
completed_at: state === 'completed' ? '2026-07-20T01:03:00Z' : null,
created_at: '2026-07-20T01:00:00Z',
document_id: overrides.documentId ?? 'document/1',
document_revision: 2,
error_code: state === 'failed' ? 'PROCESSING_FAILED' : null,
error_message: null,
id: overrides.id ?? 'task/1',
knowledge_space_id: 'space/1',
operation: 'document_processing',
progress_percent: state === 'completed' ? 100 : 45,
state,
task_kind: 'document',
updated_at: state === 'completed' ? '2026-07-20T01:03:00Z' : '2026-07-20T01:02:03Z',
})
describe('KnowledgeFS processing task events', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.useRealTimers()
})
it('uses the generated streaming client and resumes from the last event id', async () => {
mockStreamEvents.mockResolvedValue(
eventIterator(
withEventMeta(
{
data: {
progressPercent: 45,
stage: 'parsed',
state: 'running',
updatedAt: '2026-07-20T01:02:03Z',
},
event: 'progress',
},
{ id: 'task-1:2026-07-20T01:02:03Z' },
),
withEventMeta(
{
data: { state: 'succeeded' },
event: 'terminal',
},
{ id: 'task-1:terminal' },
),
),
)
it('polls the unified background-task endpoint until the task is terminal', async () => {
vi.useFakeTimers()
listBackgroundTasks
.mockResolvedValueOnce({ data: [task('running')], next_cursor: null })
.mockResolvedValueOnce({ data: [task('completed')], next_cursor: null })
const abortController = new AbortController()
const events = []
for await (const event of streamProcessingTaskEvents({
const events = streamProcessingTaskEvents({
documentId: 'document/1',
knowledgeSpaceId: 'space/1',
lastEventId: 'task-1:previous',
signal: abortController.signal,
taskId: 'task/1',
})) {
events.push(event)
}
})
expect(events).toEqual([
{
await expect(events.next()).resolves.toEqual({
done: false,
value: {
data: {
progressPercent: 45,
stage: 'parsed',
@ -70,46 +69,92 @@ describe('KnowledgeFS processing task events', () => {
updatedAt: '2026-07-20T01:02:03Z',
},
event: 'progress',
id: 'task-1:2026-07-20T01:02:03Z',
id: '2026-07-20T01:02:03Z:running:45',
},
{
data: { state: 'succeeded' },
})
const terminal = events.next()
await vi.advanceTimersByTimeAsync(5000)
await expect(terminal).resolves.toEqual({
done: false,
value: {
data: { errorCode: undefined, state: 'succeeded' },
event: 'terminal',
id: 'task-1:terminal',
id: '2026-07-20T01:03:00Z:succeeded:100',
},
])
expect(mockStreamEvents).toHaveBeenCalledWith(
})
await expect(events.next()).resolves.toEqual({ done: true, value: undefined })
expect(listBackgroundTasks).toHaveBeenNthCalledWith(
1,
{
headers: { 'last-event-id': 'task-1:previous' },
params: {
documentId: 'document/1',
id: 'space/1',
taskId: 'task/1',
},
params: { control_space_id: 'space/1' },
query: { limit: 200 },
},
{
expect.objectContaining({
context: { silent: true },
signal: abortController.signal,
},
)
})
it('rejects events without a resumable event id', async () => {
mockStreamEvents.mockResolvedValue(
eventIterator({
data: { state: 'failed' },
event: 'terminal',
signal: expect.any(AbortSignal),
}),
)
})
await expect(async () => {
for await (const event of streamProcessingTaskEvents({
documentId: 'document-1',
knowledgeSpaceId: 'space-1',
taskId: 'task-1',
})) {
void event
}
}).rejects.toThrow('missing an event id')
it('continues through cursor pages and stops when the requested task is absent', async () => {
listBackgroundTasks
.mockResolvedValueOnce({ data: [], next_cursor: 'next-page' })
.mockResolvedValueOnce({ data: [], next_cursor: null })
const events = streamProcessingTaskEvents({
documentId: 'document-1',
knowledgeSpaceId: 'space-1',
taskId: 'task-1',
})
await expect(events.next()).resolves.toEqual({ done: true, value: undefined })
expect(listBackgroundTasks).toHaveBeenNthCalledWith(
2,
{
params: { control_space_id: 'space-1' },
query: { cursor: 'next-page', limit: 200 },
},
expect.objectContaining({
context: { silent: true },
signal: expect.any(AbortSignal),
}),
)
})
it('shares one paginated snapshot across concurrent task observers', async () => {
listBackgroundTasks
.mockResolvedValueOnce({
data: [task('running', { documentId: 'document/2', id: 'task/2' })],
next_cursor: 'next-page',
})
.mockResolvedValueOnce({
data: [task('running', { documentId: 'document/1', id: 'task/1' })],
next_cursor: null,
})
const firstController = new AbortController()
const secondController = new AbortController()
const firstEvents = streamProcessingTaskEvents({
documentId: 'document/1',
knowledgeSpaceId: 'space/1',
signal: firstController.signal,
taskId: 'task/1',
})
const secondEvents = streamProcessingTaskEvents({
documentId: 'document/2',
knowledgeSpaceId: 'space/1',
signal: secondController.signal,
taskId: 'task/2',
})
const [first, second] = await Promise.all([firstEvents.next(), secondEvents.next()])
expect(first.done).toBe(false)
expect(second.done).toBe(false)
expect(listBackgroundTasks).toHaveBeenCalledTimes(2)
firstController.abort()
secondController.abort()
await firstEvents.return(undefined)
await secondEvents.return(undefined)
})
})

119
web/features/new-rag/__tests__/sources-page.spec.tsx
View File

@ -1,4 +1,4 @@
import type { Source } from '@dify/contracts/knowledge-fs/types.gen'
import type { Source } from '../source-models'
import { screen, waitFor, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import datasetTranslations from '@/i18n/en-US/dataset.json'
@ -10,6 +10,21 @@ const toastErrorMock = vi.hoisted(() => vi.fn())
const permissionState = vi.hoisted(() => ({
workspacePermissionKeys: ['dataset.acl.edit', 'dataset.external.connect'],
}))
const sourceApiResponse = vi.hoisted(() => (source: Source) => ({
connection_id: source.connectionId ?? null,
created_at: source.createdAt,
credential_configured: source.credentialConfigured ?? null,
id: source.id,
knowledge_space_id: source.knowledgeSpaceId,
metadata: source.metadata,
name: source.name,
permission_scope: source.permissionScope ?? [],
status: source.status,
type: source.type,
updated_at: source.updatedAt,
uri: source.uri,
version: source.version ?? null,
}))
vi.mock('@langgenius/dify-ui/toast', () => ({
toast: { error: toastErrorMock, info: toastInfoMock },
@ -22,12 +37,12 @@ vi.mock('@/context/permission-state', async () => {
})
type SourcesInfiniteOptions = {
getNextPageParam: (lastPage: { nextCursor?: string }) => string | undefined
getNextPageParam: (lastPage: { next_cursor?: string | null }) => string | null | undefined
input: (pageParam: string | null) => unknown
initialPageParam: string | null
refetchInterval: (query: {
state: {
data?: { pages: Array<{ items: Source[] }> }
data?: { pages: Array<{ data: ReturnType<typeof sourceApiResponse>[] }> }
}
}) => false | number
}
@ -55,7 +70,17 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
const original = await importOriginal<typeof import('@tanstack/react-query')>()
return {
...original,
useInfiniteQuery: () => sourcesQuery,
useInfiniteQuery: () => ({
...sourcesQuery,
data: sourcesQuery.data
? {
pages: sourcesQuery.data.pages.map((page) => ({
data: page.items.map(sourceApiResponse),
next_cursor: page.nextCursor ?? null,
})),
}
: undefined,
}),
useQueryClient: () => ({ invalidateQueries: invalidateQueriesMock }),
}
})
@ -63,16 +88,35 @@ vi.mock('@tanstack/react-query', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
deleteKnowledgeSpacesByIdSourcesBySourceId: clientMock.deleteSource,
patchKnowledgeSpacesByIdSourcesBySourceId: clientMock.patchSource,
postKnowledgeSpacesByIdSourcesBySourceIdSync: clientMock.syncSource,
spaces: {
byControlSpaceId: {
sources: {
bySourceId: {
delete: clientMock.deleteSource,
patch: async (input: unknown) =>
sourceApiResponse(await clientMock.patchSource(input)),
sync: { post: clientMock.syncSource },
},
get: {
infiniteOptions: infiniteOptionsMock,
key: vi.fn(() => ['sources']),
},
},
},
},
},
},
consoleQuery: {
knowledgeFs: {
getKnowledgeSpacesByIdSources: {
infiniteOptions: infiniteOptionsMock,
key: vi.fn(() => ['sources']),
spaces: {
byControlSpaceId: {
sources: {
get: {
infiniteOptions: infiniteOptionsMock,
key: vi.fn(() => ['sources']),
},
},
},
},
},
},
@ -116,23 +160,27 @@ describe('SourcesPage', () => {
expect(options).toBeDefined()
if (!options) throw new Error('Expected source infinite query options')
expect(options.input(null)).toEqual({
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
query: { limit: 50 },
})
expect(options.input('next')).toEqual({
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
query: { cursor: 'next', limit: 50 },
})
expect(options.getNextPageParam({ nextCursor: 'next' })).toBe('next')
expect(options.getNextPageParam({ next_cursor: 'next' })).toBe('next')
expect(options.initialPageParam).toBeNull()
expect(
options.refetchInterval({
state: { data: { pages: [{ items: [source({ status: 'syncing' })] }] } },
state: {
data: { pages: [{ data: [sourceApiResponse(source({ status: 'syncing' }))] }] },
},
}),
).toBe(3000)
expect(
options.refetchInterval({
state: { data: { pages: [{ items: [source({ status: 'active' })] }] } },
state: {
data: { pages: [{ data: [sourceApiResponse(source({ status: 'active' }))] }] },
},
}),
).toBe(false)
expect(screen.getByRole('status')).toBeInTheDocument()
@ -425,6 +473,13 @@ describe('SourcesPage', () => {
it('syncs a source through the real KnowledgeFS action', async () => {
const user = userEvent.setup()
sourcesQuery.data = { pages: [{ items: [source({})] }] }
let finishRefresh: (() => void) | undefined
invalidateQueriesMock.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
finishRefresh = resolve
}),
)
render(<SourcesPage knowledgeSpaceId="space-1" />)
await user.click(
@ -438,7 +493,7 @@ describe('SourcesPage', () => {
await waitFor(() =>
expect(clientMock.syncSource).toHaveBeenCalledWith({
headers: { 'Idempotency-Key': expect.any(String) },
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
}),
)
expect(
@ -446,14 +501,24 @@ describe('SourcesPage', () => {
'dataset.newKnowledge.sourceStatus.syncing',
),
).toBeInTheDocument()
finishRefresh?.()
await waitFor(() =>
expect(
within(screen.getByRole('row', { name: /Product documentation/ })).getByText(
'dataset.newKnowledge.sourceStatus.active',
),
).toBeInTheDocument(),
)
const options = infiniteOptionsMock.mock.lastCall?.[0]
expect(options).toBeDefined()
if (!options) throw new Error('Expected source infinite query options')
expect(
options.refetchInterval({
state: { data: { pages: [{ items: [source({ status: 'active' })] }] } },
state: {
data: { pages: [{ data: [sourceApiResponse(source({ status: 'active' }))] }] },
},
}),
).toBe(3000)
).toBe(false)
expect(invalidateQueriesMock).toHaveBeenCalledWith({ queryKey: ['sources'] })
})
@ -482,7 +547,7 @@ describe('SourcesPage', () => {
await waitFor(() =>
expect(clientMock.patchSource).toHaveBeenCalledWith({
body: { expectedVersion: 3, status: 'disabled' },
params: { id: 'space-1', sourceId: 'active-source' },
params: { control_space_id: 'space-1', source_id: 'active-source' },
}),
)
@ -491,7 +556,7 @@ describe('SourcesPage', () => {
await waitFor(() =>
expect(clientMock.patchSource).toHaveBeenLastCalledWith({
body: { expectedVersion: 3, status: 'active' },
params: { id: 'space-1', sourceId: 'disabled' },
params: { control_space_id: 'space-1', source_id: 'disabled' },
}),
)
})
@ -538,7 +603,7 @@ describe('SourcesPage', () => {
await waitFor(() =>
expect(clientMock.patchSource).toHaveBeenLastCalledWith({
body: { expectedVersion: 4, status: 'active' },
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
}),
)
})
@ -560,8 +625,8 @@ describe('SourcesPage', () => {
await waitFor(() =>
expect(clientMock.deleteSource).toHaveBeenCalledWith({
body: { expectedRevision: 3 },
headers: { 'idempotency-key': expect.any(String) },
params: { id: 'space-1', sourceId: 'source-1' },
headers: { 'Idempotency-Key': expect.any(String) },
params: { control_space_id: 'space-1', source_id: 'source-1' },
query: { documents: 'keep' },
}),
)
@ -594,6 +659,13 @@ describe('SourcesPage', () => {
it('retries an errored source and shows its queued state', async () => {
const user = userEvent.setup()
sourcesQuery.data = { pages: [{ items: [source({ status: 'error' })] }] }
let finishRefresh: (() => void) | undefined
invalidateQueriesMock.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
finishRefresh = resolve
}),
)
render(<SourcesPage knowledgeSpaceId="space-1" />)
await user.click(screen.getByRole('button', { name: 'common.operation.retry' }))
@ -604,6 +676,7 @@ describe('SourcesPage', () => {
'dataset.newKnowledge.sourceStatus.syncing',
),
).toBeInTheDocument()
finishRefresh?.()
})
it('supports row selection and a true indeterminate select-all state', async () => {

278
web/features/new-rag/__tests__/website-crawl-preview.spec.tsx
View File

@ -1,4 +1,4 @@
import type { Source, SourceWorkflowRun } from '@dify/contracts/knowledge-fs/types.gen'
import type { Source, SourceWorkflowRun } from '../source-models'
import { act, screen, waitFor, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { render } from '@/test/console/render'
@ -13,6 +13,63 @@ const clientMock = vi.hoisted(() => ({
retry: vi.fn(),
startPreview: vi.fn(),
}))
const sourceApiResponse = vi.hoisted(() => (source: Source) => ({
connection_id: source.connectionId ?? null,
created_at: source.createdAt,
credential_configured: source.credentialConfigured ?? null,
id: source.id,
knowledge_space_id: source.knowledgeSpaceId,
metadata: source.metadata,
name: source.name,
permission_scope: source.permissionScope ?? [],
status: source.status,
type: source.type,
updated_at: source.updatedAt,
uri: source.uri,
version: source.version ?? null,
}))
const workflowApiResponse = vi.hoisted(() => (workflow: SourceWorkflowRun) => ({
canceled_at: workflow.canceledAt ?? null,
checkpoint: workflow.checkpoint,
completed_at: workflow.completedAt ?? null,
created_at: workflow.createdAt,
cursor: workflow.cursor ?? null,
execution_attempts: workflow.executionAttempts,
id: workflow.id,
knowledge_space_id: workflow.knowledgeSpaceId,
kind: workflow.kind,
last_error_code: workflow.lastErrorCode ?? null,
max_execution_attempts: workflow.maxExecutionAttempts,
progress_completed: workflow.progressCompleted,
progress_failed: workflow.progressFailed,
progress_skipped: workflow.progressSkipped,
progress_total: workflow.progressTotal ?? null,
source_id: workflow.sourceId ?? null,
state: workflow.state,
updated_at: workflow.updatedAt,
}))
const crawlPreviewPageListApiResponse = vi.hoisted(
() =>
(response: {
items: Array<{
description?: string
etag?: string
pageId: string
sourceUrl: string
title?: string
}>
nextCursor?: string
}) => ({
data: response.items.map((page) => ({
description: page.description ?? null,
etag: page.etag ?? null,
page_id: page.pageId,
source_url: page.sourceUrl,
title: page.title ?? null,
})),
next_cursor: response.nextCursor ?? null,
}),
)
const routerMock = vi.hoisted(() => ({ push: vi.fn() }))
@ -51,13 +108,41 @@ vi.mock('../crawl-selection-form', () => ({
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
getKnowledgeSpacesByIdSources: clientMock.listSources,
getKnowledgeSpacesByIdSourceWorkflowsByRunId: clientMock.getRun,
getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages: clientMock.getPages,
postKnowledgeSpacesByIdSources: clientMock.createSource,
postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview: clientMock.startPreview,
postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel: clientMock.cancel,
postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry: clientMock.retry,
spaces: {
byControlSpaceId: {
sourceWorkflows: {
byRunId: {
cancel: {
post: async (input: unknown) => workflowApiResponse(await clientMock.cancel(input)),
},
get: async (input: unknown) => workflowApiResponse(await clientMock.getRun(input)),
pages: {
get: async (input: unknown) =>
crawlPreviewPageListApiResponse(await clientMock.getPages(input)),
},
retry: {
post: async (input: unknown) => workflowApiResponse(await clientMock.retry(input)),
},
},
},
sources: {
bySourceId: {
crawlPreview: {
post: async (input: unknown) =>
workflowApiResponse(await clientMock.startPreview(input)),
},
},
get: async (input: unknown) => {
const response = await clientMock.listSources(input)
return {
data: response.items.map(sourceApiResponse),
next_cursor: response.nextCursor ?? null,
}
},
post: async (input: unknown) => sourceApiResponse(await clientMock.createSource(input)),
},
},
},
},
},
}))
@ -116,6 +201,7 @@ describe('WebsiteCrawlPreview', () => {
beforeEach(() => {
vi.useRealTimers()
for (const mock of Object.values(clientMock)) mock.mockReset()
clientMock.cancel.mockResolvedValue(run('canceled'))
clientMock.createSource.mockResolvedValue(source())
clientMock.startPreview.mockResolvedValue(run('running'))
clientMock.getRun.mockResolvedValue(
@ -181,11 +267,11 @@ describe('WebsiteCrawlPreview', () => {
type: 'web',
uri: 'https://docs.dify.ai/',
},
params: { id: 'space-1' },
params: { control_space_id: 'space-1' },
})
expect(clientMock.startPreview).toHaveBeenCalledWith({
headers: { 'Idempotency-Key': expect.any(String) },
params: { id: 'space-1', sourceId: 'source-1' },
params: { control_space_id: 'space-1', source_id: 'source-1' },
})
expect(await screen.findByText('Getting started')).toBeInTheDocument()
expect(screen.getByText(/^dataset\.newKnowledge\.pagesCrawled/)).toHaveAttribute(
@ -198,6 +284,31 @@ describe('WebsiteCrawlPreview', () => {
).not.toBeInTheDocument()
})
it('cancels a preview-ready workflow and starts a fresh run when re-crawling', async () => {
clientMock.cancel.mockResolvedValue(run('canceled'))
render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
const user = await fillValidForm()
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
await screen.findByText('Getting started')
const firstIdempotencyKey =
clientMock.startPreview.mock.calls[0]?.[0].headers['Idempotency-Key']
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' }))
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
expect(clientMock.cancel).toHaveBeenCalledWith({
body: { reason: 'user_requested' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
})
expect(clientMock.retry).not.toHaveBeenCalled()
expect(clientMock.createSource).toHaveBeenCalledOnce()
expect(clientMock.startPreview.mock.calls[1]?.[0].headers['Idempotency-Key']).not.toBe(
firstIdempotencyKey,
)
})
it('submits the crawl form with Enter and enforces the source name contract limit', async () => {
render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
const user = await fillValidForm()
@ -234,6 +345,12 @@ describe('WebsiteCrawlPreview', () => {
await user.clear(pageLimit)
await user.type(pageLimit, '50')
expect(pageLimit).toHaveValue(50)
await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ }))
expect(
screen.getByText(
'dataset.newKnowledge.includeSubpages: dataset.newKnowledge.booleanTrue · dataset.newKnowledge.maxPages: 50',
),
).toBeInTheDocument()
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())
@ -285,7 +402,7 @@ describe('WebsiteCrawlPreview', () => {
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
expect(clientMock.cancel).toHaveBeenCalledWith({
body: { reason: 'user_requested' },
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
})
await waitFor(() =>
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/sources'),
@ -364,10 +481,14 @@ describe('WebsiteCrawlPreview', () => {
expect(clientMock.cancel).not.toHaveBeenCalled()
})
it('cancels a retry that returns after navigation discard was confirmed', async () => {
const retryRequest = deferred<SourceWorkflowRun>()
clientMock.retry.mockReturnValue(retryRequest.promise)
clientMock.cancel.mockResolvedValue(run('canceled'))
it('cancels a fresh re-crawl that returns after navigation discard was confirmed', async () => {
const recrawlRequest = deferred<SourceWorkflowRun>()
clientMock.startPreview
.mockResolvedValueOnce(run('running'))
.mockReturnValueOnce(recrawlRequest.promise)
clientMock.cancel
.mockResolvedValueOnce(run('canceled'))
.mockResolvedValueOnce(run('canceled', { id: 'run-2' }))
render(
<>
<a href="/datasets/new/space-1/documents">Documents navigation</a>
@ -378,28 +499,29 @@ describe('WebsiteCrawlPreview', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
await screen.findByText('Getting started')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' }))
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledOnce())
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
await user.click(screen.getByRole('link', { name: 'Documents navigation' }))
await user.click(
screen.getByRole('button', { name: 'dataset.newKnowledge.discardSourceChangesConfirm' }),
)
retryRequest.resolve(run('running'))
recrawlRequest.resolve(run('running', { id: 'run-2' }))
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledTimes(2))
expect(clientMock.retry).not.toHaveBeenCalled()
await waitFor(() =>
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/documents'),
)
})
it('reconciles a response-lost retry before leaving the preview', async () => {
const previousRun = run('succeeded', { progressCompleted: 1, progressTotal: 1 })
clientMock.getRun
.mockResolvedValueOnce(previousRun)
.mockResolvedValueOnce(previousRun)
.mockResolvedValueOnce(run('running', { executionAttempts: 2 }))
clientMock.retry.mockRejectedValue(new Error('response lost'))
clientMock.cancel.mockResolvedValue(run('canceled', { executionAttempts: 2 }))
it('reconciles a response-lost fresh re-crawl before leaving the preview', async () => {
clientMock.startPreview
.mockResolvedValueOnce(run('running'))
.mockRejectedValueOnce(new Error('response lost'))
.mockResolvedValueOnce(run('running', { id: 'run-2' }))
clientMock.cancel
.mockResolvedValueOnce(run('canceled'))
.mockResolvedValueOnce(run('canceled', { id: 'run-2' }))
render(
<>
<a href="/datasets/new/space-1/documents">Documents navigation</a>
@ -410,15 +532,18 @@ describe('WebsiteCrawlPreview', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
await screen.findByText('Getting started')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' }))
await waitFor(() => expect(clientMock.getRun).toHaveBeenCalledTimes(2))
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
await user.click(screen.getByRole('link', { name: 'Documents navigation' }))
await user.click(
screen.getByRole('button', { name: 'dataset.newKnowledge.discardSourceChangesConfirm' }),
)
await waitFor(() => expect(clientMock.getRun).toHaveBeenCalledTimes(3))
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(3))
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledTimes(2))
expect(clientMock.startPreview.mock.calls[1]?.[0].headers).toEqual(
clientMock.startPreview.mock.calls[2]?.[0].headers,
)
await waitFor(() =>
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/documents'),
)
@ -448,37 +573,17 @@ describe('WebsiteCrawlPreview', () => {
)
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledTimes(2))
expect(clientMock.cancel.mock.calls[0]?.[0].params.runId).toBe('run-1')
expect(clientMock.cancel.mock.calls[1]?.[0].params.runId).toBe('run-1')
expect(clientMock.cancel.mock.calls[0]?.[0].params.run_id).toBe('run-1')
expect(clientMock.cancel.mock.calls[1]?.[0].params.run_id).toBe('run-1')
await waitFor(() =>
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/sources'),
)
})
it('restores polling after cancel failure and honors the latest terminal snapshot', async () => {
const retryRequest = deferred<SourceWorkflowRun>()
clientMock.getRun
.mockResolvedValueOnce(run('succeeded', { progressCompleted: 1, progressTotal: 1 }))
.mockResolvedValueOnce(
run('canceled', {
executionAttempts: 2,
progressCompleted: 1,
updatedAt: '2026-07-20T10:02:00Z',
}),
)
clientMock.getPages
.mockResolvedValueOnce({
items: [
{
pageId: 'page-1',
sourceUrl: 'https://docs.dify.ai/getting-started',
title: 'Getting started',
},
],
})
.mockResolvedValueOnce({ items: [] })
clientMock.retry.mockReturnValue(retryRequest.promise)
clientMock.cancel.mockRejectedValue(Object.assign(new Error('conflict'), { status: 409 }))
it('keeps the preview available when re-crawl cancellation fails', async () => {
clientMock.cancel
.mockRejectedValueOnce(Object.assign(new Error('conflict'), { status: 409 }))
.mockResolvedValueOnce(run('canceled'))
render(
<>
<a href="/datasets/new/space-1/documents">Documents navigation</a>
@ -489,28 +594,16 @@ describe('WebsiteCrawlPreview', () => {
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
await screen.findByText('Getting started')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' }))
await user.click(screen.getByRole('link', { name: 'Documents navigation' }))
await user.click(
screen.getByRole('button', { name: 'dataset.newKnowledge.discardSourceChangesConfirm' }),
)
retryRequest.resolve(
run('running', { executionAttempts: 2, updatedAt: '2026-07-20T10:01:00Z' }),
)
expect(await within(screen.getByRole('alertdialog')).findByRole('alert')).toHaveTextContent(
'dataset.newKnowledge.crawlFailedDescription',
)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.keepEditing' }))
await waitFor(() => expect(clientMock.getRun).toHaveBeenCalledTimes(2))
expect(await screen.findByText('dataset.newKnowledge.crawlStopped')).toBeInTheDocument()
expect(screen.queryByText('Getting started')).not.toBeInTheDocument()
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
expect(clientMock.startPreview).toHaveBeenCalledOnce()
expect(screen.getByText('Getting started')).toBeInTheDocument()
await user.click(screen.getByRole('link', { name: 'Documents navigation' }))
await user.click(
screen.getByRole('button', { name: 'dataset.newKnowledge.discardSourceChangesConfirm' }),
)
expect(clientMock.cancel).toHaveBeenCalledOnce()
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledTimes(2))
await waitFor(() =>
expect(routerMock.push).toHaveBeenCalledWith('/datasets/new/space-1/documents'),
)
@ -622,9 +715,9 @@ describe('WebsiteCrawlPreview', () => {
expect(routerMock.push).not.toHaveBeenCalledWith('/datasets/new/space-1/documents')
})
it('shows pending feedback while a completed crawl is being restarted', async () => {
const retryRequest = deferred<SourceWorkflowRun>()
clientMock.retry.mockReturnValue(retryRequest.promise)
it('shows pending feedback while a preview-ready crawl is being restarted', async () => {
const cancelRequest = deferred<SourceWorkflowRun>()
clientMock.cancel.mockReturnValue(cancelRequest.promise)
render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
const user = await fillValidForm()
@ -633,12 +726,9 @@ describe('WebsiteCrawlPreview', () => {
await user.click(reCrawl)
expect(reCrawl).toHaveAttribute('aria-disabled', 'true')
await act(async () =>
retryRequest.resolve(
run('running', { executionAttempts: 2, updatedAt: '2026-07-20T10:01:00Z' }),
),
)
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledOnce())
await act(async () => cancelRequest.resolve(run('canceled')))
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
expect(clientMock.retry).not.toHaveBeenCalled()
})
it('streams page cursors while running and replaces them with the final snapshot', async () => {
@ -681,14 +771,14 @@ describe('WebsiteCrawlPreview', () => {
expect(await screen.findByText('Two')).toBeInTheDocument()
expect(clientMock.getRun).toHaveBeenNthCalledWith(1, {
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
})
expect(clientMock.getPages).toHaveBeenNthCalledWith(1, {
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
query: { limit: 200 },
})
expect(clientMock.getPages).toHaveBeenNthCalledWith(2, {
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
query: { cursor: 'page-2', limit: 200 },
})
expect(
@ -709,11 +799,11 @@ describe('WebsiteCrawlPreview', () => {
expect(screen.queryByText('Old one')).not.toBeInTheDocument()
expect(screen.queryByText('Deleted page')).not.toBeInTheDocument()
expect(clientMock.getPages).toHaveBeenNthCalledWith(3, {
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
query: { limit: 200 },
})
expect(clientMock.getPages).toHaveBeenNthCalledWith(4, {
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
query: { cursor: 'final-page-2', limit: 200 },
})
expect(
@ -738,7 +828,7 @@ describe('WebsiteCrawlPreview', () => {
await waitFor(() => expect(clientMock.cancel).toHaveBeenCalledOnce())
expect(clientMock.cancel).toHaveBeenCalledWith({
body: { reason: 'user_requested' },
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
})
expect(screen.getByText('Getting started')).toBeInTheDocument()
expect(await screen.findByText('dataset.newKnowledge.crawlStopped')).toHaveAttribute(
@ -822,7 +912,7 @@ describe('WebsiteCrawlPreview', () => {
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledOnce())
expect(clientMock.retry).toHaveBeenCalledWith({
params: { id: 'space-1', runId: 'run-1' },
params: { control_space_id: 'space-1', run_id: 'run-1' },
})
expect(clientMock.createSource).toHaveBeenCalledOnce()
expect(clientMock.startPreview).toHaveBeenCalledOnce()
@ -861,7 +951,7 @@ describe('WebsiteCrawlPreview', () => {
.mockResolvedValueOnce(run('running'))
.mockResolvedValueOnce(failedRun)
.mockResolvedValueOnce(
run('succeeded', { progressCompleted: 1, updatedAt: '2026-07-20T10:01:00Z' }),
run('preview_ready', { progressCompleted: 1, updatedAt: '2026-07-20T10:01:00Z' }),
)
clientMock.getPages.mockResolvedValueOnce({ items: [] }).mockResolvedValue({
items: [
@ -873,6 +963,7 @@ describe('WebsiteCrawlPreview', () => {
],
})
clientMock.retry.mockResolvedValue(run('running'))
clientMock.cancel.mockResolvedValue(run('canceled'))
render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
const user = await fillValidForm()
@ -892,7 +983,9 @@ describe('WebsiteCrawlPreview', () => {
await screen.findByText('Getting started')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.reCrawl' }))
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledTimes(2))
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
expect(clientMock.retry).toHaveBeenCalledOnce()
expect(clientMock.cancel).toHaveBeenCalledOnce()
})
it('reconciles a lost Retry response without sending retry twice', async () => {
@ -987,9 +1080,8 @@ describe('WebsiteCrawlPreview', () => {
})
it('offers an adjust-and-recrawl path after a successful zero-result crawl', async () => {
clientMock.getRun.mockResolvedValue(run('succeeded'))
clientMock.getRun.mockResolvedValue(run('zero_results'))
clientMock.getPages.mockResolvedValue({ items: [] })
clientMock.retry.mockResolvedValue(run('running'))
render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
const user = await fillValidForm()
@ -998,7 +1090,9 @@ describe('WebsiteCrawlPreview', () => {
const noPages = await screen.findByText(/^dataset\.newKnowledge\.noPagesFound:/)
expect(noPages.closest('[role="status"]')).toHaveAttribute('aria-live', 'polite')
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.adjustAndRecrawl' }))
await waitFor(() => expect(clientMock.retry).toHaveBeenCalledOnce())
await waitFor(() => expect(clientMock.startPreview).toHaveBeenCalledTimes(2))
expect(clientMock.retry).not.toHaveBeenCalled()
expect(clientMock.cancel).not.toHaveBeenCalled()
})
it('treats a superseded workflow as terminal and stops polling', async () => {

Some files were not shown because too many files have changed in this diff Show More