diff --git a/api/commands/knowledge_fs.py b/api/commands/knowledge_fs.py index b1d4233de00..fefc6f9eeab 100644 --- a/api/commands/knowledge_fs.py +++ b/api/commands/knowledge_fs.py @@ -37,6 +37,7 @@ from services.knowledge_fs.cutover import ( ShadowCompletionInput, WorkspaceInventoryInput, ) +from services.knowledge_fs.greenfield_initializer import KnowledgeFSWorkspaceGreenfieldInitializer from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote @@ -381,6 +382,15 @@ def status(tenant_id: str) -> None: _echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))) +@knowledge_fs_control_space.command("greenfield-initialize") +@click.option("--tenant-id", required=True) +def greenfield_initialize(tenant_id: str) -> None: + """Idempotently initialize one Workspace that has no KnowledgeFS state.""" + + _operator_call(lambda: _greenfield_initializer().ensure_initialized(tenant_id=tenant_id)) + _echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))) + + @knowledge_fs_control_space.command("cleanup-request") @click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True) @click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.") @@ -593,6 +603,13 @@ def _cutover_service() -> KnowledgeFSWorkspaceCutoverService: ) +def _greenfield_initializer() -> KnowledgeFSWorkspaceGreenfieldInitializer: + return KnowledgeFSWorkspaceGreenfieldInitializer( + session_factory.get_session_maker(), + cutover=_cutover_service(), + ) + + def _cleanup_service() -> KnowledgeFSCleanupService: return KnowledgeFSCleanupService(session_factory.get_session_maker()) diff --git a/api/knowledge-fs-contract.lock.json b/api/knowledge-fs-contract.lock.json index 249e8cdca06..e69b15b7765 100644 --- a/api/knowledge-fs-contract.lock.json +++ b/api/knowledge-fs-contract.lock.json @@ -1,9 +1,9 @@ { "schemaVersion": 5, - "subtreeTree": "7071b81f867b755a915f464aa9319f06debc647b", - "openapiSha256": "12c5e3674697a70129787efd7d277633a718f6a2d80fcd63a031945c73c54150", + "subtreeTree": "8f43d289905c40bfb41642192115847bacfed0bb", + "openapiSha256": "3aff366ab900fd89166f9996f5b20a9a58bbc9315701f236f8d4dfc0bda55a3e", "capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7", "capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3", - "productOperationManifestSha256": "671a915b9653608a2db79fc090681757643cd8562a8725cfd7951fa594a2b980", + "productOperationManifestSha256": "96639519cb5dc0fe523403a14a86a64be2c3e2f751b475fea1bf57fdc072aff5", "productOperationGapManifestSha256": "332c80165bd5cf8e79bc511374dde771a8175510404db603a8067f1a22d36df8" } diff --git a/api/knowledge-fs-product-operations.json b/api/knowledge-fs-product-operations.json index 35c15b4c689..72d920f5b3d 100644 --- a/api/knowledge-fs-product-operations.json +++ b/api/knowledge-fs-product-operations.json @@ -10,6 +10,9 @@ {"productOperationId":"getOverviewInventory","kfsOperationId":"getKnowledgeSpaceOverviewInventory","method":"GET","path":"/knowledge-spaces/{id}/overview/inventory","action":"knowledge_spaces.overview.inventory.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}}, {"productOperationId":"getOverviewHealth","kfsOperationId":"getKnowledgeSpaceProductHealth","method":"GET","path":"/knowledge-spaces/{id}/overview/health","action":"knowledge_spaces.overview.health.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}}, {"productOperationId":"updateSettings","kfsOperationId":"updateKnowledgeSpaceProductSettings","method":"PATCH","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}}, + {"productOperationId":"updateEmbeddingProfile","kfsOperationId":"updateKnowledgeSpaceEmbeddingProfile","method":"PUT","path":"/knowledge-spaces/{id}/embedding-profile","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}}, + {"productOperationId":"updateRetrievalProfile","kfsOperationId":"updateKnowledgeSpaceRetrievalProfile","method":"PUT","path":"/knowledge-spaces/{id}/retrieval-profile","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}}, + {"productOperationId":"getProfileMigration","kfsOperationId":"getKnowledgeSpaceProfileMigration","method":"GET","path":"/knowledge-spaces/{id}/profile-migrations/{migrationId}","action":"knowledge_spaces.settings.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}}, {"productOperationId":"listDocuments","kfsOperationId":"listDocuments","method":"GET","path":"/knowledge-spaces/{id}/documents","action":"documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}}, {"productOperationId":"listLogicalDocuments","kfsOperationId":"listLogicalDocuments","method":"GET","path":"/knowledge-spaces/{id}/logical-documents","action":"logical_documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}}, {"productOperationId":"getLogicalDocument","kfsOperationId":"getLogicalDocument","method":"GET","path":"/knowledge-spaces/{id}/logical-documents/{documentId}","action":"logical_documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}}, diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index c53524e0663..5bd76254ae3 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -7734,10 +7734,7 @@ Get instruction generation template | ---- | ----------- | ------ | | 200 | KnowledgeFS documents | **application/json**: [KnowledgeFSDocumentListResponse](#knowledgefsdocumentlistresponse)
| -### ~~[POST] /knowledge-fs/spaces/{control_space_id}/documents~~ - -***DEPRECATED*** - +### [POST] /knowledge-fs/spaces/{control_space_id}/documents #### Parameters | Name | Located in | Description | Required | Schema | @@ -7748,13 +7745,13 @@ Get instruction generation template | Required | Schema | | -------- | ------ | -| Yes | **application/json**: [KnowledgeFSDocumentCreatePayload](#knowledgefsdocumentcreatepayload)
| +| Yes | **multipart/form-data**: { **"file"**: binary }
| #### Responses | Code | Description | Schema | | ---- | ----------- | ------ | -| 201 | KnowledgeFS document created | **application/json**: [KnowledgeFSDocumentResponse](#knowledgefsdocumentresponse)
| +| 202 | KnowledgeFS document accepted for processing | **application/json**: [KnowledgeFSDocumentUploadAcceptedResponse](#knowledgefsdocumentuploadacceptedresponse)
| ### [DELETE] /knowledge-fs/spaces/{control_space_id}/documents/bulk #### Parameters @@ -8443,6 +8440,7 @@ Get instruction generation template | Name | Located in | Description | Required | Schema | | ---- | ---------- | ----------- | -------- | ------ | | cursor | query | | No | string | +| limit | query | | No | integer,
**Default:** 50 | | control_space_id | path | | Yes | string | #### Responses @@ -8678,6 +8676,27 @@ Get instruction generation template | ---- | ----------- | ------ | | 200 | KnowledgeFS source credential test | **application/json**: [KnowledgeFSSourceCredentialTestResponse](#knowledgefssourcecredentialtestresponse)
| +### [POST] /knowledge-fs/spaces/{control_space_id}/sources/{source_id}/workflow-imports +#### Parameters + +| Name | Located in | Description | Required | Schema | +| ---- | ---------- | ----------- | -------- | ------ | +| Idempotency-Key | header | Stable key used to make the mutation safe to retry | Yes | string | +| control_space_id | path | | Yes | string | +| source_id | path | | Yes | string | + +#### Request Body + +| Required | Schema | +| -------- | ------ | +| Yes | **application/json**: [KnowledgeFSSourceWorkflowImportPayload](#knowledgefssourceworkflowimportpayload)
| + +#### Responses + +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 202 | KnowledgeFS durable provider import accepted | **application/json**: [KnowledgeFSSourceWorkflowResponse](#knowledgefssourceworkflowresponse)
| + ### [GET] /knowledge-fs/spaces/{control_space_id}/traces #### Parameters @@ -20412,14 +20431,6 @@ Input field definition for snippet parameters. | updated_at | number | | Yes | | version | integer | | Yes | -#### KnowledgeFSDocumentCreatePayload - -| Name | Type | Description | Required | -| ---- | ---- | ----------- | -------- | -| idempotency_key | string | | Yes | -| name | string | | Yes | -| text | string | | Yes | - #### KnowledgeFSDocumentDeletePayload | Name | Type | Description | Required | @@ -20543,6 +20554,33 @@ Input field definition for snippet parameters. | size_bytes | integer | | Yes | | state | string,
**Available values:** "active", "candidate", "failed", "superseded" | *Enum:* `"active"`, `"candidate"`, `"failed"`, `"superseded"` | Yes | +#### KnowledgeFSDocumentUploadAcceptedResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| asset | [KnowledgeFSDocumentResponse](#knowledgefsdocumentresponse) | | Yes | +| asset_status_url | string | | No | +| compilation_job | [KnowledgeFSDocumentUploadCompilationJobResponse](#knowledgefsdocumentuploadcompilationjobresponse) | | Yes | +| document_revision | integer | | Yes | +| logical_document | [KnowledgeFSDocumentUploadLogicalDocumentResponse](#knowledgefsdocumentuploadlogicaldocumentresponse) | | Yes | +| logical_document_id | string | | Yes | +| status | string | | No | +| status_url | string | | Yes | + +#### KnowledgeFSDocumentUploadCompilationJobResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| id | string | | Yes | +| stage | string | | Yes | + +#### KnowledgeFSDocumentUploadLogicalDocumentResponse + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| id | string | | Yes | +| revision | integer | | Yes | + #### KnowledgeFSDurableDeletionAcceptedResponse | Name | Type | Description | Required | @@ -20676,6 +20714,43 @@ Input field definition for snippet parameters. | plugin_id | string | | Yes | | provider | string | | Yes | +#### KnowledgeFSOnlineDocumentWorkflowImportItemPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| etag | string | | No | +| lastEditedTime | string | | No | +| name | string | | No | +| pageId | string | | Yes | +| providerItemId | string | | Yes | +| type | string | | Yes | +| workspaceId | string | | Yes | + +#### KnowledgeFSOnlineDocumentWorkflowImportPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| items | [ [KnowledgeFSOnlineDocumentWorkflowImportItemPayload](#knowledgefsonlinedocumentworkflowimportitempayload) ] | | Yes | +| kind | string | | Yes | + +#### KnowledgeFSOnlineDriveWorkflowImportItemPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| bucket | string | | No | +| etag | string | | No | +| id | string | | Yes | +| mimeType | string | | No | +| name | string | | Yes | +| providerItemId | string | | Yes | + +#### KnowledgeFSOnlineDriveWorkflowImportPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| items | [ [KnowledgeFSOnlineDriveWorkflowImportItemPayload](#knowledgefsonlinedriveworkflowimportitempayload) ] | | Yes | +| kind | string | | Yes | + #### KnowledgeFSOverviewCountComparisonResponse | Name | Type | Description | Required | @@ -21374,6 +21449,12 @@ Input field definition for snippet parameters. | ---- | ---- | ----------- | -------- | | reason | string | | No | +#### KnowledgeFSSourceWorkflowImportPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| KnowledgeFSSourceWorkflowImportPayload | [KnowledgeFSOnlineDocumentWorkflowImportPayload](#knowledgefsonlinedocumentworkflowimportpayload)
[KnowledgeFSOnlineDriveWorkflowImportPayload](#knowledgefsonlinedriveworkflowimportpayload) | | | + #### KnowledgeFSSourceWorkflowResponse | Name | Type | Description | Required | diff --git a/api/services/knowledge_fs/cutover.py b/api/services/knowledge_fs/cutover.py index 36d96d505ad..64ca5b230d4 100644 --- a/api/services/knowledge_fs/cutover.py +++ b/api/services/knowledge_fs/cutover.py @@ -11,12 +11,13 @@ from __future__ import annotations import hashlib import json from collections.abc import Callable, Iterable -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Literal, NamedTuple, cast from uuid import NAMESPACE_URL, UUID, uuid5 import sqlalchemy as sa from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, field_validator, model_validator +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, sessionmaker from libs.datetime_utils import naive_utc_now @@ -397,6 +398,8 @@ _CUTOVER_QUARANTINE_KINDS = ( ) _DEFAULT_TRUSTED_SHADOW_PRODUCERS = frozenset({"dify-shadow-authorizer"}) _DEFAULT_TRUSTED_SHADOW_OPERATORS = frozenset({"knowledge-fs-cutover"}) +_GREENFIELD_INITIALIZATION_MAX_ATTEMPTS = 32 +_GREENFIELD_ROLLBACK_WINDOW = timedelta(days=30) _ZERO_REVISION_WATERMARK: KnowledgeFSCutoverRevisionWatermark = { "membership_epoch": 0, "space_acl_epoch": 0, @@ -434,6 +437,114 @@ class KnowledgeFSWorkspaceCutoverService: self._trusted_shadow_operators = trusted_shadow_operators self._remote_factory = remote_factory + def initialize_greenfield( + self, + *, + tenant_id: str, + owner_account_id: str, + ) -> KnowledgeFSWorkspaceCutoverLedger: + """Idempotently cut over a Workspace that has never owned KnowledgeFS state.""" + + try: + tenant_uuid = UUID(tenant_id) + owner_uuid = UUID(owner_account_id) + except ValueError as exc: + raise KnowledgeFSCutoverConflictError("Greenfield Workspace and owner identifiers must be UUIDs") from exc + if not self._trusted_shadow_producers or not self._trusted_shadow_operators: + raise KnowledgeFSCutoverGateBlockedError("Greenfield shadow attestation is not configured") + + inventory = WorkspaceInventoryInput.model_validate( + { + "tenant_id": str(tenant_uuid), + "source_revision_watermark": _ZERO_REVISION_WATERMARK, + "task_watermark": 0, + "spaces": [], + } + ) + for _ in range(_GREENFIELD_INITIALIZATION_MAX_ATTEMPTS): + ledger = self._greenfield_initialization_ledger( + tenant_id=tenant_id, + owner_account_id=owner_account_id, + ) + if ledger is not None and _has_complete_product_cutover(ledger): + return ledger + try: + if ledger is None: + self.inventory(inventory, apply=True) + elif ledger.phase is KnowledgeFSWorkspaceCutoverPhase.INVENTORY: + self.backfill(inventory, apply=True) + elif ledger.phase is KnowledgeFSWorkspaceCutoverPhase.BACKFILL: + self.begin_shadow( + tenant_id=tenant_id, + expected_cas_version=ledger.cas_version, + started_at=self._clock(), + ) + elif ledger.phase is KnowledgeFSWorkspaceCutoverPhase.SHADOW: + if ledger.shadow_completed_at is None: + completed_at = _naive_utc(self._clock()).replace(tzinfo=UTC) + self.complete_shadow( + ShadowCompletionInput.model_validate( + { + "schema_version": "knowledge-fs-p8-shadow-completion/v1", + "tenant_id": tenant_id, + "expected_cas_version": ledger.cas_version, + "producer": min(self._trusted_shadow_producers), + "completed_by_operator": min(self._trusted_shadow_operators), + "completed_by_account_id": str(owner_uuid), + "completed_at": completed_at, + "traffic_zero": True, + "traffic_zero_evidence": { + "schema_version": "knowledge-fs-greenfield-traffic-zero/v1", + "basis": "no-local-control-spaces", + }, + } + ), + apply=True, + ) + elif not ledger.legacy_dependency_ready: + self.legacy_dependency_dashboard( + tenant_id=tenant_id, + dependencies=(), + expected_cas_version=ledger.cas_version, + checked_at=self._clock(), + apply=True, + ) + else: + self.freeze( + tenant_id=tenant_id, + expected_cas_version=ledger.cas_version, + freeze_at=self._clock(), + ) + elif ledger.phase is KnowledgeFSWorkspaceCutoverPhase.FROZEN: + if not _has_complete_greenfield_final_delta(ledger): + self.apply_final_delta( + FinalDeltaInput.model_validate( + { + "tenant_id": tenant_id, + "expected_cas_version": ledger.cas_version, + "final_revision_watermark": _ZERO_REVISION_WATERMARK, + "applied_revision_watermark": _ZERO_REVISION_WATERMARK, + "final_task_watermark": 0, + "applied_task_watermark": 0, + } + ) + ) + else: + cutover_at = _naive_utc(self._clock()) + self.cutover( + tenant_id=tenant_id, + expected_cas_version=ledger.cas_version, + cutover_at=cutover_at, + rollback_cutoff_at=cutover_at + _GREENFIELD_ROLLBACK_WINDOW, + ) + else: + raise KnowledgeFSCutoverGateBlockedError( + "Workspace is not eligible for automatic greenfield initialization" + ) + except (IntegrityError, KnowledgeFSCutoverConflictError): + continue + raise KnowledgeFSCutoverConflictError("Greenfield Workspace initialization did not converge") + def inventory(self, payload: WorkspaceInventoryInput, *, apply: bool) -> CutoverInventoryReport: """Validate a read-only inventory and optionally create the initial CAS ledger.""" @@ -528,6 +639,58 @@ class KnowledgeFSWorkspaceCutoverService: phase = ledger.phase.value return CutoverBackfillReport(tenant_id, registered, granted, quarantined, open_issues, phase, True) + def _greenfield_initialization_ledger( + self, + *, + tenant_id: str, + owner_account_id: str, + ) -> KnowledgeFSWorkspaceCutoverLedger | None: + with self._session_maker() as session: + repository = SQLAlchemyKnowledgeFSCutoverRepository(session) + ledger = repository.get_ledger(tenant_id=tenant_id) + control_spaces = tuple( + session.scalars( + sa.select(KnowledgeFSControlSpace) + .where(KnowledgeFSControlSpace.tenant_id == tenant_id) + .order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id) + ).all() + ) + if ledger is None: + if control_spaces: + raise KnowledgeFSCutoverGateBlockedError( + "Workspace with existing KnowledgeFS control state is not greenfield" + ) + return None + if not _is_zero_space_cutover_ledger(ledger) or ledger.rolled_back_at is not None: + raise KnowledgeFSCutoverGateBlockedError( + "Workspace migration state is not eligible for automatic greenfield initialization" + ) + if ledger.legacy_dependency_report not in (None, []): + raise KnowledgeFSCutoverGateBlockedError( + "Workspace with legacy dependencies is not eligible for automatic greenfield initialization" + ) + if control_spaces: + anchor_id = _greenfield_activation_anchor_id(tenant_id) + provisioning_key = _greenfield_activation_provisioning_key(tenant_id) + if len(control_spaces) != 1: + raise KnowledgeFSCutoverGateBlockedError( + "Workspace with existing KnowledgeFS control state is not greenfield" + ) + anchor = control_spaces[0] + if ( + anchor.id != anchor_id + or anchor.state is not KnowledgeFSControlSpaceState.DELETED + or anchor.knowledge_space_id is not None + or anchor.provisioning_key != provisioning_key + or anchor.owner_account_id != owner_account_id + or anchor.lifecycle_operation_id != anchor_id + ): + raise KnowledgeFSCutoverGateBlockedError( + "Workspace greenfield activation audit anchor is inconsistent" + ) + session.expunge(ledger) + return ledger + def begin_shadow( self, *, @@ -2410,6 +2573,32 @@ def _is_zero_space_cutover_ledger(ledger: KnowledgeFSWorkspaceCutoverLedger) -> ) +def _has_complete_greenfield_final_delta(ledger: KnowledgeFSWorkspaceCutoverLedger) -> bool: + return ( + ledger.final_revision_watermark == _ZERO_REVISION_WATERMARK + and ledger.applied_revision_watermark == _ZERO_REVISION_WATERMARK + and ledger.final_task_watermark == 0 + and ledger.applied_task_watermark == 0 + ) + + +def _has_complete_product_cutover(ledger: KnowledgeFSWorkspaceCutoverLedger) -> bool: + return bool( + ledger.phase + in { + KnowledgeFSWorkspaceCutoverPhase.CUTOVER, + KnowledgeFSWorkspaceCutoverPhase.OBSERVING, + KnowledgeFSWorkspaceCutoverPhase.READY_FOR_CLEANUP, + } + and ledger.cutover_at is not None + and ledger.rolled_back_at is None + and ledger.product_routes_enabled + and ledger.capability_v2_enabled + and ledger.integrated_mode_enabled + and ledger.legacy_acl_read_only + ) + + def _revision_digest(revision: KnowledgeFSCutoverRevisionWatermark) -> str: canonical = json.dumps(revision, sort_keys=True, separators=(",", ":")).encode() return f"sha256:{hashlib.sha256(canonical).hexdigest()}" diff --git a/api/services/knowledge_fs/cutover_runtime_gate.py b/api/services/knowledge_fs/cutover_runtime_gate.py index 8adf527912d..0e0daacd66f 100644 --- a/api/services/knowledge_fs/cutover_runtime_gate.py +++ b/api/services/knowledge_fs/cutover_runtime_gate.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from typing import Protocol import sqlalchemy as sa @@ -13,6 +14,8 @@ from models.knowledge_fs_cutover import ( ) from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError +logger = logging.getLogger(__name__) + _TRAFFIC_PHASES = frozenset( { KnowledgeFSWorkspaceCutoverPhase.CUTOVER, @@ -30,11 +33,23 @@ class KnowledgeFSWorkspaceRuntimeGatePort(Protocol): def require_capability_v2(self, *, tenant_id: str) -> None: ... -class SQLKnowledgeFSWorkspaceRuntimeGate: - """Read the atomic cutover row and reject missing or partial rollout state.""" +class KnowledgeFSWorkspaceInitializerPort(Protocol): + """Lazily establish a fail-closed greenfield cutover for one Workspace.""" - def __init__(self, session_maker: sessionmaker[Session]) -> None: + def ensure_initialized(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger: ... + + +class SQLKnowledgeFSWorkspaceRuntimeGate: + """Read the atomic cutover row, lazily bootstrap greenfield, and fail closed.""" + + def __init__( + self, + session_maker: sessionmaker[Session], + *, + initializer: KnowledgeFSWorkspaceInitializerPort | None = None, + ) -> None: self._session_maker = session_maker + self._initializer = initializer def require_product_routes(self, *, tenant_id: str) -> None: self._require_complete_cutover(tenant_id=tenant_id) @@ -43,23 +58,47 @@ class SQLKnowledgeFSWorkspaceRuntimeGate: self._require_complete_cutover(tenant_id=tenant_id) def _require_complete_cutover(self, *, tenant_id: str) -> None: + ledger = self._read_ledger(tenant_id=tenant_id) + if not _is_complete_cutover(ledger) and self._initializer is not None: + try: + self._initializer.ensure_initialized(tenant_id=tenant_id) + except Exception as exc: + logger.warning( + "KnowledgeFS automatic Workspace initialization failed", + extra={"tenant_id": tenant_id}, + exc_info=True, + ) + raise KnowledgeFSOperationUnavailableError( + "KnowledgeFS Workspace is not cut over for product traffic" + ) from exc + ledger = self._read_ledger(tenant_id=tenant_id) + if not _is_complete_cutover(ledger): + raise KnowledgeFSOperationUnavailableError("KnowledgeFS Workspace is not cut over for product traffic") + + def _read_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None: with self._session_maker() as session: - ledger = session.scalar( + return session.scalar( sa.select(KnowledgeFSWorkspaceCutoverLedger).where( KnowledgeFSWorkspaceCutoverLedger.tenant_id == tenant_id ) ) - if ( - ledger is None - or ledger.phase not in _TRAFFIC_PHASES - or ledger.cutover_at is None - or ledger.rolled_back_at is not None - or not ledger.product_routes_enabled - or not ledger.capability_v2_enabled - or not ledger.integrated_mode_enabled - or not ledger.legacy_acl_read_only - ): - raise KnowledgeFSOperationUnavailableError("KnowledgeFS Workspace is not cut over for product traffic") -__all__ = ["KnowledgeFSWorkspaceRuntimeGatePort", "SQLKnowledgeFSWorkspaceRuntimeGate"] +def _is_complete_cutover(ledger: KnowledgeFSWorkspaceCutoverLedger | None) -> bool: + return bool( + ledger is not None + and ledger.phase in _TRAFFIC_PHASES + and ledger.cutover_at is not None + and ledger.rolled_back_at is None + and ledger.product_routes_enabled + and ledger.capability_v2_enabled + and ledger.integrated_mode_enabled + and ledger.legacy_acl_read_only + ) + + +__all__ = [ + "KnowledgeFSWorkspaceInitializerPort", + "KnowledgeFSWorkspaceRuntimeGatePort", + "SQLKnowledgeFSWorkspaceRuntimeGate", +] diff --git a/api/services/knowledge_fs/greenfield_initializer.py b/api/services/knowledge_fs/greenfield_initializer.py new file mode 100644 index 00000000000..c3b4a0c14b4 --- /dev/null +++ b/api/services/knowledge_fs/greenfield_initializer.py @@ -0,0 +1,58 @@ +"""Automatic bootstrap for Workspaces that have never owned KnowledgeFS state.""" + +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.orm import Session, sessionmaker + +from models.account import Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus +from models.knowledge_fs_cutover import KnowledgeFSWorkspaceCutoverLedger +from services.knowledge_fs.cutover import ( + KnowledgeFSCutoverGateBlockedError, + KnowledgeFSWorkspaceCutoverService, +) + + +class KnowledgeFSWorkspaceGreenfieldInitializer: + """Resolve the Workspace owner and delegate the auditable zero-state cutover.""" + + def __init__( + self, + session_maker: sessionmaker[Session], + *, + cutover: KnowledgeFSWorkspaceCutoverService, + ) -> None: + self._session_maker = session_maker + self._cutover = cutover + + def ensure_initialized(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger: + with self._session_maker() as session: + tenant = session.scalar( + sa.select(Tenant).where( + Tenant.id == tenant_id, + Tenant.status == TenantStatus.NORMAL, + ) + ) + if tenant is None: + raise KnowledgeFSCutoverGateBlockedError("KnowledgeFS Workspace is not active") + owner_account_ids = tuple( + session.scalars( + sa.select(TenantAccountJoin.account_id) + .where( + TenantAccountJoin.tenant_id == tenant_id, + TenantAccountJoin.role == TenantAccountRole.OWNER, + ) + .order_by(TenantAccountJoin.account_id) + ).all() + ) + if len(owner_account_ids) != 1: + raise KnowledgeFSCutoverGateBlockedError( + "Automatic KnowledgeFS initialization requires exactly one Workspace owner" + ) + return self._cutover.initialize_greenfield( + tenant_id=tenant_id, + owner_account_id=owner_account_ids[0], + ) + + +__all__ = ["KnowledgeFSWorkspaceGreenfieldInitializer"] diff --git a/api/services/knowledge_fs/runtime.py b/api/services/knowledge_fs/runtime.py index 587e0600ad9..6e3c50b0548 100644 --- a/api/services/knowledge_fs/runtime.py +++ b/api/services/knowledge_fs/runtime.py @@ -24,8 +24,10 @@ from services.knowledge_fs.control_plane_service import ( ) from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService from services.knowledge_fs.credential_service import KnowledgeFSCredentialService +from services.knowledge_fs.cutover import KnowledgeFSWorkspaceCutoverService from services.knowledge_fs.cutover_runtime_gate import SQLKnowledgeFSWorkspaceRuntimeGate from services.knowledge_fs.data_facade import KnowledgeFSDataFacade +from services.knowledge_fs.greenfield_initializer import KnowledgeFSWorkspaceGreenfieldInitializer from services.knowledge_fs.operation_admission import ( DifyKnowledgeFSBillingPort, DifyKnowledgeFSWeightedRateLimitPort, @@ -38,6 +40,7 @@ from services.knowledge_fs.product_authorization import DifyKnowledgeFSProductRB from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError from services.knowledge_fs.product_remote_http import HTTPKnowledgeFSProductRemoteClient from services.knowledge_fs.product_service import KnowledgeFSProductService +from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote from services.knowledge_fs.revocation_commands import KnowledgeFSRevocationCommandProducer from services.knowledge_fs_capability import create_configured_knowledge_fs_capability_issuer @@ -68,7 +71,18 @@ def create_knowledge_fs_runtime(session_maker: sessionmaker[Session]) -> Knowled issuer = create_configured_knowledge_fs_capability_issuer( audit=SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor(session_maker) ) - cutover_gate = SQLKnowledgeFSWorkspaceRuntimeGate(session_maker) + greenfield_cutover = KnowledgeFSWorkspaceCutoverService( + session_maker, + remote_factory=get_knowledge_fs_lifecycle_remote, + ) + greenfield_initializer = KnowledgeFSWorkspaceGreenfieldInitializer( + session_maker, + cutover=greenfield_cutover, + ) + cutover_gate = SQLKnowledgeFSWorkspaceRuntimeGate( + session_maker, + initializer=greenfield_initializer, + ) batch_capabilities = KnowledgeFSBatchCapabilityBroker( session_maker, cutover_gate=cutover_gate, diff --git a/api/tests/unit_tests/commands/test_knowledge_fs_commands.py b/api/tests/unit_tests/commands/test_knowledge_fs_commands.py index 37bec131f59..b20bc331259 100644 --- a/api/tests/unit_tests/commands/test_knowledge_fs_commands.py +++ b/api/tests/unit_tests/commands/test_knowledge_fs_commands.py @@ -106,6 +106,7 @@ def test_control_space_command_exposes_p8_migration_and_cutover_operations() -> "legacy-check", "freeze", "final-delta", + "greenfield-initialize", "cutover", "smoke", "observe", @@ -120,6 +121,31 @@ def test_control_space_command_exposes_p8_migration_and_cutover_operations() -> } <= set(knowledge_fs_control_space.commands) +def test_greenfield_initialize_cli_runs_the_automatic_initializer() -> None: + with ( + patch("commands.knowledge_fs._greenfield_initializer") as initializer_factory, + patch("commands.knowledge_fs._cutover_service") as cutover_factory, + ): + cutover_factory.return_value.status.return_value = { + "tenant_id": "00000000-0000-0000-0000-000000000001", + "phase": "cutover", + } + result = CliRunner().invoke( + knowledge_fs_control_space, + [ + "greenfield-initialize", + "--tenant-id", + "00000000-0000-0000-0000-000000000001", + ], + ) + + assert result.exit_code == 0 + initializer_factory.return_value.ensure_initialized.assert_called_once_with( + tenant_id="00000000-0000-0000-0000-000000000001" + ) + assert '"phase": "cutover"' in result.output + + def test_operator_cutover_service_installs_remote_registry_lazily() -> None: with patch("commands.knowledge_fs.get_knowledge_fs_lifecycle_remote") as remote_factory: service = _cutover_service() diff --git a/api/tests/unit_tests/services/test_knowledge_fs_cutover.py b/api/tests/unit_tests/services/test_knowledge_fs_cutover.py index ba9f52dbf0e..e22cb70936f 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_cutover.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_cutover.py @@ -479,6 +479,82 @@ def test_empty_workspace_uses_a_stable_greenfield_anchor_for_remote_freeze_and_a ) +def test_greenfield_initialization_advances_an_empty_workspace_idempotently( + cutover_context: tuple[KnowledgeFSWorkspaceCutoverService, sessionmaker[Session]], +) -> None: + _, session_maker = cutover_context + remote = FakeActivationRemote() + service = _service_with_remote(session_maker, remote) + tenant_id = str(uuid4()) + owner_account_id = str(uuid4()) + + ledger = service.initialize_greenfield( + tenant_id=tenant_id, + owner_account_id=owner_account_id, + ) + replay = service.initialize_greenfield( + tenant_id=tenant_id, + owner_account_id=owner_account_id, + ) + + assert ledger.phase is KnowledgeFSWorkspaceCutoverPhase.CUTOVER + assert ledger.product_routes_enabled is True + assert ledger.capability_v2_enabled is True + assert ledger.integrated_mode_enabled is True + assert ledger.legacy_acl_read_only is True + assert replay.cas_version == ledger.cas_version + assert len(remote.freeze_requests) == 1 + assert len(remote.requests) == 1 + with session_maker() as session: + audit_anchor = session.scalar( + sa.select(KnowledgeFSControlSpace).where(KnowledgeFSControlSpace.tenant_id == tenant_id) + ) + assert audit_anchor is not None + assert audit_anchor.owner_account_id == owner_account_id + assert audit_anchor.state is KnowledgeFSControlSpaceState.DELETED + + +def test_greenfield_initialization_resumes_after_a_lost_remote_freeze_ack( + cutover_context: tuple[KnowledgeFSWorkspaceCutoverService, sessionmaker[Session]], +) -> None: + _, session_maker = cutover_context + remote = FakeActivationRemote() + remote.freeze_fail_after_persist = True + service = _service_with_remote(session_maker, remote) + tenant_id = str(uuid4()) + owner_account_id = str(uuid4()) + + with pytest.raises(KnowledgeFSCutoverGateBlockedError, match="freeze was not acknowledged"): + service.initialize_greenfield( + tenant_id=tenant_id, + owner_account_id=owner_account_id, + ) + + ledger = service.initialize_greenfield( + tenant_id=tenant_id, + owner_account_id=owner_account_id, + ) + + assert ledger.phase is KnowledgeFSWorkspaceCutoverPhase.CUTOVER + assert len(remote.freeze_requests) == 2 + assert remote.freeze_requests[0] == remote.freeze_requests[1] + + +def test_greenfield_initialization_rejects_existing_migration_state( + cutover_context: tuple[KnowledgeFSWorkspaceCutoverService, sessionmaker[Session]], +) -> None: + service, _ = cutover_context + payload = _inventory() + tenant_id = str(payload.tenant_id) + service.inventory(payload, apply=True) + + with pytest.raises(KnowledgeFSCutoverGateBlockedError, match="not eligible"): + service.initialize_greenfield( + tenant_id=tenant_id, + owner_account_id=str(uuid4()), + ) + + def test_empty_workspace_freeze_rejects_a_nonempty_remote_namespace( cutover_context: tuple[KnowledgeFSWorkspaceCutoverService, sessionmaker[Session]], ) -> None: diff --git a/api/tests/unit_tests/services/test_knowledge_fs_cutover_runtime_gate.py b/api/tests/unit_tests/services/test_knowledge_fs_cutover_runtime_gate.py index e4410979952..d0a32580216 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_cutover_runtime_gate.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_cutover_runtime_gate.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import UTC, datetime +from unittest.mock import MagicMock import pytest from sqlalchemy.orm import Session, sessionmaker @@ -32,6 +33,37 @@ def test_runtime_gate_fails_closed_without_a_cutover_ledger(sqlite_session: Sess gate.require_capability_v2(tenant_id="tenant-1") +@pytest.mark.parametrize("sqlite_session", [(KnowledgeFSWorkspaceCutoverLedger,)], indirect=True) +def test_runtime_gate_initializes_a_missing_greenfield_workspace_once(sqlite_session: Session) -> None: + session_maker = sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False) + initializer = MagicMock() + + def initialize(*, tenant_id: str) -> None: + with session_maker.begin() as session: + session.add( + KnowledgeFSWorkspaceCutoverLedger( + tenant_id=tenant_id, + source_revision_watermark=_WATERMARK, + applied_revision_watermark=_WATERMARK, + phase=KnowledgeFSWorkspaceCutoverPhase.CUTOVER, + freeze_at=datetime(2026, 7, 21, 11, 59, tzinfo=UTC), + cutover_at=datetime(2026, 7, 21, 12, 0, tzinfo=UTC), + product_routes_enabled=True, + capability_v2_enabled=True, + integrated_mode_enabled=True, + legacy_acl_read_only=True, + ) + ) + + initializer.ensure_initialized.side_effect = initialize + gate = SQLKnowledgeFSWorkspaceRuntimeGate(session_maker, initializer=initializer) + + gate.require_product_routes(tenant_id="tenant-1") + gate.require_capability_v2(tenant_id="tenant-1") + + initializer.ensure_initialized.assert_called_once_with(tenant_id="tenant-1") + + @pytest.mark.parametrize("sqlite_session", [(KnowledgeFSWorkspaceCutoverLedger,)], indirect=True) def test_runtime_gate_requires_the_complete_atomic_cutover_state(sqlite_session: Session) -> None: ledger = KnowledgeFSWorkspaceCutoverLedger( diff --git a/api/tests/unit_tests/services/test_knowledge_fs_greenfield_initializer.py b/api/tests/unit_tests/services/test_knowledge_fs_greenfield_initializer.py new file mode 100644 index 00000000000..c6ef84c3bed --- /dev/null +++ b/api/tests/unit_tests/services/test_knowledge_fs_greenfield_initializer.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from unittest.mock import MagicMock +from uuid import uuid4 + +import pytest +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, sessionmaker + +from models.account import Tenant, TenantAccountJoin, TenantAccountRole +from models.base import TypeBase +from services.knowledge_fs.cutover import KnowledgeFSCutoverGateBlockedError +from services.knowledge_fs.greenfield_initializer import KnowledgeFSWorkspaceGreenfieldInitializer + + +@pytest.fixture +def greenfield_session_maker(sqlite_engine: Engine) -> sessionmaker[Session]: + tables = [ + Tenant.metadata.tables[Tenant.__tablename__], + TenantAccountJoin.metadata.tables[TenantAccountJoin.__tablename__], + ] + TypeBase.metadata.create_all(sqlite_engine, tables=tables) + return sessionmaker(bind=sqlite_engine, expire_on_commit=False) + + +def test_greenfield_initializer_uses_the_unique_workspace_owner( + greenfield_session_maker: sessionmaker[Session], +) -> None: + tenant = Tenant(name="Greenfield") + owner_account_id = str(uuid4()) + with greenfield_session_maker.begin() as session: + session.add(tenant) + session.flush() + tenant_id = tenant.id + session.add( + TenantAccountJoin( + tenant_id=tenant_id, + account_id=owner_account_id, + role=TenantAccountRole.OWNER, + ) + ) + cutover = MagicMock() + initializer = KnowledgeFSWorkspaceGreenfieldInitializer(greenfield_session_maker, cutover=cutover) + + initializer.ensure_initialized(tenant_id=tenant_id) + + cutover.initialize_greenfield.assert_called_once_with( + tenant_id=tenant_id, + owner_account_id=owner_account_id, + ) + + +def test_greenfield_initializer_rejects_a_workspace_without_one_owner( + greenfield_session_maker: sessionmaker[Session], +) -> None: + tenant = Tenant(name="No owner") + with greenfield_session_maker.begin() as session: + session.add(tenant) + session.flush() + tenant_id = tenant.id + initializer = KnowledgeFSWorkspaceGreenfieldInitializer(greenfield_session_maker, cutover=MagicMock()) + + with pytest.raises(KnowledgeFSCutoverGateBlockedError, match="exactly one Workspace owner"): + initializer.ensure_initialized(tenant_id=tenant_id) diff --git a/api/tests/unit_tests/services/test_knowledge_fs_runtime.py b/api/tests/unit_tests/services/test_knowledge_fs_runtime.py index ef6f2b6ae6b..1a817cf8736 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_runtime.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_runtime.py @@ -40,12 +40,15 @@ def test_runtime_wires_one_shared_authorization_and_remote_graph(monkeypatch: py "KnowledgeFSProductApplicationService", "KnowledgeFSProductService", "KnowledgeFSRevocationCommandProducer", + "KnowledgeFSWorkspaceCutoverService", + "KnowledgeFSWorkspaceGreenfieldInitializer", "LoggingKnowledgeFSRateLimitAudit", "SQLKnowledgeFSAppCatalog", "SQLKnowledgeFSWorkspaceMemberPort", "SQLKnowledgeFSWorkspaceRuntimeGate", "SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor", "create_configured_knowledge_fs_capability_issuer", + "get_knowledge_fs_lifecycle_remote", ) factories: dict[str, MagicMock] = {} for name in factory_names: @@ -80,6 +83,18 @@ def test_runtime_wires_one_shared_authorization_and_remote_graph(monkeypatch: py factories["KnowledgeFSProductService"].call_args.kwargs["cutover_gate"] is factories["SQLKnowledgeFSWorkspaceRuntimeGate"].return_value ) + factories["KnowledgeFSWorkspaceCutoverService"].assert_called_once_with( + session_maker, + remote_factory=factories["get_knowledge_fs_lifecycle_remote"], + ) + factories["KnowledgeFSWorkspaceGreenfieldInitializer"].assert_called_once_with( + session_maker, + cutover=factories["KnowledgeFSWorkspaceCutoverService"].return_value, + ) + factories["SQLKnowledgeFSWorkspaceRuntimeGate"].assert_called_once_with( + session_maker, + initializer=factories["KnowledgeFSWorkspaceGreenfieldInitializer"].return_value, + ) assert ( factories["KnowledgeFSProductApplicationService"].call_args.kwargs["rbac"] is factories["DifyKnowledgeFSProductRBACPort"].return_value diff --git a/knowledge-fs/docs/api-reference.md b/knowledge-fs/docs/api-reference.md index b12e37d2285..f4bbb4b7186 100644 --- a/knowledge-fs/docs/api-reference.md +++ b/knowledge-fs/docs/api-reference.md @@ -15,20 +15,53 @@ the live spec and this companion document differ. streaming answers/progress are `text/event-stream`; raw multimodal assets are `application/octet-stream`. +### Dify-integrated request path + +In a Dify deployment, browser clients call the Dify Console API under +`/console/api/knowledge-fs/...`; service clients use the narrower Service API under +`/v1/knowledge-fs/...`. They do not call this gateway directly or manage model, datasource, or +object-storage credentials. Dify performs Workspace/account authorization, resolves the local +control-space to a KnowledgeFS space, and issues a short-lived Capability v2 grant for each upstream +operation. + +The generated Dify contracts are: + +- [Console KnowledgeFS API](../../api/openapi/markdown/console-openapi.md), including Space, + Overview, Document, Source, Background Task, Evidence, and Quality routes. +- [Service KnowledgeFS API](../../api/openapi/markdown/service-openapi.md), which intentionally + exposes a smaller application-facing surface. + +On first product access, Dify automatically performs the auditable greenfield cutover only when the +Workspace has one owner, no local KnowledgeFS control state, and an empty remote namespace. +`503 knowledge_fs_operation_unavailable` means the integration is not configured, that strict +greenfield initialization could not complete, or the Workspace has migration state that still +requires operator-controlled cutover. It is not a signal for the client to bypass Dify and invoke +the gateway directly. + ## Authentication & scopes -- **Bearer token** (JWT) on every guarded route: `Authorization: Bearer `. The tenant is taken - from the JWT; a knowledge space is always resolved as `spaces.get({ id, tenantId })`, so one tenant - can never read another tenant's space (returns `404`). -- **Scopes** (`getRequiredScope`): every `GET` needs `knowledge-spaces:read`; every `POST/PATCH/DELETE` - needs `knowledge-spaces:write` — **except** `POST /queries`, `POST /research-tasks/plan`, and - `POST /agent-workspace-snapshots/{id}/replay`, which need only `read`. The wildcard scope - `knowledge-spaces:*` satisfies both. +- **Production auth profile**: guarded routes accept a short-lived, Dify-issued RS256 Capability v2 + JWT as `Authorization: Bearer `. KnowledgeFS has no independently managed production + caller credentials and does not accept the removed legacy tenant JWT/API-key authorization + profile. +- **Exact binding**: each capability binds the caller kind, Dify Workspace namespace, control-space, + action, resource type/id, optional parent, authorization/content-policy revisions, candidate + content scopes, and trace/grant identity. The maximum token lifetime is 60 seconds. A valid token + for one operation or resource cannot authorize another. +- **Route policy labels** (`getRequiredScope`): OpenAPI retains the coarse + `knowledge-spaces:read|write` labels for route classification. In production, Dify converts the + requested product operation into an exact Capability v2 action/resource grant; clients cannot + broaden access by supplying those labels themselves. Most `GET` operations classify as `read` and + mutations as `write`, with explicitly read-only POST exceptions such as query planning/replay. +- **Tenant isolation**: the Workspace namespace comes from the signed capability. A knowledge space + is always resolved with that namespace, so cross-Workspace resource IDs fail closed (normally + `404`). - **Auth is path-prefix middleware**, not per-route: it guards `/knowledge-spaces*`, `/queries*`, `/jobs*`, `/research-tasks*`, `/agent-workspace-snapshots*`, `/bulk-jobs*`, `/retention-policy`. - **Public (no auth)**: `GET /health` and `GET /openapi.json`. -- **Auth failures**: `401` (missing/invalid token), `403` (token lacks the required scope), both - `{ error: string }`. + Internal lifecycle routes additionally require the exact `internal_worker` capability action. + **Public (no auth)**: `GET /health`, `GET /ready`, and `GET /openapi.json`. +- **Auth failures**: `401` (missing/invalid token) or `403` (wrong action, caller kind, resource, + revision, or lifecycle state), both `{ error: string, code?: string }`. ## Conventions @@ -250,6 +283,119 @@ the live spec and this companion document differ. --- +## Overview & background tasks + +The Overview endpoints are bounded, read-only product projections. They never expose model +credentials, datasource secrets, internal leases, or worker fencing data. In the Dify Console API, +the corresponding routes use a local `control_space_id`; Dify resolves it to the upstream +KnowledgeFS `id`. + +### `GET /knowledge-spaces/{id}/overview/stats` +**Description**: Return current inventory counters plus query/answer totals for the fixed `24h`, +`7d`, and `30d` windows. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). +**Responses**: +- `200`: `{ knowledgeSpaceId, generatedAt, current, windows }`. `current` contains + `knowledgeCount`, `linkedAppCount`, source totals/freshness, and `latestSourceSyncAt?`; each window + contains `{ since, queryCount, answeredQueryCount, answerRate }`. +- `404`; `503` Overview backend unavailable; `401`/`403`. + +### `GET /knowledge-spaces/{id}/overview/query-outcomes` +**Description**: Return requester-scoped current/previous outcome totals and a bounded time series. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). **Query**: `window` (`24h|7d|30d`, optional, default `24h`). +**Responses**: +- `200`: `{ knowledgeSpaceId, window, since, previousSince, generatedAt, current, previous, buckets }`. + Totals contain `queryCount`, `answered`, `lowConfidence`, `noEvidence`, and `answerRate`; buckets + additionally contain `startAt` and `endAt`. +- `400`; `404`; `503`; `401`/`403`. + +### `GET /knowledge-spaces/{id}/overview/inventory` +**Description**: Return source-category, graph, and active-index inventory. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). +**Responses**: +- `200`: `{ knowledgeSpaceId, generatedAt, sourceCategories, graphEntities, graphRelations, + indexCoverage }`; index coverage contains `{ indexed, total, percentage }`. +- `404`; `503`; `401`/`403`. + +### `GET /knowledge-spaces/{id}/overview/activity` +**Description**: Cursor-paginated append-only product activity. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). +**Query**: `cursor`; `limit` (1–100); `from`/`to` (date-time); optional `action`, +`resourceType`, and `result` filters. +**Responses**: +- `200`: `{ items: [{ id, action, actor, resource, result, details, occurredAt }], nextCursor? }`. +- `400`; `404`; `503`; `401`/`403`. + +This upstream route is not currently exposed by the Dify Console controller; the Console Overview +uses its existing activity integration until that product contract is migrated. + +### `GET /knowledge-spaces/{id}/overview/attention` +**Description**: List rule-backed Needs Attention findings. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). **Query**: `limit` (1–100); `includeDismissed` (boolean, default +`false`). +**Responses**: +- `200`: `{ items }`; each item includes `issueKey`, `ruleId`, `severity`, `status`, `title`, + `resource`, `action`, `evidence`, `revision`, `updatedAt`, and optional `dismissedUntil`. +- `400`; `404`; `503`; `401`/`403`. + +### `PATCH /knowledge-spaces/{id}/overview/attention/{issueKey}` +**Description**: Dismiss, resolve, or reactivate one finding with revision CAS. +**Auth**: Bearer; scope `knowledge-spaces:write`. +**Path params**: `id` (uuid); `issueKey` (string). +**Body** (`application/json`, strict): `{ expectedRevision, status: +"active"|"dismissed"|"resolved", dismissedUntil? }`. +**Responses**: `200` updated finding; `404`; `409` revision conflict; `503`; `401`/`403`. + +The attention list/transition routes are upstream contracts and are not currently proxied by the +Dify Console API. + +### `GET /knowledge-spaces/{id}/overview/health` +**Description**: Return the stable aggregate product-health state. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). +**Responses**: +- `200`: `{ knowledgeSpaceId, generatedAt, state, components }`, where `state` and each component + state are `healthy|degraded|unavailable|unknown`. Components cover index, ingestion, profile + publication, query availability, source freshness, and worker readiness. +- `404`; `503`; `401`/`403`. + +### `GET /knowledge-spaces/{id}/background-tasks` +**Description**: List newest durable document, bulk-document, and source tasks across their native +job stores. +**Auth**: Bearer; scope `knowledge-spaces:read`. +**Path params**: `id` (uuid). **Query**: `cursor` (opaque, optional); `limit` (1–100, default 50). +**Responses**: +- `200`: `{ items: BackgroundTask[], nextCursor? }`. +- `BackgroundTask`: `{ id, knowledgeSpaceId, taskKind, operation, state, canCancel, canRetry, + progressCompleted, progressFailed, progressTotal, progressPercent, createdAt, updatedAt, + completedAt?, documentId?, documentRevision?, sourceId?, errorCode?, errorMessage? }`. +- `taskKind`: `document|document_bulk|source`; `state`: + `queued|running|completed|failed|canceled`. +- `400`; `404`; `409`; `401`/`403`. + +### `POST /knowledge-spaces/{id}/background-tasks/{taskKind}/{taskId}/cancel` +**Description**: Request cancellation through the task's native durable state machine. +**Auth**: Bearer; scope `knowledge-spaces:write`. +**Path params**: `id` (uuid); `taskKind` (`document|document_bulk|source`); `taskId` (uuid). +**Body**: None. +**Responses**: `200` updated `BackgroundTask`; `400`; `404`; `409` transition not allowed; +`401`/`403`. + +### `POST /knowledge-spaces/{id}/background-tasks/{taskKind}/{taskId}/retry` +**Description**: Retry a failed or canceled task through the task's native durable state machine. +**Auth**: Bearer; scope `knowledge-spaces:write`. +**Path params**: `id` (uuid); `taskKind` (`document|document_bulk|source`); `taskId` (uuid). +**Body**: None. +**Responses**: `200` updated `BackgroundTask`; `400`; `404`; `409` transition not allowed; +`401`/`403`. + +--- + ## Query, answers, golden & failed queries ### `POST /queries` @@ -743,11 +889,49 @@ compatibility path does not admit Auto. **Auth**: **None (public)**. **Responses**: `200` `{ ok: boolean, runtime: enum(cloudflare-workers|node-docker), components: Record }`. +### `GET /ready` +**Description**: Deployment readiness gate. Unlike `/health`, this fails closed until every required +runtime dependency and Dify-integrated configuration is ready to receive traffic. +**Auth**: **None (public)**. +**Responses**: +- `200`: `{ ok: true, runtime, components }`. +- `503`: `{ ok: false, runtime, components }`. + ### `GET /openapi.json` **Description**: The machine-readable OpenAPI 3 document for the whole API (served via `app.doc`). **Auth**: **None (public)**. **Responses**: `200` OpenAPI JSON document. +### `POST /internal/dify-integration/freeze` +**Description**: Persist or idempotently replay a Dify Workspace maintenance freeze before product +traffic activation. +**Auth**: Dify-issued Capability v2 token with the exact internal freeze operation and namespace; +not callable by browser or Service API clients. +**Body** (`application/json`, strict): `{ freezeId, freezeRevision, sourceRevisionDigest, +sourceTaskWatermark }`. IDs/digests use `sha256:<64 lowercase hex>` and numeric evidence must be a +JavaScript-safe integer. +**Responses**: +- `200`: freeze evidence plus `{ namespaceId, frozen: true, frozenAt, updatedAt, applied, replayed }`. +- `401`; `403` wrong capability; `409` conflicting revision/evidence. + +The namespace/tenant is taken from the signed capability, not from request JSON. Reusing an +operation grant is rejected; an exact durable evidence replay uses a newly issued grant. + +### `POST /internal/dify-integration/activate` +**Description**: Persist or idempotently replay monotonic Dify Workspace integration activation. +**Auth**: Dify-issued Capability v2 token with the exact internal activation operation and +namespace; not callable by browser or Service API clients. +**Body** (`application/json`, strict): `{ activationId, activationRevision, +sourceRevisionDigest }`. +**Responses**: +- `200`: activation evidence plus `{ namespaceId, active: true, activatedAt, updatedAt, applied, + replayed }`. +- `401`; `403` wrong capability; `409` non-monotonic or conflicting activation evidence. + +Activation is an acknowledgement to Dify's Workspace cutover coordinator. Dify publishes its local +product-route, Capability v2, integrated-mode, and legacy-ACL-read-only switches only after this +exact durable acknowledgement is validated. + ### `GET /bulk-jobs/{id}` **Description**: Progress for a bulk document operation job. **Auth**: Bearer; scope `knowledge-spaces:read`. diff --git a/knowledge-fs/packages/api/src/knowledge-space-profile-migration-routes.ts b/knowledge-fs/packages/api/src/knowledge-space-profile-migration-routes.ts index 3e078d99d2d..309140fb08f 100644 --- a/knowledge-fs/packages/api/src/knowledge-space-profile-migration-routes.ts +++ b/knowledge-fs/packages/api/src/knowledge-space-profile-migration-routes.ts @@ -54,6 +54,7 @@ export const requestKnowledgeSpaceProfileMigrationRoute = createRoute({ export const getKnowledgeSpaceProfileMigrationRoute = createRoute({ method: "get", + operationId: "getKnowledgeSpaceProfileMigration", path: "/knowledge-spaces/{id}/profile-migrations/{migrationId}", request: { params: KnowledgeSpaceProfileMigrationParamsSchema }, responses: { diff --git a/knowledge-fs/packages/api/src/knowledge-space-routes.ts b/knowledge-fs/packages/api/src/knowledge-space-routes.ts index 5f0a0d67f12..5cf884e1b1a 100644 --- a/knowledge-fs/packages/api/src/knowledge-space-routes.ts +++ b/knowledge-fs/packages/api/src/knowledge-space-routes.ts @@ -373,6 +373,7 @@ export const getKnowledgeSpaceManifestRoute = createRoute({ export const updateKnowledgeSpaceEmbeddingProfileRoute = createRoute({ method: "put", + operationId: "updateKnowledgeSpaceEmbeddingProfile", path: "/knowledge-spaces/{id}/embedding-profile", request: { body: { @@ -436,6 +437,7 @@ export const updateKnowledgeSpaceEmbeddingProfileRoute = createRoute({ export const updateKnowledgeSpaceRetrievalProfileRoute = createRoute({ method: "put", + operationId: "updateKnowledgeSpaceRetrievalProfile", path: "/knowledge-spaces/{id}/retrieval-profile", request: { body: {