From e4e9271fbe6dc549617d5b0751a5c87c74b7de8a Mon Sep 17 00:00:00 2001 From: Stephen Zhou <38493346+hyoban@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:08:01 +0800 Subject: [PATCH] feat(knowledge-fs): create spaces with initial sources --- .../console/knowledge_fs/resources.py | 27 ++ .../knowledge_fs/initial_source_preview.py | 177 ++++++++ .../product_application_service.py | 6 +- api/services/knowledge_fs/product_dto.py | 136 ++++-- .../knowledge_fs_initial_source_tasks.py | 420 ++++++++++++++---- ...est_knowledge_fs_initial_source_preview.py | 184 ++++++++ ...nowledge_fs_product_application_service.py | 55 ++- .../services/test_knowledge_fs_product_dto.py | 70 +++ .../test_knowledge_fs_initial_source_tasks.py | 178 +++++++- .../api/console/knowledge-fs/orpc.gen.ts | 166 ++++--- .../api/console/knowledge-fs/types.gen.ts | 133 +++++- .../api/console/knowledge-fs/zod.gen.ts | 201 ++++++--- .../__tests__/create-knowledge-page.spec.tsx | 355 ++++++++++++++- .../new-rag/create-connected-source-setup.tsx | 410 +++++++++++++++++ .../new-rag/create-knowledge-page.tsx | 85 +--- .../new-rag/create-knowledge-workflow.ts | 3 +- web/features/new-rag/create-source-setup.tsx | 416 +++++++++-------- 17 files changed, 2467 insertions(+), 555 deletions(-) create mode 100644 api/services/knowledge_fs/initial_source_preview.py create mode 100644 api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py create mode 100644 web/features/new-rag/create-connected-source-setup.tsx diff --git a/api/controllers/console/knowledge_fs/resources.py b/api/controllers/console/knowledge_fs/resources.py index 5711eb2a3a6..d6937b28b20 100644 --- a/api/controllers/console/knowledge_fs/resources.py +++ b/api/controllers/console/knowledge_fs/resources.py @@ -48,6 +48,7 @@ from services.knowledge_fs.control_plane_service import ( from services.knowledge_fs.credential_service import ( KnowledgeFSCredentialPolicyError, ) +from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService from services.knowledge_fs.product_authorization import ( KnowledgeFSProductNotFoundError, ) @@ -100,6 +101,8 @@ from services.knowledge_fs.product_dto import ( KnowledgeFSGoldenQuestionPayload, KnowledgeFSGoldenQuestionResponse, KnowledgeFSIdempotencyHeader, + KnowledgeFSInitialSourcePreviewPayload, + KnowledgeFSInitialSourcePreviewResponse, KnowledgeFSJWKSResponse, KnowledgeFSLogicalDocumentDeletePayload, KnowledgeFSLogicalDocumentListResponse, @@ -246,6 +249,7 @@ register_schema_models( KnowledgeFSSourceConnectionRefreshPayload, KnowledgeFSCrawlImportPayload, KnowledgeFSCrawlPreviewSelectionPayload, + KnowledgeFSInitialSourcePreviewPayload, KnowledgeFSSourceDeletePayload, KnowledgeFSSourceDeleteQuery, KnowledgeFSSourceFilesQuery, @@ -340,6 +344,7 @@ register_response_schema_models( KnowledgeFSOverviewQueryOutcomesResponse, KnowledgeFSOverviewStatsResponse, KnowledgeFSPresignedUploadResponse, + KnowledgeFSInitialSourcePreviewResponse, KnowledgeFSUploadSessionCreateResponse, KnowledgeFSUploadSessionMutationResponse, ) @@ -579,6 +584,28 @@ def _overview_stats_response( ) +@console_ns.route("/knowledge-fs/source-provider-preview") +class KnowledgeFSInitialSourcePreviewApi(Resource): + @console_ns.expect(console_ns.models[KnowledgeFSInitialSourcePreviewPayload.__name__]) + @console_ns.response( + HTTPStatus.OK, + "Datasource resources available for an initial Source", + console_ns.models[KnowledgeFSInitialSourcePreviewResponse.__name__], + ) + @setup_required + @login_required + @account_initialization_required + @_knowledge_fs_errors + def post(self): + account, tenant_id = current_account_with_tenant() + result = KnowledgeFSInitialSourcePreviewService(session_factory.get_session_maker()).preview( + tenant_id=tenant_id, + account=account, + payload=_payload(KnowledgeFSInitialSourcePreviewPayload), + ) + return dump_response(KnowledgeFSInitialSourcePreviewResponse, result) + + @console_ns.route("/knowledge-fs/spaces") class KnowledgeFSSpacesApi(Resource): @console_ns.doc(params=query_params_from_model(KnowledgeFSSpaceListQuery)) diff --git a/api/services/knowledge_fs/initial_source_preview.py b/api/services/knowledge_fs/initial_source_preview.py new file mode 100644 index 00000000000..53282822564 --- /dev/null +++ b/api/services/knowledge_fs/initial_source_preview.py @@ -0,0 +1,177 @@ +"""Read-only datasource discovery used before a KnowledgeFS Space exists.""" + +from __future__ import annotations + +import json +from typing import Any, cast + +from sqlalchemy import select + +from core.datasource.datasource_manager import DatasourceManager +from core.datasource.entities.datasource_entities import ( + DatasourceProviderType, + OnlineDriveBrowseFilesRequest, +) +from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin +from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin +from models.account import Account +from models.credential_permission import CredentialType +from models.oauth import DatasourceProvider +from services.credential_permission_service import CredentialPermissionService +from services.datasource_provider_service import DatasourceProviderService +from services.knowledge_fs.product_dto import ( + KnowledgeFSInitialSourcePreviewDocumentResponse, + KnowledgeFSInitialSourcePreviewFileResponse, + KnowledgeFSInitialSourcePreviewPayload, + KnowledgeFSInitialSourcePreviewResponse, +) + +_MAX_PREVIEW_ITEMS = 200 + + +class KnowledgeFSInitialSourcePreviewService: + def __init__(self, session_maker) -> None: + self._session_maker = session_maker + + def _require_visible_credential( + self, + *, + tenant_id: str, + account: Account, + payload: KnowledgeFSInitialSourcePreviewPayload, + ) -> None: + query = select(DatasourceProvider).where( + DatasourceProvider.tenant_id == tenant_id, + DatasourceProvider.id == payload.credential_id, + DatasourceProvider.provider == payload.provider, + DatasourceProvider.plugin_id == payload.plugin_id, + ) + query = CredentialPermissionService.apply_visibility_filter( + query, + model_id_column=DatasourceProvider.id, + model_user_id_column=DatasourceProvider.user_id, + model_visibility_column=DatasourceProvider.visibility, + credential_type=CredentialType.DATASOURCE_PROVIDER, + user=account, + ) + with self._session_maker() as session: + if session.scalar(query.limit(1)) is None: + raise PermissionError("Datasource credential is unavailable") + + def preview( + self, + *, + tenant_id: str, + account: Account, + payload: KnowledgeFSInitialSourcePreviewPayload, + ) -> KnowledgeFSInitialSourcePreviewResponse: + self._require_visible_credential(tenant_id=tenant_id, account=account, payload=payload) + credentials = DatasourceProviderService().get_datasource_credentials( + tenant_id=tenant_id, + provider=payload.provider, + plugin_id=payload.plugin_id, + credential_id=payload.credential_id, + ) + if not credentials: + raise PermissionError("Datasource credential is unavailable") + provider_type = DatasourceProviderType(payload.kind) + runtime = DatasourceManager.get_datasource_runtime( + provider_id=f"{payload.plugin_id}/{payload.provider}", + datasource_name=payload.datasource, + tenant_id=tenant_id, + datasource_type=provider_type, + ) + runtime.runtime.credentials = credentials + parameters = dict(payload.parameters) + if payload.kind == "online_document": + document_runtime = cast(OnlineDocumentDatasourcePlugin, runtime) + documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = [] + for document_message in document_runtime.get_online_document_pages( + user_id=account.id, + datasource_parameters=parameters, + provider_type=document_runtime.datasource_provider_type(), + ): + for workspace in document_message.result: + workspace_id = workspace.workspace_id or payload.provider + for page in workspace.pages: + documents.append( + KnowledgeFSInitialSourcePreviewDocumentResponse( + last_edited_time=page.last_edited_time, + name=page.page_name, + page_id=page.page_id, + provider_item_id=json.dumps([workspace_id, page.page_id], separators=(",", ":")), + type=page.type, + workspace_id=workspace_id, + workspace_name=workspace.workspace_name, + ) + ) + if len(documents) >= _MAX_PREVIEW_ITEMS: + return KnowledgeFSInitialSourcePreviewResponse( + documents=documents, + kind=payload.kind, + ) + return KnowledgeFSInitialSourcePreviewResponse(documents=documents, kind=payload.kind) + + drive_runtime = cast(OnlineDriveDatasourcePlugin, runtime) + files: list[KnowledgeFSInitialSourcePreviewFileResponse] = [] + next_page_parameters = None + max_keys = parameters.get("max_keys", _MAX_PREVIEW_ITEMS) + if not isinstance(max_keys, int) or isinstance(max_keys, bool): + max_keys = _MAX_PREVIEW_ITEMS + max_keys = min(max(max_keys, 1), _MAX_PREVIEW_ITEMS) + bucket = parameters.get("bucket") + prefix = parameters.get("prefix") + raw_next_page_parameters = parameters.get("next_page_parameters") + request = OnlineDriveBrowseFilesRequest( + bucket=bucket if isinstance(bucket, str) else None, + prefix=prefix if isinstance(prefix, str) else "", + max_keys=max_keys, + next_page_parameters=( + cast(dict[str, Any], raw_next_page_parameters) if isinstance(raw_next_page_parameters, dict) else None + ), + ) + for drive_message in drive_runtime.online_drive_browse_files( + user_id=account.id, + request=request, + provider_type=drive_runtime.datasource_provider_type(), + ): + for group in drive_message.result: + if group.is_truncated and group.next_page_parameters: + next_page_parameters = group.next_page_parameters + if group.bucket and not group.files: + files.append( + KnowledgeFSInitialSourcePreviewFileResponse( + bucket=group.bucket, + id="", + name=group.bucket, + provider_item_id=json.dumps([group.bucket, ""], separators=(",", ":")), + size=0, + type="bucket", + ) + ) + for file in group.files: + files.append( + KnowledgeFSInitialSourcePreviewFileResponse( + bucket=group.bucket, + id=file.id, + mime_type=file.type if "/" in file.type else None, + name=file.name, + provider_item_id=json.dumps([group.bucket or "", file.id], separators=(",", ":")), + size=file.size, + type=file.type, + ) + ) + if len(files) >= _MAX_PREVIEW_ITEMS: + return KnowledgeFSInitialSourcePreviewResponse( + files=files, + kind=payload.kind, + next_page_parameters=next_page_parameters, + ) + return KnowledgeFSInitialSourcePreviewResponse( + files=files, + kind=payload.kind, + next_page_parameters=next_page_parameters, + ) + + +__all__ = ["KnowledgeFSInitialSourcePreviewService"] diff --git a/api/services/knowledge_fs/product_application_service.py b/api/services/knowledge_fs/product_application_service.py index e44735eae49..1d95a7f9344 100644 --- a/api/services/knowledge_fs/product_application_service.py +++ b/api/services/knowledge_fs/product_application_service.py @@ -111,14 +111,14 @@ class KnowledgeFSProductApplicationService: visibility=payload.visibility, ) if payload.initial_source is not None: - from tasks.knowledge_fs_initial_source_tasks import import_initial_website_source + from tasks.knowledge_fs_initial_source_tasks import import_initial_source - import_initial_website_source.delay( + import_initial_source.delay( tenant_id=tenant_id, account_id=account_id, control_space_id=result.control_space.id, operation_id=operation_id, - payload=payload.initial_source.model_dump(mode="json"), + payload=payload.initial_source.model_dump(mode="json", exclude_none=True), ) return KnowledgeFSSpaceCreateResponse( control_space_id=result.control_space.id, diff --git a/api/services/knowledge_fs/product_dto.py b/api/services/knowledge_fs/product_dto.py index 906e72a6a2c..92fadcf8707 100644 --- a/api/services/knowledge_fs/product_dto.py +++ b/api/services/knowledge_fs/product_dto.py @@ -113,16 +113,51 @@ class KnowledgeFSInitialWebsiteSelectionPayload(BaseModel): return source_url.strip() +class KnowledgeFSOnlineDocumentWorkflowImportItemPayload(BaseModel): + etag: str | None = Field(default=None, max_length=1_024) + last_edited_time: str | None = Field(default=None, max_length=128, alias="lastEditedTime") + name: str | None = Field(default=None, max_length=500) + page_id: str = Field(min_length=1, max_length=1_024, alias="pageId") + provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId") + type: str = Field(min_length=1, max_length=128) + workspace_id: str = Field(min_length=1, max_length=1_024, alias="workspaceId") + + model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) + + +class KnowledgeFSOnlineDriveWorkflowImportItemPayload(BaseModel): + bucket: str | None = Field(default=None, max_length=1_024) + etag: str | None = Field(default=None, max_length=1_024) + id: str = Field(min_length=1, max_length=1_024) + mime_type: str | None = Field(default=None, max_length=255, alias="mimeType") + name: str = Field(min_length=1, max_length=500) + provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId") + + model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) + + +class KnowledgeFSInitialDatasourceBindingPayload(BaseModel): + credential_id: str | None = Field(default=None, min_length=1, max_length=255, alias="credentialId") + datasource: str = Field(min_length=1, max_length=255) + plugin_id: str = Field(min_length=1, max_length=255, alias="pluginId") + provider: str = Field(min_length=1, max_length=255) + + model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) + + class KnowledgeFSInitialWebsiteSourcePayload(BaseModel): kind: Literal["website_crawl"] name: str = Field(min_length=1, max_length=200) - provider: Literal["firecrawl"] + provider: str = Field(min_length=1, max_length=255) + plugin_id: str | None = Field(default=None, min_length=1, max_length=255, alias="pluginId") + datasource: str = Field(default="crawl", min_length=1, max_length=255) + credential_id: str | None = Field(default=None, min_length=1, max_length=255, alias="credentialId") root_url: str = Field(min_length=1, max_length=4_096) crawl_options: KnowledgeFSInitialWebsiteCrawlOptionsPayload selection: list[KnowledgeFSInitialWebsiteSelectionPayload] = Field(min_length=1, max_length=200) sync_policy: Literal["provider", "daily", "manual"] = "provider" - model_config = ConfigDict(extra="forbid") + model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) @model_validator(mode="after") def validate_selection(self) -> KnowledgeFSInitialWebsiteSourcePayload: @@ -132,6 +167,68 @@ class KnowledgeFSInitialWebsiteSourcePayload(BaseModel): return self +class KnowledgeFSInitialOnlineDocumentSourcePayload(KnowledgeFSInitialDatasourceBindingPayload): + credential_id: str = Field(min_length=1, max_length=255, alias="credentialId") + kind: Literal["online_document"] + name: str = Field(min_length=1, max_length=200) + selection: list[KnowledgeFSOnlineDocumentWorkflowImportItemPayload] = Field(min_length=1, max_length=200) + sync_policy: Literal["provider", "daily", "manual"] = "provider" + + +class KnowledgeFSInitialOnlineDriveSourcePayload(KnowledgeFSInitialDatasourceBindingPayload): + credential_id: str = Field(min_length=1, max_length=255, alias="credentialId") + kind: Literal["online_drive"] + name: str = Field(min_length=1, max_length=200) + selection: list[KnowledgeFSOnlineDriveWorkflowImportItemPayload] = Field(min_length=1, max_length=200) + sync_policy: Literal["provider", "daily", "manual"] = "provider" + + +class KnowledgeFSInitialSourcePreviewPayload(KnowledgeFSInitialDatasourceBindingPayload): + credential_id: str = Field(min_length=1, max_length=255, alias="credentialId") + kind: Literal["online_document", "online_drive"] + parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50) + + +class KnowledgeFSInitialSourcePreviewDocumentResponse(ResponseModel): + last_edited_time: str | None = Field( + default=None, validation_alias=AliasChoices("last_edited_time", "lastEditedTime") + ) + name: str + page_id: str = Field(validation_alias=AliasChoices("page_id", "pageId")) + provider_item_id: str = Field(validation_alias=AliasChoices("provider_item_id", "providerItemId")) + type: str + workspace_id: str = Field(validation_alias=AliasChoices("workspace_id", "workspaceId")) + workspace_name: str | None = Field(default=None, validation_alias=AliasChoices("workspace_name", "workspaceName")) + + +class KnowledgeFSInitialSourcePreviewFileResponse(ResponseModel): + bucket: str | None = None + id: str + mime_type: str | None = Field(default=None, validation_alias=AliasChoices("mime_type", "mimeType")) + name: str + provider_item_id: str = Field(validation_alias=AliasChoices("provider_item_id", "providerItemId")) + size: int = Field(ge=0) + type: str + + +class KnowledgeFSInitialSourcePreviewResponse(ResponseModel): + documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = Field(default_factory=list) + files: list[KnowledgeFSInitialSourcePreviewFileResponse] = Field(default_factory=list) + kind: Literal["online_document", "online_drive"] + next_page_parameters: dict[str, JsonValue] | None = Field( + default=None, + validation_alias=AliasChoices("next_page_parameters", "nextPageParameters"), + ) + + +KnowledgeFSInitialSourcePayload = Annotated[ + KnowledgeFSInitialWebsiteSourcePayload + | KnowledgeFSInitialOnlineDocumentSourcePayload + | KnowledgeFSInitialOnlineDriveSourcePayload, + Field(discriminator="kind"), +] + + class KnowledgeFSSpaceCreatePayload(BaseModel): name: str = Field(min_length=1, max_length=40) slug: str = Field(min_length=1, max_length=160, pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$") @@ -141,7 +238,7 @@ class KnowledgeFSSpaceCreatePayload(BaseModel): embedding: KnowledgeFSModelIntent | None = None retrieval: KnowledgeFSRetrievalProfileIntent | None = None idempotency_key: str | None = Field(default=None, min_length=1, max_length=255) - initial_source: KnowledgeFSInitialWebsiteSourcePayload | None = None + initial_source: KnowledgeFSInitialSourcePayload | None = None model_config = ConfigDict(extra="forbid") @@ -1573,29 +1670,6 @@ class KnowledgeFSSourceWorkflowResponse(ResponseModel): updated_at: datetime = Field(validation_alias=AliasChoices("updated_at", "updatedAt")) -class KnowledgeFSOnlineDocumentWorkflowImportItemPayload(BaseModel): - etag: str | None = Field(default=None, max_length=1_024) - last_edited_time: str | None = Field(default=None, max_length=128, alias="lastEditedTime") - name: str | None = Field(default=None, max_length=500) - page_id: str = Field(min_length=1, max_length=1_024, alias="pageId") - provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId") - type: str = Field(min_length=1, max_length=128) - workspace_id: str = Field(min_length=1, max_length=1_024, alias="workspaceId") - - model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) - - -class KnowledgeFSOnlineDriveWorkflowImportItemPayload(BaseModel): - bucket: str | None = Field(default=None, max_length=1_024) - etag: str | None = Field(default=None, max_length=1_024) - id: str = Field(min_length=1, max_length=1_024) - mime_type: str | None = Field(default=None, max_length=255, alias="mimeType") - name: str = Field(min_length=1, max_length=500) - provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId") - - model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) - - class KnowledgeFSOnlineDocumentWorkflowImportPayload(BaseModel): items: list[KnowledgeFSOnlineDocumentWorkflowImportItemPayload] = Field(min_length=1, max_length=200) kind: Literal["online-document-import"] @@ -2798,6 +2872,14 @@ __all__ = [ "KnowledgeFSExternalAccessPayload", "KnowledgeFSExternalAccessResponse", "KnowledgeFSIdempotencyHeader", + "KnowledgeFSInitialDatasourceBindingPayload", + "KnowledgeFSInitialOnlineDocumentSourcePayload", + "KnowledgeFSInitialOnlineDriveSourcePayload", + "KnowledgeFSInitialSourcePayload", + "KnowledgeFSInitialSourcePreviewDocumentResponse", + "KnowledgeFSInitialSourcePreviewFileResponse", + "KnowledgeFSInitialSourcePreviewPayload", + "KnowledgeFSInitialSourcePreviewResponse", "KnowledgeFSInitialWebsiteCrawlOptionsPayload", "KnowledgeFSInitialWebsiteSelectionPayload", "KnowledgeFSInitialWebsiteSourcePayload", @@ -2809,6 +2891,8 @@ __all__ = [ "KnowledgeFSMemberBindingPayload", "KnowledgeFSMembersReplacePayload", "KnowledgeFSModelIntent", + "KnowledgeFSOnlineDocumentWorkflowImportItemPayload", + "KnowledgeFSOnlineDriveWorkflowImportItemPayload", "KnowledgeFSOverviewBaseStatsResponse", "KnowledgeFSOverviewCountComparisonResponse", "KnowledgeFSOverviewHealthComponentResponse", diff --git a/api/tasks/knowledge_fs_initial_source_tasks.py b/api/tasks/knowledge_fs_initial_source_tasks.py index b8d3dbaed00..9b273d652a6 100644 --- a/api/tasks/knowledge_fs_initial_source_tasks.py +++ b/api/tasks/knowledge_fs_initial_source_tasks.py @@ -1,44 +1,110 @@ -"""Durable follow-up that starts the first website import after Space provisioning.""" +"""Durable follow-up that imports the first Source after Space provisioning.""" from __future__ import annotations import logging +from dataclasses import dataclass +from typing import Literal from celery import shared_task +from pydantic import TypeAdapter from sqlalchemy import select from core.db.session_factory import session_factory +from models.account import Account +from models.credential_permission import CredentialType from models.knowledge_fs import KnowledgeFSControlSpaceState from models.oauth import DatasourceProvider from repositories.sqlalchemy_knowledge_fs_control_space_repository import ( SQLAlchemyKnowledgeFSControlSpaceRepository, ) +from services.credential_permission_service import CredentialPermissionService from services.knowledge_fs.product_dto import ( KnowledgeFSCrawlImportPayload, + KnowledgeFSInitialOnlineDocumentSourcePayload, + KnowledgeFSInitialSourcePayload, KnowledgeFSInitialWebsiteSourcePayload, + KnowledgeFSOnlineDocumentWorkflowImportPayload, + KnowledgeFSOnlineDriveWorkflowImportPayload, KnowledgeFSSourceConnectionCreatePayload, KnowledgeFSSourceCreatePayload, KnowledgeFSSourceSyncPolicyPayload, KnowledgeFSSourceUpdatePayload, + KnowledgeFSSourceWorkflowImportPayload, ) from services.knowledge_fs.product_remote import KnowledgeFSProductResourceNotFoundError from services.knowledge_fs.runtime import get_knowledge_fs_runtime -_FIRECRAWL_PROVIDER_ID = "plugin-daemon-website" -_FIRECRAWL_PLUGIN_ID = "langgenius/firecrawl_datasource" +_LEGACY_WEBSITE_PLUGIN_IDS = { + "firecrawl": "langgenius/firecrawl_datasource", + "jinareader": "langgenius/jina_datasource", + "watercrawl": "watercrawl/watercrawl_datasource", +} _PAGE_SIZE = 200 +_INITIAL_SOURCE_ADAPTER: TypeAdapter[KnowledgeFSInitialSourcePayload] = TypeAdapter(KnowledgeFSInitialSourcePayload) logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class _DatasourceBinding: + credential_id: str | None + datasource: str + plugin_id: str + provider: str + provider_id: str + provider_kind: str + + class KnowledgeFSInitialSourceNotReadyError(RuntimeError): - """The Space or Source workflow is still progressing and should be retried.""" + """The Space, connection, or Source workflow is progressing and should be retried.""" def __init__(self, message: str, *, workflow_id: str | None = None) -> None: super().__init__(message) self.workflow_id = workflow_id +def _binding(payload: KnowledgeFSInitialSourcePayload) -> _DatasourceBinding: + if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload): + normalized_provider = "".join(character for character in payload.provider.lower() if character.isalnum()) + plugin_id = payload.plugin_id or _LEGACY_WEBSITE_PLUGIN_IDS.get(normalized_provider) + if plugin_id is None: + raise RuntimeError("Website datasource plugin binding is required") + return _DatasourceBinding( + credential_id=payload.credential_id, + datasource=payload.datasource, + plugin_id=plugin_id, + provider=payload.provider, + provider_id="plugin-daemon-website", + provider_kind="website", + ) + if isinstance(payload, KnowledgeFSInitialOnlineDocumentSourcePayload): + return _DatasourceBinding( + credential_id=payload.credential_id, + datasource=payload.datasource, + plugin_id=payload.plugin_id, + provider=payload.provider, + provider_id="plugin-daemon-online-document", + provider_kind="online-document", + ) + return _DatasourceBinding( + credential_id=payload.credential_id, + datasource=payload.datasource, + plugin_id=payload.plugin_id, + provider=payload.provider, + provider_id="plugin-daemon-online-drive", + provider_kind="online-drive", + ) + + +def _request_id(*, operation_id: str, payload: KnowledgeFSInitialSourcePayload) -> str: + # Preserve the original website request ID so retries from the previous rollout + # reconcile with the same provisional Source. + if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload): + return f"initial-website-source:{operation_id}" + return f"initial-source:{operation_id}" + + def _find_initial_source(*, facade, tenant_id: str, account_id: str, control_space_id: str, request_id: str): cursor: str | None = None while True: @@ -57,29 +123,41 @@ def _find_initial_source(*, facade, tenant_id: str, account_id: str, control_spa cursor = response.next_cursor -def _find_firecrawl_credential(*, session_maker, tenant_id: str) -> tuple[str, str]: +def _find_credential(*, session_maker, tenant_id: str, account_id: str, binding: _DatasourceBinding) -> tuple[str, str]: + query = select(DatasourceProvider).where( + DatasourceProvider.tenant_id == tenant_id, + DatasourceProvider.provider == binding.provider, + DatasourceProvider.plugin_id == binding.plugin_id, + ) with session_maker() as session: + account = session.get(Account, account_id) + if account is None: + raise RuntimeError("Initial Source account was not found") + query = CredentialPermissionService.apply_visibility_filter( + query, + model_id_column=DatasourceProvider.id, + model_user_id_column=DatasourceProvider.user_id, + model_visibility_column=DatasourceProvider.visibility, + credential_type=CredentialType.DATASOURCE_PROVIDER, + user=account, + ) + if binding.credential_id is not None: + query = query.where(DatasourceProvider.id == binding.credential_id) credential = session.scalar( - select(DatasourceProvider) - .where( - DatasourceProvider.tenant_id == tenant_id, - DatasourceProvider.provider == "firecrawl", - DatasourceProvider.plugin_id == _FIRECRAWL_PLUGIN_ID, - ) - .order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc()) - .limit(1) + query.order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc()).limit(1) ) if credential is None: - raise RuntimeError("Firecrawl credential is unavailable") - return str(credential.id), credential.name or "Firecrawl" + raise RuntimeError("Datasource credential is unavailable") + return str(credential.id), credential.name or binding.provider -def _find_or_create_firecrawl_connection( +def _find_or_create_connection( *, facade, tenant_id: str, account_id: str, control_space_id: str, + binding: _DatasourceBinding, credential_id: str, credential_name: str, ): @@ -88,9 +166,16 @@ def _find_or_create_firecrawl_connection( account_id=account_id, control_space_id=control_space_id, ) - if not any(provider.id == _FIRECRAWL_PROVIDER_ID and provider.available for provider in providers.data): - raise RuntimeError("Firecrawl provider is unavailable") + if not any(provider.id == binding.provider_id and provider.available for provider in providers.data): + raise RuntimeError(f"{binding.provider_kind} provider is unavailable") + expected_configuration: dict[str, bool | int | str] = { + "credentialId": credential_id, + "datasource": binding.datasource, + "pluginId": binding.plugin_id, + "provider": binding.provider, + "providerKind": binding.provider_kind, + } cursor: str | None = None while True: response = facade.list_source_connections( @@ -101,16 +186,15 @@ def _find_or_create_firecrawl_connection( limit=_PAGE_SIZE, ) for connection in response.data: - if ( - connection.provider_id != _FIRECRAWL_PROVIDER_ID - or connection.configuration.get("credentialId") != credential_id - ): + if connection.provider_id != binding.provider_id: + continue + if any(connection.configuration.get(key) != value for key, value in expected_configuration.items()): continue if connection.status == "active": return connection if connection.status == "provisioning": - raise KnowledgeFSInitialSourceNotReadyError("Firecrawl connection is still provisioning") - raise RuntimeError(f"Firecrawl connection is unavailable in state {connection.status}") + raise KnowledgeFSInitialSourceNotReadyError("Datasource connection is still provisioning") + raise RuntimeError(f"Datasource connection is unavailable in state {connection.status}") if not response.next_cursor: break cursor = response.next_cursor @@ -121,33 +205,131 @@ def _find_or_create_firecrawl_connection( control_space_id=control_space_id, payload=KnowledgeFSSourceConnectionCreatePayload( authKind="endpoint", - configuration={ - "credentialId": credential_id, - "datasource": "crawl", - "pluginId": _FIRECRAWL_PLUGIN_ID, - "provider": "firecrawl", - "providerKind": "website", - }, + configuration=expected_configuration, credentials={}, name=credential_name, - providerId=_FIRECRAWL_PROVIDER_ID, + providerId=binding.provider_id, ), ) if connection.status != "active": - raise KnowledgeFSInitialSourceNotReadyError("Firecrawl connection is still provisioning") + raise KnowledgeFSInitialSourceNotReadyError("Datasource connection is still provisioning") return connection -def start_initial_website_source_import( +def _source_payload( + *, + payload: KnowledgeFSInitialSourcePayload, + binding: _DatasourceBinding, + connection_id: str, + request_id: str, +) -> KnowledgeFSSourceCreatePayload: + metadata: dict[str, object] = { + "clientRequestId": request_id, + "preview": True, + "providerId": binding.provider_id, + "providerKind": binding.provider_kind, + "providerName": payload.provider, + } + if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload): + metadata["crawlOptions"] = { + "includeSubpages": payload.crawl_options.include_subpages, + "limit": payload.crawl_options.limit, + } + source_type: Literal["connector", "web"] = "web" + uri = payload.root_url + else: + source_type = "connector" + uri = f"connector://{connection_id}" + return KnowledgeFSSourceCreatePayload( + connectionId=connection_id, + metadata=metadata, + name=payload.name, + status="disabled", + type=source_type, + uri=uri, + ) + + +def _start_workflow( + *, + facade, + tenant_id: str, + account_id: str, + control_space_id: str, + source_id: str, + request_id: str, + payload: KnowledgeFSInitialSourcePayload, +): + if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload): + return facade.import_selected_source_crawl( + tenant_id=tenant_id, + account_id=account_id, + control_space_id=control_space_id, + source_id=source_id, + payload=KnowledgeFSCrawlImportPayload( + sourceUrls=[selection.source_url for selection in payload.selection], + ), + idempotency_key=f"{request_id}:crawl-import", + ) + if isinstance(payload, KnowledgeFSInitialOnlineDocumentSourcePayload): + import_payload = KnowledgeFSSourceWorkflowImportPayload( + KnowledgeFSOnlineDocumentWorkflowImportPayload( + kind="online-document-import", + items=payload.selection, + ) + ) + else: + import_payload = KnowledgeFSSourceWorkflowImportPayload( + KnowledgeFSOnlineDriveWorkflowImportPayload( + kind="online-drive-import", + items=payload.selection, + ) + ) + return facade.import_source_workflow( + tenant_id=tenant_id, + account_id=account_id, + control_space_id=control_space_id, + source_id=source_id, + payload=import_payload, + idempotency_key=f"{request_id}:connector-import", + ) + + +def _sync_policy_payload( + *, payload: KnowledgeFSInitialSourcePayload, expected_revision: int, source_version: int +) -> KnowledgeFSSourceSyncPolicyPayload: + if payload.sync_policy == "manual": + return KnowledgeFSSourceSyncPolicyPayload( + enabled=False, + mode="manual", + expectedRevision=expected_revision, + expectedSourceVersion=source_version, + ) + if payload.sync_policy == "daily": + return KnowledgeFSSourceSyncPolicyPayload( + enabled=True, + mode="interval", + expectedRevision=expected_revision, + expectedSourceVersion=source_version, + ) + return KnowledgeFSSourceSyncPolicyPayload( + enabled=True, + mode="provider", + expectedRevision=expected_revision, + expectedSourceVersion=source_version, + ) + + +def start_initial_source_import( *, tenant_id: str, account_id: str, control_space_id: str, operation_id: str, - payload: KnowledgeFSInitialWebsiteSourcePayload, + payload: KnowledgeFSInitialSourcePayload, workflow_id: str | None = None, ) -> str: - """Idempotently create the provisional Source and start its selected crawl import.""" + """Idempotently create a provisional Source, import its selection, and commit it.""" session_maker = session_factory.get_session_maker() with session_maker() as session: @@ -173,10 +355,10 @@ def start_initial_website_source_import( run_id=workflow_id, ) if workflow.source_id is None: - raise RuntimeError("Initial website Source import workflow has no Source") + raise RuntimeError("Initial Source import workflow has no Source") source_id = workflow.source_id else: - request_id = f"initial-website-source:{operation_id}" + request_id = _request_id(operation_id=operation_id, payload=payload) source = _find_initial_source( facade=facade, tenant_id=tenant_id, @@ -185,15 +367,19 @@ def start_initial_website_source_import( request_id=request_id, ) if source is None: - credential_id, credential_name = _find_firecrawl_credential( + binding = _binding(payload) + credential_id, credential_name = _find_credential( session_maker=session_maker, tenant_id=tenant_id, + account_id=account_id, + binding=binding, ) - connection = _find_or_create_firecrawl_connection( + connection = _find_or_create_connection( facade=facade, tenant_id=tenant_id, account_id=account_id, control_space_id=control_space_id, + binding=binding, credential_id=credential_id, credential_name=credential_name, ) @@ -201,38 +387,27 @@ def start_initial_website_source_import( tenant_id=tenant_id, account_id=account_id, control_space_id=control_space_id, - payload=KnowledgeFSSourceCreatePayload( - connectionId=connection.id, - metadata={ - "clientRequestId": request_id, - "crawlOptions": { - "includeSubpages": payload.crawl_options.include_subpages, - "limit": payload.crawl_options.limit, - }, - "preview": True, - "providerId": _FIRECRAWL_PROVIDER_ID, - }, - name=payload.name, - status="disabled", - type="web", - uri=payload.root_url, + payload=_source_payload( + payload=payload, + binding=binding, + connection_id=connection.id, + request_id=request_id, ), ) source_id = source.id - workflow = facade.import_selected_source_crawl( + workflow = _start_workflow( + facade=facade, tenant_id=tenant_id, account_id=account_id, control_space_id=control_space_id, source_id=source_id, - payload=KnowledgeFSCrawlImportPayload( - sourceUrls=[selection.source_url for selection in payload.selection], - ), - idempotency_key=f"{request_id}:crawl-import", + request_id=request_id, + payload=payload, ) if workflow.state in {"queued", "running", "crawling", "importing", "syncing"}: raise KnowledgeFSInitialSourceNotReadyError( - "Initial website Source import is still running", + "Initial Source import is still running", workflow_id=workflow.id, ) if workflow.state != "completed": @@ -269,7 +444,7 @@ def start_initial_website_source_import( ), ) logger.error( - "Initial website Source import failed", + "Initial Source import failed", extra={ "control_space_id": control_space_id, "error_code": workflow.last_error_code, @@ -311,43 +486,99 @@ def start_initial_website_source_import( expected_revision = current_policy.revision except KnowledgeFSProductResourceNotFoundError: expected_revision = 0 - if payload.sync_policy == "manual": - sync_policy = KnowledgeFSSourceSyncPolicyPayload( - enabled=False, - mode="manual", - expectedRevision=expected_revision, - expectedSourceVersion=committed_source.version, - ) - elif payload.sync_policy == "daily": - sync_policy = KnowledgeFSSourceSyncPolicyPayload( - enabled=True, - mode="interval", - expectedRevision=expected_revision, - expectedSourceVersion=committed_source.version, - ) - else: - sync_policy = KnowledgeFSSourceSyncPolicyPayload( - enabled=True, - mode="provider", - expectedRevision=expected_revision, - expectedSourceVersion=committed_source.version, - ) facade.update_source_sync_policy( tenant_id=tenant_id, account_id=account_id, control_space_id=control_space_id, source_id=source_id, - payload=sync_policy, + payload=_sync_policy_payload( + payload=payload, + expected_revision=expected_revision, + source_version=committed_source.version, + ), ) return workflow.id -@shared_task( - bind=True, - queue="knowledge_fs_lifecycle", - max_retries=180, - default_retry_delay=2, -) +def start_initial_website_source_import( + *, + tenant_id: str, + account_id: str, + control_space_id: str, + operation_id: str, + payload: KnowledgeFSInitialWebsiteSourcePayload, + workflow_id: str | None = None, +) -> str: + """Compatibility wrapper for callers using the original website-only protocol.""" + + return start_initial_source_import( + tenant_id=tenant_id, + account_id=account_id, + control_space_id=control_space_id, + operation_id=operation_id, + payload=payload, + workflow_id=workflow_id, + ) + + +def _run_initial_source_task( + task, + *, + tenant_id: str, + account_id: str, + control_space_id: str, + operation_id: str, + payload: dict[str, object], + workflow_id: str | None, +) -> str: + try: + return start_initial_source_import( + tenant_id=tenant_id, + account_id=account_id, + control_space_id=control_space_id, + operation_id=operation_id, + payload=_INITIAL_SOURCE_ADAPTER.validate_python(payload), + workflow_id=workflow_id, + ) + except KnowledgeFSInitialSourceNotReadyError as exc: + if exc.workflow_id is not None: + raise task.retry( + exc=exc, + kwargs={ + "tenant_id": tenant_id, + "account_id": account_id, + "control_space_id": control_space_id, + "operation_id": operation_id, + "payload": payload, + "workflow_id": exc.workflow_id, + }, + ) + raise task.retry(exc=exc) + + +@shared_task(bind=True, queue="knowledge_fs_lifecycle", max_retries=180, default_retry_delay=2) +def import_initial_source( + self, + *, + tenant_id: str, + account_id: str, + control_space_id: str, + operation_id: str, + payload: dict[str, object], + workflow_id: str | None = None, +) -> str: + return _run_initial_source_task( + self, + tenant_id=tenant_id, + account_id=account_id, + control_space_id=control_space_id, + operation_id=operation_id, + payload=payload, + workflow_id=workflow_id, + ) + + +@shared_task(bind=True, queue="knowledge_fs_lifecycle", max_retries=180, default_retry_delay=2) def import_initial_website_source( self, *, @@ -358,6 +589,8 @@ def import_initial_website_source( payload: dict[str, object], workflow_id: str | None = None, ) -> str: + """Compatibility task for already-enqueued website-only messages.""" + try: return start_initial_website_source_import( tenant_id=tenant_id, @@ -383,4 +616,9 @@ def import_initial_website_source( raise self.retry(exc=exc) -__all__ = ["import_initial_website_source", "start_initial_website_source_import"] +__all__ = [ + "import_initial_source", + "import_initial_website_source", + "start_initial_source_import", + "start_initial_website_source_import", +] diff --git a/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py b/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py new file mode 100644 index 00000000000..de25f1bf310 --- /dev/null +++ b/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py @@ -0,0 +1,184 @@ +from types import SimpleNamespace +from typing import cast +from unittest.mock import MagicMock, patch + +import pytest + +from core.datasource.entities.datasource_entities import ( + OnlineDocumentInfo, + OnlineDocumentPage, + OnlineDocumentPagesMessage, + OnlineDriveBrowseFilesResponse, + OnlineDriveFile, + OnlineDriveFileBucket, +) +from models.account import Account +from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService +from services.knowledge_fs.product_dto import KnowledgeFSInitialSourcePreviewPayload + +_CREDENTIAL = object() + + +def _service(credential=_CREDENTIAL) -> tuple[KnowledgeFSInitialSourcePreviewService, MagicMock]: + session = MagicMock() + session.scalar.return_value = credential + context = MagicMock() + context.__enter__.return_value = session + return KnowledgeFSInitialSourcePreviewService(MagicMock(return_value=context)), session + + +def _payload(kind: str) -> KnowledgeFSInitialSourcePreviewPayload: + return KnowledgeFSInitialSourcePreviewPayload.model_validate( + { + "credentialId": "credential-1", + "datasource": "pages" if kind == "online_document" else "drive", + "kind": kind, + "parameters": {}, + "pluginId": "langgenius/provider", + "provider": "provider", + } + ) + + +def test_preview_lists_online_documents_with_stable_import_identity() -> None: + service, _session = _service() + runtime = MagicMock() + runtime.datasource_provider_type.return_value = "online_document" + runtime.get_online_document_pages.return_value = [ + OnlineDocumentPagesMessage( + result=[ + OnlineDocumentInfo( + pages=[ + OnlineDocumentPage( + last_edited_time="2026-08-10T00:00:00Z", + page_id="page-1", + page_name="Roadmap", + parent_id=None, + type="page", + ) + ], + total=1, + workspace_id="workspace-1", + workspace_name="Product", + ) + ] + ) + ] + with ( + patch( + "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials", + return_value={"token": "secret"}, + ), + patch( + "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime", + return_value=runtime, + ), + ): + response = service.preview( + tenant_id="tenant-1", + account=cast(Account, SimpleNamespace(id="account-1")), + payload=_payload("online_document"), + ) + + assert response.kind == "online_document" + assert response.documents[0].provider_item_id == '["workspace-1","page-1"]' + assert response.documents[0].name == "Roadmap" + assert response.files == [] + assert runtime.runtime.credentials == {"token": "secret"} + + +def test_preview_browses_online_drive_and_preserves_pagination() -> None: + service, _session = _service() + runtime = MagicMock() + runtime.datasource_provider_type.return_value = "online_drive" + runtime.online_drive_browse_files.return_value = [ + OnlineDriveBrowseFilesResponse( + result=[ + OnlineDriveFileBucket( + bucket="manuals", + files=[ + OnlineDriveFile( + id="file-1", + name="Plan.pdf", + size=128, + type="application/pdf", + ) + ], + is_truncated=True, + next_page_parameters={"cursor": "next"}, + ) + ] + ) + ] + payload = _payload("online_drive") + payload.parameters = {"prefix": "folder-1"} + with ( + patch( + "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials", + return_value={"token": "secret"}, + ), + patch( + "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime", + return_value=runtime, + ), + ): + response = service.preview( + tenant_id="tenant-1", + account=cast(Account, SimpleNamespace(id="account-1")), + payload=payload, + ) + + assert response.kind == "online_drive" + assert response.files[0].provider_item_id == '["manuals","file-1"]' + assert response.files[0].mime_type == "application/pdf" + assert response.next_page_parameters == {"cursor": "next"} + request = runtime.online_drive_browse_files.call_args.kwargs["request"] + assert request.prefix == "folder-1" + + +def test_preview_exposes_an_empty_drive_bucket_as_a_browsable_container() -> None: + service, _session = _service() + runtime = MagicMock() + runtime.datasource_provider_type.return_value = "online_drive" + runtime.online_drive_browse_files.return_value = [ + OnlineDriveBrowseFilesResponse( + result=[OnlineDriveFileBucket(bucket="manuals", files=[], is_truncated=False)] + ) + ] + with ( + patch( + "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials", + return_value={"token": "secret"}, + ), + patch( + "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime", + return_value=runtime, + ), + ): + response = service.preview( + tenant_id="tenant-1", + account=cast(Account, SimpleNamespace(id="account-1")), + payload=_payload("online_drive"), + ) + + assert response.files[0].bucket == "manuals" + assert response.files[0].id == "" + assert response.files[0].provider_item_id == '["manuals",""]' + assert response.files[0].type == "bucket" + + +def test_preview_rejects_a_credential_hidden_from_the_account() -> None: + service, _session = _service(credential=None) + with ( + patch( + "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials" + ) as get_credentials, + pytest.raises(PermissionError, match="credential is unavailable"), + ): + service.preview( + tenant_id="tenant-1", + account=cast(Account, SimpleNamespace(id="account-1")), + payload=_payload("online_document"), + ) + + get_credentials.assert_not_called() diff --git a/api/tests/unit_tests/services/test_knowledge_fs_product_application_service.py b/api/tests/unit_tests/services/test_knowledge_fs_product_application_service.py index 201e4de47d6..6bff2aac1b8 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_product_application_service.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_product_application_service.py @@ -245,7 +245,7 @@ def test_product_application_create_schedules_selected_website_import() -> None: "services.knowledge_fs.product_application_service.uuid.uuid5", return_value="operation-1", ), - patch("tasks.knowledge_fs_initial_source_tasks.import_initial_website_source.delay") as schedule_import, + patch("tasks.knowledge_fs_initial_source_tasks.import_initial_source.delay") as schedule_import, ): application.create_space(tenant_id="tenant-1", account_id="account-1", payload=payload) @@ -258,6 +258,7 @@ def test_product_application_create_schedules_selected_website_import() -> None: "kind": "website_crawl", "name": "Dify docs", "provider": "firecrawl", + "datasource": "crawl", "root_url": "https://docs.dify.ai", "crawl_options": {"include_subpages": True, "limit": 25}, "selection": [ @@ -271,6 +272,58 @@ def test_product_application_create_schedules_selected_website_import() -> None: ) +def test_product_application_create_schedules_selected_connector_import() -> None: + application, _product, _control_plane, _commands, _facade, _rbac = _application() + payload = _create_payload( + initial_source={ + "kind": "online_document", + "name": "Product wiki", + "pluginId": "langgenius/notion_datasource", + "provider": "notion", + "datasource": "pages", + "credentialId": "credential-1", + "selection": [ + { + "name": "Roadmap", + "pageId": "page-1", + "providerItemId": "notion:page-1", + "type": "page", + "workspaceId": "workspace-1", + } + ], + } + ) + + with ( + patch( + "services.knowledge_fs.product_application_service.uuid.uuid5", + return_value="operation-1", + ), + patch("tasks.knowledge_fs_initial_source_tasks.import_initial_source.delay") as schedule_import, + ): + application.create_space(tenant_id="tenant-1", account_id="account-1", payload=payload) + + scheduled_payload = schedule_import.call_args.kwargs["payload"] + assert scheduled_payload == { + "credential_id": "credential-1", + "datasource": "pages", + "kind": "online_document", + "name": "Product wiki", + "plugin_id": "langgenius/notion_datasource", + "provider": "notion", + "selection": [ + { + "name": "Roadmap", + "page_id": "page-1", + "provider_item_id": "notion:page-1", + "type": "page", + "workspace_id": "workspace-1", + } + ], + "sync_policy": "provider", + } + + def test_product_application_create_generates_idempotency_and_skips_default_visibility_update() -> None: application, _product, control_plane, commands, _facade, _rbac = _application() diff --git a/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py b/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py index fba36a4a911..326effb124e 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_product_dto.py @@ -45,6 +45,7 @@ from services.knowledge_fs.product_dto import ( KnowledgeFSSourceListQuery, KnowledgeFSSourceUpdatePayload, KnowledgeFSSourceWorkflowImportPayload, + KnowledgeFSSpaceCreatePayload, KnowledgeFSSpaceListItemResponse, KnowledgeFSStatResponse, KnowledgeFSTraceResponse, @@ -55,6 +56,75 @@ from services.knowledge_fs.product_dto import ( ) +def test_space_create_initial_source_is_a_backward_compatible_discriminated_union() -> None: + website = KnowledgeFSSpaceCreatePayload.model_validate( + { + "name": "Docs", + "slug": "docs", + "initial_source": { + "kind": "website_crawl", + "name": "Website", + "provider": "firecrawl", + "root_url": "https://docs.example.com", + "crawl_options": {}, + "selection": [{"source_url": "https://docs.example.com/start"}], + }, + } + ) + assert website.initial_source is not None + assert website.initial_source.kind == "website_crawl" + + document = KnowledgeFSSpaceCreatePayload.model_validate( + { + "name": "Docs", + "slug": "docs", + "initial_source": { + "kind": "online_document", + "name": "Wiki", + "pluginId": "langgenius/notion_datasource", + "provider": "notion", + "datasource": "pages", + "credentialId": "credential-1", + "selection": [ + { + "pageId": "page-1", + "providerItemId": "notion:page-1", + "type": "page", + "workspaceId": "workspace-1", + } + ], + }, + } + ) + assert document.initial_source is not None + assert document.initial_source.kind == "online_document" + assert document.initial_source.credential_id == "credential-1" + + +def test_connector_initial_source_requires_an_exact_credential_binding() -> None: + with pytest.raises(ValidationError): + KnowledgeFSSpaceCreatePayload.model_validate( + { + "name": "Docs", + "slug": "docs", + "initial_source": { + "kind": "online_drive", + "name": "Drive", + "pluginId": "langgenius/google_drive", + "provider": "google_drive", + "datasource": "google_drive", + "selection": [ + { + "id": "file-1", + "name": "Plan.pdf", + "providerItemId": "google-drive:file-1", + } + ], + }, + } + ) + + def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() -> None: payload = KnowledgeFSRetrievalTestPayload.model_validate( { diff --git a/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py b/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py index 0b5ae0c74bc..9e3dca2aaa6 100644 --- a/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py +++ b/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py @@ -5,11 +5,17 @@ from unittest.mock import MagicMock, patch import pytest from models.knowledge_fs import KnowledgeFSControlSpaceState -from services.knowledge_fs.product_dto import KnowledgeFSInitialWebsiteSourcePayload +from services.knowledge_fs.product_dto import ( + KnowledgeFSInitialOnlineDocumentSourcePayload, + KnowledgeFSInitialOnlineDriveSourcePayload, + KnowledgeFSInitialWebsiteSourcePayload, +) from services.knowledge_fs.product_remote import KnowledgeFSProductResourceNotFoundError from tasks.knowledge_fs_initial_source_tasks import ( KnowledgeFSInitialSourceNotReadyError, + import_initial_source, import_initial_website_source, + start_initial_source_import, start_initial_website_source_import, ) @@ -33,6 +39,51 @@ def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePaylo ) +def _document_payload() -> KnowledgeFSInitialOnlineDocumentSourcePayload: + return KnowledgeFSInitialOnlineDocumentSourcePayload.model_validate( + { + "kind": "online_document", + "name": "Product wiki", + "pluginId": "langgenius/notion_datasource", + "provider": "notion", + "datasource": "pages", + "credentialId": "notion-credential-1", + "selection": [ + { + "lastEditedTime": "2026-08-10T00:00:00Z", + "name": "Roadmap", + "pageId": "page-1", + "providerItemId": "notion:page-1", + "type": "page", + "workspaceId": "workspace-1", + } + ], + } + ) + + +def _drive_payload() -> KnowledgeFSInitialOnlineDriveSourcePayload: + return KnowledgeFSInitialOnlineDriveSourcePayload.model_validate( + { + "kind": "online_drive", + "name": "Team drive", + "pluginId": "langgenius/google_drive", + "provider": "google_drive", + "datasource": "google_drive", + "credentialId": "drive-credential-1", + "selection": [ + { + "id": "file-1", + "mimeType": "application/pdf", + "name": "Plan.pdf", + "providerItemId": "google-drive:file-1", + } + ], + "sync_policy": "manual", + } + ) + + def _facade() -> MagicMock: facade = MagicMock() facade.list_sources.return_value = SimpleNamespace(data=[], next_cursor=None) @@ -42,7 +93,13 @@ def _facade() -> MagicMock: facade.list_source_connections.return_value = SimpleNamespace( data=[ SimpleNamespace( - configuration={"credentialId": "firecrawl-credential-1"}, + configuration={ + "credentialId": "firecrawl-credential-1", + "datasource": "crawl", + "pluginId": "langgenius/firecrawl_datasource", + "provider": "firecrawl", + "providerKind": "website", + }, id="connection-1", provider_id="plugin-daemon-website", status="active", @@ -165,6 +222,85 @@ def test_initial_website_source_import_recrawls_exact_selection_and_configures_d assert sync_payload.expected_source_version == 4 +@pytest.mark.parametrize( + ("payload", "provider_id", "credential_id", "workflow_kind", "expected_source_name"), + [ + ( + _document_payload(), + "plugin-daemon-online-document", + "notion-credential-1", + "online-document-import", + "Product wiki", + ), + ( + _drive_payload(), + "plugin-daemon-online-drive", + "drive-credential-1", + "online-drive-import", + "Team drive", + ), + ], +) +def test_initial_connector_source_import_uses_exact_binding_and_selection( + payload: KnowledgeFSInitialOnlineDocumentSourcePayload | KnowledgeFSInitialOnlineDriveSourcePayload, + provider_id: str, + credential_id: str, + workflow_kind: str, + expected_source_name: str, +) -> None: + facade = _facade() + facade.list_source_providers.return_value = SimpleNamespace(data=[SimpleNamespace(id=provider_id, available=True)]) + facade.list_source_connections.return_value = SimpleNamespace(data=[], next_cursor=None) + facade.create_source_connection.return_value = SimpleNamespace(id="connector-1", status="active") + facade.import_source_workflow.return_value = SimpleNamespace( + id="connector-workflow-1", + source_id="source-1", + state="completed", + ) + facade.get_source.return_value = SimpleNamespace( + metadata={"clientRequestId": "initial-source:operation-1", "preview": True}, + status="disabled", + version=3, + ) + facade.update_source.return_value = SimpleNamespace( + metadata={"clientRequestId": "initial-source:operation-1", "preview": False}, + status="active", + version=4, + ) + credential = SimpleNamespace(id=credential_id, name=expected_source_name) + + with _runtime(facade, credential=credential): + result = start_initial_source_import( + tenant_id="tenant-1", + account_id="account-1", + control_space_id="control-1", + operation_id="operation-1", + payload=payload, + ) + + assert result == "connector-workflow-1" + connection_payload = facade.create_source_connection.call_args.kwargs["payload"] + assert connection_payload.provider_id == provider_id + assert connection_payload.configuration == { + "credentialId": credential_id, + "datasource": payload.datasource, + "pluginId": payload.plugin_id, + "provider": payload.provider, + "providerKind": "online-document" if workflow_kind == "online-document-import" else "online-drive", + } + source_payload = facade.create_source.call_args.kwargs["payload"] + assert source_payload.connection_id == "connector-1" + assert source_payload.name == expected_source_name + assert source_payload.type == "connector" + assert source_payload.uri == "connector://connector-1" + assert source_payload.metadata["providerId"] == provider_id + import_payload = facade.import_source_workflow.call_args.kwargs["payload"].root + assert import_payload.kind == workflow_kind + assert import_payload.items == payload.selection + sync_payload = facade.update_source_sync_policy.call_args.kwargs["payload"] + assert sync_payload.mode == ("manual" if workflow_kind == "online-drive-import" else "provider") + + def test_initial_website_source_import_reuses_source_across_pages_and_preserves_failure() -> None: facade = _facade() existing_source = SimpleNamespace( @@ -216,7 +352,13 @@ def test_initial_website_source_import_configures_remaining_sync_modes( SimpleNamespace( data=[ SimpleNamespace( - configuration={"credentialId": "firecrawl-credential-1"}, + configuration={ + "credentialId": "firecrawl-credential-1", + "datasource": "crawl", + "pluginId": "langgenius/firecrawl_datasource", + "provider": "firecrawl", + "providerKind": "website", + }, id="connection-2", provider_id="plugin-daemon-website", status="active", @@ -365,7 +507,13 @@ def test_initial_website_source_import_retries_provisioning_firecrawl_connection facade.list_source_connections.return_value = SimpleNamespace( data=[ SimpleNamespace( - configuration={"credentialId": "firecrawl-credential-1"}, + configuration={ + "credentialId": "firecrawl-credential-1", + "datasource": "crawl", + "pluginId": "langgenius/firecrawl_datasource", + "provider": "firecrawl", + "providerKind": "website", + }, id="connection-1", provider_id="plugin-daemon-website", status="provisioning", @@ -523,3 +671,25 @@ def test_initial_website_source_task_retries_with_workflow_id() -> None: "workflow_id": "workflow-1", }, ) + + +def test_initial_source_task_validates_discriminated_connector_payload() -> None: + serialized_payload = _document_payload().model_dump(mode="json", by_alias=True) + with patch( + "tasks.knowledge_fs_initial_source_tasks.start_initial_source_import", + return_value="workflow-1", + ) as start_import: + assert ( + import_initial_source.run( + tenant_id="tenant-1", + account_id="account-1", + control_space_id="control-1", + operation_id="operation-1", + payload=serialized_payload, + ) + == "workflow-1" + ) + + parsed_payload = start_import.call_args.kwargs["payload"] + assert isinstance(parsed_payload, KnowledgeFSInitialOnlineDocumentSourcePayload) + assert parsed_payload.credential_id == "notion-credential-1" diff --git a/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts index b9715cdf0cb..b296f17064e 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts @@ -179,6 +179,8 @@ import { zPatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponse, zPostKnowledgeFsQueryStreamBody, zPostKnowledgeFsQueryStreamResponse, + zPostKnowledgeFsSourceProviderPreviewBody, + zPostKnowledgeFsSourceProviderPreviewResponse, zPostKnowledgeFsSpacesBody, zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelPath, zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse, @@ -363,6 +365,21 @@ export const researchTasks = { byTaskId, } +export const post2 = oc + .route({ + inputStructure: 'detailed', + method: 'POST', + operationId: 'postKnowledgeFsSourceProviderPreview', + path: '/knowledge-fs/source-provider-preview', + tags: ['console'], + }) + .input(z.object({ body: zPostKnowledgeFsSourceProviderPreviewBody })) + .output(zPostKnowledgeFsSourceProviderPreviewResponse) + +export const sourceProviderPreview = { + post: post2, +} + export const delete_ = oc .route({ inputStructure: 'detailed', @@ -420,7 +437,7 @@ export const appBindings = { byCallerKind, } -export const post2 = oc +export const post3 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -436,10 +453,10 @@ export const post2 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse) export const cancel = { - post: post2, + post: post3, } -export const post3 = oc +export const post4 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -455,7 +472,7 @@ export const post3 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponse) export const retry = { - post: post3, + post: post4, } export const byTaskId2 = { @@ -536,7 +553,7 @@ export const get6 = oc .input(z.object({ params: zGetKnowledgeFsSpacesByControlSpaceIdCredentialsPath })) .output(zGetKnowledgeFsSpacesByControlSpaceIdCredentialsResponse) -export const post4 = oc +export const post5 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -555,7 +572,7 @@ export const post4 = oc export const credentials = { get: get6, - post: post4, + post: post5, byCredentialId, } @@ -581,7 +598,7 @@ export const bulk = { delete: delete3, } -export const post5 = oc +export const post6 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -598,7 +615,7 @@ export const post5 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdDocumentsReindexResponse) export const reindex = { - post: post5, + post: post6, } export const get7 = oc @@ -761,7 +778,7 @@ export const get12 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdDocumentsResponse) -export const post6 = oc +export const post7 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -780,7 +797,7 @@ export const post6 = oc export const documents = { get: get12, - post: post6, + post: post7, bulk, reindex, byDocumentId, @@ -818,7 +835,7 @@ export const externalAccess = { put: put2, } -export const post7 = oc +export const post8 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -836,10 +853,10 @@ export const post7 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsBulkImportResponse) export const bulkImport = { - post: post7, + post: post8, } -export const post8 = oc +export const post9 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -856,7 +873,7 @@ export const post8 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsEvidenceMatchesResponse) export const evidenceMatches = { - post: post8, + post: post9, } export const delete5 = oc @@ -910,7 +927,7 @@ export const get14 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsResponse) -export const post9 = oc +export const post10 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -929,13 +946,13 @@ export const post9 = oc export const goldenQuestions = { get: get14, - post: post9, + post: post10, bulkImport, evidenceMatches, byQuestionId, } -export const post10 = oc +export const post11 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -947,7 +964,7 @@ export const post10 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponse) export const retry2 = { - post: post10, + post: post11, } export const delete6 = oc @@ -1112,7 +1129,7 @@ export const get18 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdMetadataResponse) -export const post11 = oc +export const post12 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1131,7 +1148,7 @@ export const post11 = oc export const metadata = { get: get18, - post: post11, + post: post12, byFieldId, } @@ -1337,7 +1354,7 @@ export const get28 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdQualityBadCasesResponse) -export const post12 = oc +export const post13 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1356,11 +1373,11 @@ export const post12 = oc export const badCases = { get: get28, - post: post12, + post: post13, byBadCaseId, } -export const post13 = oc +export const post14 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1379,7 +1396,7 @@ export const post13 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQualityReplayRunsResponse) export const replayRuns = { - post: post13, + post: post14, } export const quality = { @@ -1387,7 +1404,7 @@ export const quality = { replayRuns, } -export const post14 = oc +export const post15 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1404,13 +1421,13 @@ export const post14 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQueriesAdmissionResponse) export const admission = { - post: post14, + post: post15, } /** * @deprecated */ -export const post15 = oc +export const post16 = oc .route({ deprecated: true, inputStructure: 'detailed', @@ -1429,14 +1446,14 @@ export const post15 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQueriesResponse) export const queries = { - post: post15, + post: post16, admission, } /** * @deprecated */ -export const post16 = oc +export const post17 = oc .route({ deprecated: true, inputStructure: 'detailed', @@ -1449,10 +1466,10 @@ export const post16 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQueryStreamCapabilityResponse) export const queryStreamCapability = { - post: post16, + post: post17, } -export const post17 = oc +export const post18 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1469,7 +1486,7 @@ export const post17 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdResearchTasksPlanResponse) export const plan = { - post: post17, + post: post18, } export const get29 = oc @@ -1536,7 +1553,7 @@ export const get31 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdResearchTasksResponse) -export const post18 = oc +export const post19 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1555,7 +1572,7 @@ export const post18 = oc export const researchTasks2 = { get: get31, - post: post18, + post: post19, plan, byTaskId: byTaskId3, } @@ -1614,7 +1631,7 @@ export const settings = { migrations, } -export const post19 = oc +export const post20 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1631,7 +1648,7 @@ export const post19 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponse) export const refresh = { - post: post19, + post: post20, } export const byConnectionId = { @@ -1654,7 +1671,7 @@ export const get34 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse) -export const post20 = oc +export const post21 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1673,7 +1690,7 @@ export const post20 = oc export const sourceConnections = { get: get34, - post: post20, + post: post21, byConnectionId, } @@ -1692,7 +1709,7 @@ export const sourceProviders = { get: get35, } -export const post21 = oc +export const post22 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1709,7 +1726,7 @@ export const post21 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponse) export const cancel2 = { - post: post21, + post: post22, } export const get36 = oc @@ -1732,7 +1749,7 @@ export const pages = { get: get36, } -export const post22 = oc +export const post23 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1746,10 +1763,10 @@ export const post22 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponse) export const retry3 = { - post: post22, + post: post23, } -export const post23 = oc +export const post24 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1768,7 +1785,7 @@ export const post23 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponse) export const selection = { - post: post23, + post: post24, } export const get37 = oc @@ -1794,7 +1811,7 @@ export const sourceWorkflows = { byRunId, } -export const post24 = oc +export const post25 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1813,10 +1830,10 @@ export const post24 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlImportResponse) export const crawlImport = { - post: post24, + post: post25, } -export const post25 = oc +export const post26 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1834,7 +1851,7 @@ export const post25 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponse) export const crawlPreview = { - post: post25, + post: post26, } export const get38 = oc @@ -1857,7 +1874,7 @@ export const files = { get: get38, } -export const post26 = oc +export const post27 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1874,10 +1891,10 @@ export const post26 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdImportResponse) export const import_ = { - post: post26, + post: post27, } -export const post27 = oc +export const post28 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1894,7 +1911,7 @@ export const post27 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdImportFilesResponse) export const importFiles = { - post: post27, + post: post28, } export const get39 = oc @@ -1917,7 +1934,7 @@ export const pages2 = { get: get39, } -export const post28 = oc +export const post29 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1935,7 +1952,7 @@ export const post28 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponse) export const sync = { - post: post28, + post: post29, } export const get40 = oc @@ -1970,7 +1987,7 @@ export const syncPolicy = { put: put4, } -export const post29 = oc +export const post30 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1982,10 +1999,10 @@ export const post29 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdTestResponse) export const test = { - post: post29, + post: post30, } -export const post30 = oc +export const post31 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2004,7 +2021,7 @@ export const post30 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdWorkflowImportsResponse) export const workflowImports = { - post: post30, + post: post31, } export const delete10 = oc @@ -2085,7 +2102,7 @@ export const get42 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdSourcesResponse) -export const post31 = oc +export const post32 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2104,7 +2121,7 @@ export const post31 = oc export const sources = { get: get42, - post: post31, + post: post32, bySourceId, } @@ -2207,7 +2224,7 @@ export const traces = { byTraceId, } -export const post32 = oc +export const post33 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2224,10 +2241,10 @@ export const post32 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdAbortResponse) export const abort = { - post: post32, + post: post33, } -export const post33 = oc +export const post34 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2244,10 +2261,10 @@ export const post33 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdCompleteResponse) export const complete = { - post: post33, + post: post34, } -export const post34 = oc +export const post35 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2268,7 +2285,7 @@ export const post34 = oc ) export const presign = { - post: post34, + post: post35, } export const byPartNumber = { @@ -2279,7 +2296,7 @@ export const parts = { byPartNumber, } -export const post35 = oc +export const post36 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2296,7 +2313,7 @@ export const post35 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdSmallFileResponse) export const smallFile = { - post: post35, + post: post36, } export const byUploadSessionId = { @@ -2306,7 +2323,7 @@ export const byUploadSessionId = { smallFile, } -export const post36 = oc +export const post37 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2325,7 +2342,7 @@ export const post36 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsResponse) export const uploadSessions = { - post: post36, + post: post37, byUploadSessionId, } @@ -2409,7 +2426,7 @@ export const get49 = oc .input(z.object({ query: zGetKnowledgeFsSpacesQuery.optional() })) .output(zGetKnowledgeFsSpacesResponse) -export const post37 = oc +export const post38 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2423,11 +2440,11 @@ export const post37 = oc export const spaces = { get: get49, - post: post37, + post: post38, byControlSpaceId, } -export const post38 = oc +export const post39 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2444,7 +2461,7 @@ export const post38 = oc .output(zPostKnowledgeFsTasksByTaskIdStreamCapabilityResponse) export const streamCapability = { - post: post38, + post: post39, } export const byTaskId4 = { @@ -2459,6 +2476,7 @@ export const knowledgeFs = { wellKnown, queryStream, researchTasks, + sourceProviderPreview, spaces, tasks, } diff --git a/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts index ba53f257d35..dd2ccb7e9f8 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts @@ -18,6 +18,26 @@ export type KnowledgeFsAdmittedQueryRequest = { sessionId?: string | null } +export type KnowledgeFsInitialSourcePreviewPayload = { + credentialId: string + datasource: string + kind: 'online_document' | 'online_drive' + parameters?: { + [key: string]: JsonValue + } + pluginId: string + provider: string +} + +export type KnowledgeFsInitialSourcePreviewResponse = { + documents?: Array + files?: Array + kind: 'online_document' | 'online_drive' + next_page_parameters?: { + [key: string]: JsonValue + } | null +} + export type KnowledgeFsSpaceListResponse = { data: Array has_more: boolean @@ -30,7 +50,17 @@ export type KnowledgeFsSpaceCreatePayload = { embedding?: KnowledgeFsModelIntent | null icon?: string | null idempotency_key?: string | null - initial_source?: KnowledgeFsInitialWebsiteSourcePayload | null + initial_source?: + | ({ + kind: 'website_crawl' + } & KnowledgeFsInitialWebsiteSourcePayload) + | ({ + kind: 'online_document' + } & KnowledgeFsInitialOnlineDocumentSourcePayload) + | ({ + kind: 'online_drive' + } & KnowledgeFsInitialOnlineDriveSourcePayload) + | null name: string retrieval?: KnowledgeFsRetrievalProfileIntent | null slug: string @@ -947,6 +977,28 @@ export type KnowledgeFsQueryImageReference = { uploadFileId: string } +export type JsonValue = unknown + +export type KnowledgeFsInitialSourcePreviewDocumentResponse = { + last_edited_time?: string | null + name: string + page_id: string + provider_item_id: string + type: string + workspace_id: string + workspace_name?: string | null +} + +export type KnowledgeFsInitialSourcePreviewFileResponse = { + bucket?: string | null + id: string + mime_type?: string | null + name: string + provider_item_id: string + size: number + type: string +} + export type KnowledgeFsSpaceListItemResponse = { control_space_id: string created_at: string @@ -970,14 +1022,39 @@ export type KnowledgeFsModelIntent = { export type KnowledgeFsInitialWebsiteSourcePayload = { crawl_options: KnowledgeFsInitialWebsiteCrawlOptionsPayload + credentialId?: string | null + datasource?: string kind: 'website_crawl' name: string - provider: 'firecrawl' + pluginId?: string | null + provider: string root_url: string selection: Array sync_policy?: 'daily' | 'manual' | 'provider' } +export type KnowledgeFsInitialOnlineDocumentSourcePayload = { + credentialId: string + datasource: string + kind: 'online_document' + name: string + pluginId: string + provider: string + selection: Array + sync_policy?: 'daily' | 'manual' | 'provider' +} + +export type KnowledgeFsInitialOnlineDriveSourcePayload = { + credentialId: string + datasource: string + kind: 'online_drive' + name: string + pluginId: string + provider: string + selection: Array + sync_policy?: 'daily' | 'manual' | 'provider' +} + export type KnowledgeFsRetrievalProfileIntent = { default_mode: 'deep' | 'fast' | 'research' reasoning_model: KnowledgeFsModelIntent @@ -1478,6 +1555,25 @@ export type KnowledgeFsInitialWebsiteSelectionPayload = { title?: string | null } +export type KnowledgeFsOnlineDocumentWorkflowImportItemPayload = { + etag?: string | null + lastEditedTime?: string | null + name?: string | null + pageId: string + providerItemId: string + type: string + workspaceId: string +} + +export type KnowledgeFsOnlineDriveWorkflowImportItemPayload = { + bucket?: string | null + etag?: string | null + id: string + mimeType?: string | null + name: string + providerItemId: string +} + export type KnowledgeFsRerankIntent = { enabled: boolean model?: KnowledgeFsModelIntent | null @@ -1578,25 +1674,6 @@ export type KnowledgeFsSourcePageResponse = { type: string } -export type KnowledgeFsOnlineDocumentWorkflowImportItemPayload = { - etag?: string | null - lastEditedTime?: string | null - name?: string | null - pageId: string - providerItemId: string - type: string - workspaceId: string -} - -export type KnowledgeFsOnlineDriveWorkflowImportItemPayload = { - bucket?: string | null - etag?: string | null - id: string - mimeType?: string | null - name: string - providerItemId: string -} - export type KnowledgeFsTraceProfileResponse = { embedding_model?: string | null embedding_vector_space_id?: string | null @@ -1671,6 +1748,20 @@ export type GetKnowledgeFsResearchTasksByTaskIdEventsResponses = { export type GetKnowledgeFsResearchTasksByTaskIdEventsResponse = GetKnowledgeFsResearchTasksByTaskIdEventsResponses[keyof GetKnowledgeFsResearchTasksByTaskIdEventsResponses] +export type PostKnowledgeFsSourceProviderPreviewData = { + body: KnowledgeFsInitialSourcePreviewPayload + path?: never + query?: never + url: '/knowledge-fs/source-provider-preview' +} + +export type PostKnowledgeFsSourceProviderPreviewResponses = { + 200: KnowledgeFsInitialSourcePreviewResponse +} + +export type PostKnowledgeFsSourceProviderPreviewResponse = + PostKnowledgeFsSourceProviderPreviewResponses[keyof PostKnowledgeFsSourceProviderPreviewResponses] + export type GetKnowledgeFsSpacesData = { body?: never path?: never diff --git a/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts index c8163e81ab1..a5fc6e8caa9 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts @@ -733,6 +733,56 @@ export const zKnowledgeFsResearchTaskPlanPayload = z.object({ topK: z.int().gte(1).lte(50).nullish(), }) +export const zJsonValue = z.unknown() + +/** + * KnowledgeFSInitialSourcePreviewPayload + */ +export const zKnowledgeFsInitialSourcePreviewPayload = z.object({ + credentialId: z.string().min(1).max(255), + datasource: z.string().min(1).max(255), + kind: z.enum(['online_document', 'online_drive']), + parameters: z.record(z.string(), zJsonValue).optional(), + pluginId: z.string().min(1).max(255), + provider: z.string().min(1).max(255), +}) + +/** + * KnowledgeFSInitialSourcePreviewDocumentResponse + */ +export const zKnowledgeFsInitialSourcePreviewDocumentResponse = z.object({ + last_edited_time: z.string().nullish(), + name: z.string(), + page_id: z.string(), + provider_item_id: z.string(), + type: z.string(), + workspace_id: z.string(), + workspace_name: z.string().nullish(), +}) + +/** + * KnowledgeFSInitialSourcePreviewFileResponse + */ +export const zKnowledgeFsInitialSourcePreviewFileResponse = z.object({ + bucket: z.string().nullish(), + id: z.string(), + mime_type: z.string().nullish(), + name: z.string(), + provider_item_id: z.string(), + size: z.int().gte(0), + type: z.string(), +}) + +/** + * KnowledgeFSInitialSourcePreviewResponse + */ +export const zKnowledgeFsInitialSourcePreviewResponse = z.object({ + documents: z.array(zKnowledgeFsInitialSourcePreviewDocumentResponse).optional(), + files: z.array(zKnowledgeFsInitialSourcePreviewFileResponse).optional(), + kind: z.enum(['online_document', 'online_drive']), + next_page_parameters: z.record(z.string(), zJsonValue).nullish(), +}) + /** * KnowledgeFSModelIntent */ @@ -1599,14 +1649,96 @@ export const zKnowledgeFsInitialWebsiteSelectionPayload = z.object({ */ export const zKnowledgeFsInitialWebsiteSourcePayload = z.object({ crawl_options: zKnowledgeFsInitialWebsiteCrawlOptionsPayload, + credentialId: z.string().min(1).max(255).nullish(), + datasource: z.string().min(1).max(255).optional().default('crawl'), kind: z.literal('website_crawl'), name: z.string().min(1).max(200), - provider: z.literal('firecrawl'), + pluginId: z.string().min(1).max(255).nullish(), + provider: z.string().min(1).max(255), root_url: z.string().min(1).max(4096), selection: z.array(zKnowledgeFsInitialWebsiteSelectionPayload).min(1).max(200), sync_policy: z.enum(['daily', 'manual', 'provider']).optional().default('provider'), }) +/** + * KnowledgeFSOnlineDocumentWorkflowImportItemPayload + */ +export const zKnowledgeFsOnlineDocumentWorkflowImportItemPayload = z.object({ + etag: z.string().max(1024).nullish(), + lastEditedTime: z.string().max(128).nullish(), + name: z.string().max(500).nullish(), + pageId: z.string().min(1).max(1024), + providerItemId: z.string().min(1).max(1024), + type: z.string().min(1).max(128), + workspaceId: z.string().min(1).max(1024), +}) + +/** + * KnowledgeFSInitialOnlineDocumentSourcePayload + */ +export const zKnowledgeFsInitialOnlineDocumentSourcePayload = z.object({ + credentialId: z.string().min(1).max(255), + datasource: z.string().min(1).max(255), + kind: z.literal('online_document'), + name: z.string().min(1).max(200), + pluginId: z.string().min(1).max(255), + provider: z.string().min(1).max(255), + selection: z.array(zKnowledgeFsOnlineDocumentWorkflowImportItemPayload).min(1).max(200), + sync_policy: z.enum(['daily', 'manual', 'provider']).optional().default('provider'), +}) + +/** + * KnowledgeFSOnlineDocumentWorkflowImportPayload + */ +export const zKnowledgeFsOnlineDocumentWorkflowImportPayload = z.object({ + items: z.array(zKnowledgeFsOnlineDocumentWorkflowImportItemPayload).min(1).max(200), + kind: z.literal('online-document-import'), +}) + +/** + * KnowledgeFSOnlineDriveWorkflowImportItemPayload + */ +export const zKnowledgeFsOnlineDriveWorkflowImportItemPayload = z.object({ + bucket: z.string().max(1024).nullish(), + etag: z.string().max(1024).nullish(), + id: z.string().min(1).max(1024), + mimeType: z.string().max(255).nullish(), + name: z.string().min(1).max(500), + providerItemId: z.string().min(1).max(1024), +}) + +/** + * KnowledgeFSInitialOnlineDriveSourcePayload + */ +export const zKnowledgeFsInitialOnlineDriveSourcePayload = z.object({ + credentialId: z.string().min(1).max(255), + datasource: z.string().min(1).max(255), + kind: z.literal('online_drive'), + name: z.string().min(1).max(200), + pluginId: z.string().min(1).max(255), + provider: z.string().min(1).max(255), + selection: z.array(zKnowledgeFsOnlineDriveWorkflowImportItemPayload).min(1).max(200), + sync_policy: z.enum(['daily', 'manual', 'provider']).optional().default('provider'), +}) + +/** + * KnowledgeFSOnlineDriveWorkflowImportPayload + */ +export const zKnowledgeFsOnlineDriveWorkflowImportPayload = z.object({ + items: z.array(zKnowledgeFsOnlineDriveWorkflowImportItemPayload).min(1).max(200), + kind: z.literal('online-drive-import'), +}) + +/** + * KnowledgeFSSourceWorkflowImportPayload + */ +export const zKnowledgeFsSourceWorkflowImportPayload = z.discriminatedUnion('kind', [ + zKnowledgeFsOnlineDocumentWorkflowImportPayload.extend({ + kind: z.literal('online-document-import'), + }), + zKnowledgeFsOnlineDriveWorkflowImportPayload.extend({ kind: z.literal('online-drive-import') }), +]) + /** * KnowledgeFSRerankIntent */ @@ -1647,7 +1779,13 @@ export const zKnowledgeFsSpaceCreatePayload = z.object({ .regex(/^(?:builtin:)?[+a-z0-9_-]{1,64}$/) .nullish(), idempotency_key: z.string().min(1).max(255).nullish(), - initial_source: zKnowledgeFsInitialWebsiteSourcePayload.nullish(), + initial_source: z + .discriminatedUnion('kind', [ + zKnowledgeFsInitialWebsiteSourcePayload.extend({ kind: z.literal('website_crawl') }), + zKnowledgeFsInitialOnlineDocumentSourcePayload.extend({ kind: z.literal('online_document') }), + zKnowledgeFsInitialOnlineDriveSourcePayload.extend({ kind: z.literal('online_drive') }), + ]) + .nullish(), name: z.string().min(1).max(40), retrieval: zKnowledgeFsRetrievalProfileIntent.nullish(), slug: z @@ -2077,57 +2215,6 @@ export const zKnowledgeFsSourcePagesResponse = z.object({ workspaces: z.array(zKnowledgeFsSourceWorkspacePagesResponse), }) -/** - * KnowledgeFSOnlineDocumentWorkflowImportItemPayload - */ -export const zKnowledgeFsOnlineDocumentWorkflowImportItemPayload = z.object({ - etag: z.string().max(1024).nullish(), - lastEditedTime: z.string().max(128).nullish(), - name: z.string().max(500).nullish(), - pageId: z.string().min(1).max(1024), - providerItemId: z.string().min(1).max(1024), - type: z.string().min(1).max(128), - workspaceId: z.string().min(1).max(1024), -}) - -/** - * KnowledgeFSOnlineDocumentWorkflowImportPayload - */ -export const zKnowledgeFsOnlineDocumentWorkflowImportPayload = z.object({ - items: z.array(zKnowledgeFsOnlineDocumentWorkflowImportItemPayload).min(1).max(200), - kind: z.literal('online-document-import'), -}) - -/** - * KnowledgeFSOnlineDriveWorkflowImportItemPayload - */ -export const zKnowledgeFsOnlineDriveWorkflowImportItemPayload = z.object({ - bucket: z.string().max(1024).nullish(), - etag: z.string().max(1024).nullish(), - id: z.string().min(1).max(1024), - mimeType: z.string().max(255).nullish(), - name: z.string().min(1).max(500), - providerItemId: z.string().min(1).max(1024), -}) - -/** - * KnowledgeFSOnlineDriveWorkflowImportPayload - */ -export const zKnowledgeFsOnlineDriveWorkflowImportPayload = z.object({ - items: z.array(zKnowledgeFsOnlineDriveWorkflowImportItemPayload).min(1).max(200), - kind: z.literal('online-drive-import'), -}) - -/** - * KnowledgeFSSourceWorkflowImportPayload - */ -export const zKnowledgeFsSourceWorkflowImportPayload = z.discriminatedUnion('kind', [ - zKnowledgeFsOnlineDocumentWorkflowImportPayload.extend({ - kind: z.literal('online-document-import'), - }), - zKnowledgeFsOnlineDriveWorkflowImportPayload.extend({ kind: z.literal('online-drive-import') }), -]) - /** * KnowledgeFSTraceProfileResponse */ @@ -2213,6 +2300,14 @@ export const zGetKnowledgeFsResearchTasksByTaskIdEventsQuery = z.object({ */ export const zGetKnowledgeFsResearchTasksByTaskIdEventsResponse = z.record(z.string(), z.unknown()) +export const zPostKnowledgeFsSourceProviderPreviewBody = zKnowledgeFsInitialSourcePreviewPayload + +/** + * Datasource resources available for an initial Source + */ +export const zPostKnowledgeFsSourceProviderPreviewResponse = + zKnowledgeFsInitialSourcePreviewResponse + export const zGetKnowledgeFsSpacesQuery = z.object({ creator_ids: z.array(z.string().min(1).max(255)).max(100).optional(), limit: z.int().gte(1).lte(100).optional().default(20), diff --git a/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx b/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx index 69c358cee30..8ee890b0ea8 100644 --- a/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx +++ b/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx @@ -10,6 +10,7 @@ const serviceMock = vi.hoisted(() => ({ createKfsSource: vi.fn(), getKfsSource: vi.fn(), getCrawlStatus: vi.fn(), + previewInitialSource: vi.fn(), getDefaultModel: vi.fn(), getSpace: vi.fn(), getSyncPolicy: vi.fn(), @@ -27,6 +28,21 @@ const serviceMock = vi.hoisted(() => ({ documentsKey: vi.fn(() => ['console', 'knowledgeFs', 'documents']), })) +const datasourceQueryMock = vi.hoisted(() => ({ + auth: { + data: { result: [] as Array> }, + error: null as unknown, + isPending: false, + refetch: vi.fn(), + }, + plugins: { + data: [] as Array>, + error: null as unknown, + isPending: false, + refetch: vi.fn(), + }, +})) + const routerMock = vi.hoisted(() => ({ back: vi.fn(), replace: vi.fn(), @@ -82,6 +98,9 @@ vi.mock('jotai', async (importOriginal) => { vi.mock('@/service/client', () => ({ consoleClient: { knowledgeFs: { + sourceProviderPreview: { + post: serviceMock.previewInitialSource, + }, spaces: { byControlSpaceId: { get: serviceMock.getSpace, @@ -153,9 +172,161 @@ vi.mock('@/service/client', () => ({ vi.mock('@/service/datasets', () => ({ checkFirecrawlTaskStatus: serviceMock.getCrawlStatus, + checkJinaReaderTaskStatus: serviceMock.getCrawlStatus, + checkWatercrawlTaskStatus: serviceMock.getCrawlStatus, createFirecrawlTask: serviceMock.createCrawl, + createJinaReaderTask: serviceMock.createCrawl, + createWatercrawlTask: serviceMock.createCrawl, })) +vi.mock('@/service/use-pipeline', () => ({ + useDataSourceList: () => datasourceQueryMock.plugins, +})) + +vi.mock('@/service/use-datasource', () => ({ + useGetDataSourceListAuth: () => datasourceQueryMock.auth, +})) + +const firecrawlDatasourcePlugin = { + declaration: { + credentials_schema: [], + datasources: [ + { + description: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' }, + identity: { + author: 'langgenius', + label: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' }, + name: 'crawl', + provider: 'firecrawl', + }, + parameters: [], + }, + ], + identity: { + author: 'langgenius', + description: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' }, + icon: 'icon.svg', + label: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' }, + name: 'firecrawl', + tags: [], + }, + provider_type: 'website_crawl', + }, + is_authorized: true, + plugin_id: 'langgenius/firecrawl_datasource', + plugin_unique_identifier: 'langgenius/firecrawl_datasource:1.0.0@local', + provider: 'firecrawl', +} + +const firecrawlDatasourceAuth = { + author: 'langgenius', + credentials_list: [ + { + avatar_url: '', + credential: {}, + id: 'firecrawl-credential-1', + is_default: true, + name: 'Default Firecrawl', + type: 'api-key', + }, + ], + description: { en_US: 'Firecrawl' }, + icon: 'icon.svg', + label: { en_US: 'Firecrawl' }, + name: 'firecrawl', + plugin_id: 'langgenius/firecrawl_datasource', + plugin_unique_identifier: 'langgenius/firecrawl_datasource:1.0.0@local', + provider: 'firecrawl', +} + +const notionDatasourcePlugin = { + ...firecrawlDatasourcePlugin, + declaration: { + ...firecrawlDatasourcePlugin.declaration, + datasources: [ + { + description: { en_US: 'Notion', zh_Hans: 'Notion' }, + identity: { + author: 'langgenius', + label: { en_US: 'Notion', zh_Hans: 'Notion' }, + name: 'notion', + provider: 'notion', + }, + parameters: [], + }, + ], + identity: { + ...firecrawlDatasourcePlugin.declaration.identity, + label: { en_US: 'Notion', zh_Hans: 'Notion' }, + name: 'notion', + }, + provider_type: 'online_document', + }, + plugin_id: 'langgenius/notion_datasource', + plugin_unique_identifier: 'langgenius/notion_datasource:1.0.0@local', + provider: 'notion', +} + +const notionDatasourceAuth = { + ...firecrawlDatasourceAuth, + credentials_list: [ + { + ...firecrawlDatasourceAuth.credentials_list[0], + id: 'notion-credential-1', + name: 'Default Notion', + }, + ], + label: { en_US: 'Notion' }, + name: 'notion', + plugin_id: 'langgenius/notion_datasource', + plugin_unique_identifier: 'langgenius/notion_datasource:1.0.0@local', + provider: 'notion', +} + +const googleDriveDatasourcePlugin = { + ...firecrawlDatasourcePlugin, + declaration: { + ...firecrawlDatasourcePlugin.declaration, + datasources: [ + { + description: { en_US: 'Google Drive', zh_Hans: 'Google Drive' }, + identity: { + author: 'langgenius', + label: { en_US: 'Google Drive', zh_Hans: 'Google Drive' }, + name: 'google_drive', + provider: 'google_drive', + }, + parameters: [], + }, + ], + identity: { + ...firecrawlDatasourcePlugin.declaration.identity, + label: { en_US: 'Google Drive', zh_Hans: 'Google Drive' }, + name: 'google_drive', + }, + provider_type: 'online_drive', + }, + plugin_id: 'langgenius/google_drive', + plugin_unique_identifier: 'langgenius/google_drive:1.0.0@local', + provider: 'google_drive', +} + +const googleDriveDatasourceAuth = { + ...firecrawlDatasourceAuth, + credentials_list: [ + { + ...firecrawlDatasourceAuth.credentials_list[0], + id: 'google-drive-credential-1', + name: 'Default Google Drive', + }, + ], + label: { en_US: 'Google Drive' }, + name: 'google_drive', + plugin_id: 'langgenius/google_drive', + plugin_unique_identifier: 'langgenius/google_drive:1.0.0@local', + provider: 'google_drive', +} + const createdKnowledge = { control_space_id: 'e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084', model_setup_required: false, @@ -249,6 +420,12 @@ describe('CreateKnowledgePage', () => { beforeEach(() => { vi.clearAllMocks() globalThis.sessionStorage.clear() + datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin] + datasourceQueryMock.plugins.error = null + datasourceQueryMock.plugins.isPending = false + datasourceQueryMock.auth.data = { result: [firecrawlDatasourceAuth] } + datasourceQueryMock.auth.error = null + datasourceQueryMock.auth.isPending = false serviceMock.create.mockResolvedValue(createdKnowledge) serviceMock.createCrawl.mockResolvedValue({ job_id: 'crawl-job-1' }) serviceMock.createKfsSource.mockResolvedValue(kfsSourceResponse()) @@ -735,7 +912,7 @@ describe('CreateKnowledgePage', () => { ) }) - it('enables only source options supported by atomic creation', async () => { + it('enables every atomic source type and distinguishes installed providers', async () => { const user = userEvent.setup() renderPage() @@ -756,24 +933,17 @@ describe('CreateKnowledgePage', () => { const onlineDocuments = screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments', }) - expect(onlineDocuments).toHaveAttribute('aria-disabled', 'true') - expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDrive' })).toHaveAttribute( - 'aria-disabled', - 'true', - ) + expect(onlineDocuments).toBeEnabled() + expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDrive' })).toBeEnabled() expect(screen.getByRole('radio', { name: 'Firecrawl' })).toBeChecked() - for (const unavailableProvider of ['Jina Reader', 'WaterCrawl', 'FakeCrawler']) { - expect(screen.getByRole('radio', { name: unavailableProvider })).toHaveAttribute( - 'aria-disabled', - 'true', - ) - } + expect(screen.getByRole('radio', { name: 'Jina Reader' })).toBeEnabled() + expect(screen.getByRole('radio', { name: 'WaterCrawl' })).toBeEnabled() await user.click(onlineDocuments) - expect(onlineDocuments).not.toBeChecked() - expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' })).toBeChecked() - expect( - screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' }), - ).toBeDisabled() + expect(onlineDocuments).toBeChecked() + expect(screen.getByRole('radio', { name: 'Notion' })).toBeChecked() + expect(screen.getByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument() + await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' })) + expect(screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' })).toBeEnabled() expect(screen.queryByRole('alert')).not.toBeInTheDocument() expect(screen.getByText('dataset.newKnowledge.crawlOptions')).toBeInTheDocument() expect( @@ -998,8 +1168,11 @@ describe('CreateKnowledgePage', () => { include_subpages: true, limit: 100, }, + credentialId: 'firecrawl-credential-1', + datasource: 'crawl', kind: 'website_crawl', name: 'Dify docs', + pluginId: 'langgenius/firecrawl_datasource', provider: 'firecrawl', root_url: 'https://docs.dify.ai', selection: [ @@ -1017,6 +1190,154 @@ describe('CreateKnowledgePage', () => { expect(serviceMock.selectWorkflowPages).not.toHaveBeenCalled() }) + it('distinguishes an installed document provider with no credential', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, notionDatasourcePlugin] + renderPage() + + await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' })) + + expect( + screen.getByText('dataset.newKnowledge.providerNotConfigured:{"provider":"Notion"}'), + ).toBeInTheDocument() + expect( + screen.getByRole('button', { + name: 'dataset.newKnowledge.connectProvider:{"provider":"Notion"}', + }), + ).toBeEnabled() + expect(screen.queryByText('workflow.nodes.common.pluginNotInstalled')).not.toBeInTheDocument() + }) + + it('creates a knowledge space atomically with selected online documents', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, notionDatasourcePlugin] + datasourceQueryMock.auth.data = { + result: [firecrawlDatasourceAuth, notionDatasourceAuth], + } + serviceMock.previewInitialSource.mockResolvedValue({ + documents: [ + { + last_edited_time: '2026-08-10T08:00:00Z', + name: 'Product handbook', + page_id: 'page-1', + provider_item_id: '["workspace-1","page-1"]', + type: 'page', + workspace_id: 'workspace-1', + workspace_name: 'Dify', + }, + ], + kind: 'online_document', + next_page_parameters: null, + }) + renderPage() + await fillRequiredFields(user) + await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' })) + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Notion handbook', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })) + await user.click(await screen.findByRole('checkbox', { name: 'Product handbook' })) + await waitFor(() => + expect( + screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }), + ).toBeEnabled(), + ) + + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' })) + + await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce()) + expect(serviceMock.create).toHaveBeenCalledWith({ + body: expect.objectContaining({ + initial_source: { + credentialId: 'notion-credential-1', + datasource: 'notion', + kind: 'online_document', + name: 'Notion handbook', + pluginId: 'langgenius/notion_datasource', + provider: 'notion', + selection: [ + { + lastEditedTime: '2026-08-10T08:00:00Z', + name: 'Product handbook', + pageId: 'page-1', + providerItemId: '["workspace-1","page-1"]', + type: 'page', + workspaceId: 'workspace-1', + }, + ], + sync_policy: 'provider', + }, + }), + }) + }) + + it('creates a knowledge space atomically with a selected drive file', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, googleDriveDatasourcePlugin] + datasourceQueryMock.auth.data = { + result: [firecrawlDatasourceAuth, googleDriveDatasourceAuth], + } + serviceMock.previewInitialSource.mockResolvedValue({ + files: [ + { + bucket: null, + id: 'file-1', + mime_type: 'application/pdf', + name: 'Runbook.pdf', + provider_item_id: '["","file-1"]', + size: 2048, + type: 'application/pdf', + }, + ], + kind: 'online_drive', + next_page_parameters: null, + }) + renderPage() + await fillRequiredFields(user) + await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDrive' })) + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Drive runbook', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })) + await user.click(await screen.findByRole('checkbox', { name: 'Runbook.pdf' })) + await waitFor(() => + expect( + screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }), + ).toBeEnabled(), + ) + + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' })) + + await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce()) + expect(serviceMock.create).toHaveBeenCalledWith({ + body: expect.objectContaining({ + initial_source: { + credentialId: 'google-drive-credential-1', + datasource: 'google_drive', + kind: 'online_drive', + name: 'Drive runbook', + pluginId: 'langgenius/google_drive', + provider: 'google_drive', + selection: [ + { + bucket: undefined, + id: 'file-1', + mimeType: 'application/pdf', + name: 'Runbook.pdf', + providerItemId: '["","file-1"]', + }, + ], + sync_policy: 'provider', + }, + }), + }) + }) + it('requires a selected website preview page before creating with an initial source', async () => { const user = userEvent.setup() navigationMock.startMode = 'source' diff --git a/web/features/new-rag/create-connected-source-setup.tsx b/web/features/new-rag/create-connected-source-setup.tsx new file mode 100644 index 00000000000..52c9cbbdd26 --- /dev/null +++ b/web/features/new-rag/create-connected-source-setup.tsx @@ -0,0 +1,410 @@ +'use client' + +import type { + KnowledgeFsInitialSourcePreviewDocumentResponse, + KnowledgeFsInitialSourcePreviewFileResponse, + KnowledgeFsSpaceCreatePayload, +} from '@dify/contracts/api/console/knowledge-fs/types.gen' +import type { + NewKnowledgeOnlineDocumentsSourceDraft, + NewKnowledgeOnlineDriveSourceDraft, + NewKnowledgeSourceDraft, +} from './routes' +import type { InstalledSourceProviderOption } from './source-provider-options' +import type { DataSourceCredential } from '@/app/components/header/account-setting/data-source-page-new/types' +import { Button } from '@langgenius/dify-ui/button' +import { Checkbox } from '@langgenius/dify-ui/checkbox' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { useTranslation } from 'react-i18next' +import Loading from '@/app/components/base/loading' +import { consoleClient } from '@/service/client' +import { SourceNameField, SourceSyncPolicyField } from './source-setup-fields' + +type ConnectedDraft = NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft +type InitialSource = NonNullable +type PreviewDocument = KnowledgeFsInitialSourcePreviewDocumentResponse +type PreviewFile = KnowledgeFsInitialSourcePreviewFileResponse +type PreviewResource = + | { + depth: number + document: PreviewDocument + key: string + kind: 'document' + parentKey?: string + } + | { depth: number; file: PreviewFile; key: string; kind: 'file'; parentKey?: string } +type NextPageRequest = { + bucket?: string + depth: number + nextPage: Record + parentKey?: string + prefix?: string +} + +function isDriveContainer(file: PreviewFile) { + return /bucket|directory|folder|workspace/i.test(file.type) +} + +function resourceName(resource: PreviewResource) { + return resource.kind === 'document' ? resource.document.name : resource.file.name +} + +function resourceIcon(resource: PreviewResource) { + if (resource.kind === 'file' && isDriveContainer(resource.file)) + return 'i-ri-folder-3-fill text-text-warning' + return resource.kind === 'document' + ? 'i-ri-file-text-line text-text-tertiary' + : 'i-ri-file-3-line text-text-tertiary' +} + +export function CreateConnectedSourceSetup({ + credential, + disabled, + draft, + providerOption, + onDraftChange, + onInitialSourceChange, +}: { + credential: DataSourceCredential + disabled: boolean + draft: ConnectedDraft + providerOption: InstalledSourceProviderOption + onDraftChange: (draft: NewKnowledgeSourceDraft) => void + onInitialSourceChange: (source?: InitialSource) => void +}) { + const { t } = useTranslation('dataset') + const [resources, setResources] = useState([]) + const [selected, setSelected] = useState>(() => new Set()) + const [expanded, setExpanded] = useState>(() => new Set()) + const [loading, setLoading] = useState(false) + const [loadingMore, setLoadingMore] = useState(false) + const [error, setError] = useState(false) + const [previewed, setPreviewed] = useState(false) + const [nextPageRequest, setNextPageRequest] = useState() + const selectableResources = useMemo( + () => + resources.filter( + (resource) => resource.kind === 'document' || !isDriveContainer(resource.file), + ), + [resources], + ) + const visibleResources = useMemo(() => { + const byKey = new Map(resources.map((resource) => [resource.key, resource])) + return resources.filter((resource) => { + let parentKey = resource.parentKey + while (parentKey) { + if (!expanded.has(parentKey)) return false + parentKey = byKey.get(parentKey)?.parentKey + } + return true + }) + }, [expanded, resources]) + + useEffect(() => { + const selectedResources = selectableResources.filter((resource) => selected.has(resource.key)) + const name = draft.sourceName.trim() + if (!name || !selectedResources.length) { + onInitialSourceChange(undefined) + return + } + const binding = { + credentialId: credential.id, + datasource: providerOption.datasource.identity.name, + pluginId: providerOption.plugin.plugin_id, + provider: providerOption.plugin.provider, + } + if (draft.sourceType === 'onlineDocuments') { + onInitialSourceChange({ + ...binding, + kind: 'online_document', + name, + selection: selectedResources.flatMap((resource) => + resource.kind === 'document' + ? [ + { + lastEditedTime: resource.document.last_edited_time ?? undefined, + name: resource.document.name, + pageId: resource.document.page_id, + providerItemId: resource.document.provider_item_id, + type: resource.document.type, + workspaceId: resource.document.workspace_id, + }, + ] + : [], + ), + sync_policy: draft.syncPolicy, + }) + return + } + onInitialSourceChange({ + ...binding, + kind: 'online_drive', + name, + selection: selectedResources.flatMap((resource) => + resource.kind === 'file' + ? [ + { + bucket: resource.file.bucket ?? undefined, + id: resource.file.id, + mimeType: resource.file.mime_type ?? undefined, + name: resource.file.name, + providerItemId: resource.file.provider_item_id, + }, + ] + : [], + ), + sync_policy: draft.syncPolicy, + }) + }, [ + credential.id, + draft, + onInitialSourceChange, + providerOption.datasource.identity.name, + providerOption.plugin.plugin_id, + providerOption.plugin.provider, + selectableResources, + selected, + ]) + + const requestPreview = useCallback( + async ({ + append = false, + bucket, + depth = 0, + nextPage, + parentKey, + prefix, + }: { + append?: boolean + bucket?: string + depth?: number + nextPage?: Record + parentKey?: string + prefix?: string + } = {}) => { + append ? setLoadingMore(true) : setLoading(true) + setError(false) + try { + const response = await consoleClient.knowledgeFs.sourceProviderPreview.post({ + body: { + credentialId: credential.id, + datasource: providerOption.datasource.identity.name, + kind: draft.sourceType === 'onlineDocuments' ? 'online_document' : 'online_drive', + parameters: { + ...(bucket ? { bucket } : {}), + ...(prefix ? { prefix } : {}), + ...(nextPage ? { next_page_parameters: nextPage } : {}), + }, + pluginId: providerOption.plugin.plugin_id, + provider: providerOption.plugin.provider, + }, + }) + const nextResources: PreviewResource[] = + draft.sourceType === 'onlineDocuments' + ? (response.documents ?? []).map((document) => ({ + depth, + document, + key: `document:${document.provider_item_id}`, + kind: 'document' as const, + parentKey, + })) + : (response.files ?? []).map((file) => ({ + depth, + file, + key: `file:${file.provider_item_id}`, + kind: 'file' as const, + parentKey, + })) + setResources((current) => { + const next = new Map((append ? current : []).map((resource) => [resource.key, resource])) + for (const resource of nextResources) next.set(resource.key, resource) + return [...next.values()] + }) + if (parentKey) setExpanded((current) => new Set(current).add(parentKey)) + setNextPageRequest( + response.next_page_parameters + ? { + bucket, + depth, + nextPage: response.next_page_parameters, + parentKey, + prefix, + } + : null, + ) + setPreviewed(true) + } catch { + setError(true) + } finally { + setLoading(false) + setLoadingMore(false) + } + }, + [credential.id, draft.sourceType, providerOption], + ) + + const toggle = (key: string) => { + setSelected((current) => { + const next = new Set(current) + if (next.has(key)) next.delete(key) + else next.add(key) + return next + }) + } + const toggleAll = () => { + setSelected((current) => { + const next = new Set(current) + const allSelected = + selectableResources.length > 0 && + selectableResources.every((resource) => current.has(resource.key)) + for (const resource of selectableResources) { + if (allSelected) next.delete(resource.key) + else next.add(resource.key) + } + return next + }) + } + const expandContainer = (resource: Extract) => { + if (expanded.has(resource.key)) { + setExpanded((current) => { + const next = new Set(current) + next.delete(resource.key) + return next + }) + return + } + void requestPreview({ + append: true, + bucket: resource.file.bucket ?? undefined, + depth: resource.depth + 1, + parentKey: resource.key, + prefix: resource.file.id || undefined, + }) + } + + return ( +
+
+ + +
+ {!previewed && ( + + )} + {loading && ( +
+ +
+ )} + {error && ( +
+

+ {t(($) => $['newKnowledge.providerLoadFailed'])} +

+ +
+ )} + {previewed && !loading && ( +
+
+ $['newKnowledge.selectAll'])} + checked={ + selectableResources.length > 0 && + selectableResources.every((resource) => selected.has(resource.key)) + } + disabled={disabled || !selectableResources.length} + onCheckedChange={toggleAll} + /> + + {draft.sourceType === 'onlineDocuments' + ? t(($) => $['newKnowledge.selectPagesToSync']) + : t(($) => $['newKnowledge.selectFilesAndFolders'])} + + + {t(($) => $['newKnowledge.pagesSelected'], { count: selected.size })} + +
+
    + {visibleResources.map((resource) => { + const container = resource.kind === 'file' && isDriveContainer(resource.file) + return ( +
  • + {container ? ( + + ) : ( + toggle(resource.key)} + /> + )} + + + {resourceName(resource)} + +
    • + ) + })} +
+ {nextPageRequest && ( +
+ +
+ )} +
+ )} +
+ ) +} diff --git a/web/features/new-rag/create-knowledge-page.tsx b/web/features/new-rag/create-knowledge-page.tsx index e17905d25b9..cbd4c05d5f4 100644 --- a/web/features/new-rag/create-knowledge-page.tsx +++ b/web/features/new-rag/create-knowledge-page.tsx @@ -1,12 +1,11 @@ 'use client' import type { - KnowledgeFsInitialWebsiteSourcePayload, + KnowledgeFsSpaceCreatePayload, KnowledgeFsSpaceCreateResponse, } from '@dify/contracts/api/console/knowledge-fs/types.gen' import type { CreateKnowledgeExitReason } from './components/create-knowledge-exit-dialog' import type { KnowledgeVisibility } from './create-knowledge-workflow' -import type { WebsiteCrawlPreviewSelection } from './create-source-setup' import type { QueuedUpload } from './create-upload-queue' import type { KnowledgeFsUploadPhase, KnowledgeFsUploadProgress } from './knowledge-fs-upload' import type { NewKnowledgeSourceDraft, NewKnowledgeStartMode } from './routes' @@ -64,58 +63,19 @@ import { uploadKnowledgeFsDocuments } from './knowledge-fs-upload' import { createRequestId } from './request-id' import { createNewKnowledgeSourceDraft, - isValidWebsiteSourceDraft, newKnowledgeDetailPath, newKnowledgeDocumentsPath, newKnowledgeListPath, newKnowledgeSettingsPath, } from './routes' +type InitialSource = NonNullable + function normalizeStartMode(value: string | null): NewKnowledgeStartMode { if (value === 'source' || value === 'upload') return value return 'empty' } -function initialWebsiteSourceFromSelection( - draft: NewKnowledgeSourceDraft, - selection?: WebsiteCrawlPreviewSelection, -): KnowledgeFsInitialWebsiteSourcePayload | undefined { - if ( - draft.sourceType !== 'websiteCrawl' || - draft.provider !== 'Firecrawl' || - !selection || - selection.draft.sourceType !== 'websiteCrawl' || - selection.draft.provider !== draft.provider || - selection.draft.rootUrl !== draft.rootUrl || - selection.draft.sourceName !== draft.sourceName || - selection.draft.includeSubpages !== draft.includeSubpages || - selection.draft.maxPages !== draft.maxPages || - selection.draft.syncPolicy !== draft.syncPolicy - ) - return undefined - - const selectedPages = selection.pages.filter((page) => - selection.selectedPageIds.includes(page.pageId), - ) - if (!selectedPages.length) return undefined - - return { - crawl_options: { - include_subpages: draft.includeSubpages, - limit: draft.maxPages, - }, - kind: 'website_crawl', - name: draft.sourceName.trim(), - provider: 'firecrawl', - root_url: draft.rootUrl, - selection: selectedPages.map((page) => ({ - source_url: page.sourceUrl, - ...(page.title ? { title: page.title } : {}), - })), - sync_policy: draft.syncPolicy, - } -} - export function CreateKnowledgePage() { const { t } = useTranslation('dataset') const { t: tCommon } = useTranslation('common') @@ -141,9 +101,8 @@ export function CreateKnowledgePage() { const [sourceDraft, setSourceDraft] = useState(() => createNewKnowledgeSourceDraft('websiteCrawl'), ) - const [websitePreviewSelection, setWebsitePreviewSelection] = - useState() - const websitePreviewSelectionRef = useRef(undefined) + const [initialSource, setInitialSource] = useState() + const initialSourceRef = useRef(undefined) const [uploads, setUploads] = useState([]) const [createdKnowledge, setCreatedKnowledge] = useState() const [modelSetupDialogOpen, setModelSetupDialogOpen] = useState(false) @@ -167,15 +126,7 @@ export function CreateKnowledgePage() { const uploadSubmissionBlocked = startMode === 'upload' && (!uploadAvailable || !uploads.length || uploads.some((upload) => upload.issue)) - const initialWebsiteSource = initialWebsiteSourceFromSelection( - sourceDraft, - websitePreviewSelection, - ) - const sourceSubmissionBlocked = - startMode === 'source' && - (sourceDraft.sourceType === 'websiteCrawl' - ? !isValidWebsiteSourceDraft(sourceDraft) || !initialWebsiteSource - : !sourceDraft.sourceName.trim()) + const sourceSubmissionBlocked = startMode === 'source' && !initialSource const sourceDraftChanged = JSON.stringify(sourceDraft) !== JSON.stringify(createNewKnowledgeSourceDraft(sourceDraft.sourceType)) @@ -189,9 +140,9 @@ export function CreateKnowledgePage() { createdKnowledge, ) - const updateWebsitePreviewSelection = useCallback((selection?: WebsiteCrawlPreviewSelection) => { - websitePreviewSelectionRef.current = selection - setWebsitePreviewSelection(selection) + const updateInitialSource = useCallback((source?: InitialSource) => { + initialSourceRef.current = source + setInitialSource(source) }, []) const armHistoryGuard = useCallback(() => { @@ -321,14 +272,8 @@ export function CreateKnowledgePage() { const normalizedDescription = description.trim() if (!normalizedName) return - const latestWebsitePreviewSelection = - websitePreviewSelectionRef.current ?? websitePreviewSelection - const initialSource = initialWebsiteSourceFromSelection( - sourceDraft, - latestWebsitePreviewSelection, - ) - if (startMode === 'source' && sourceDraft.sourceType === 'websiteCrawl' && !initialSource) - return + const latestInitialSource = initialSourceRef.current ?? initialSource + if (startMode === 'source' && !latestInitialSource) return idempotencyKeyRef.current ??= createRequestId() setSubmissionLocked(true) @@ -337,7 +282,7 @@ export function CreateKnowledgePage() { existingKnowledge: createdKnowledge, description: normalizedDescription, idempotencyKey: idempotencyKeyRef.current, - initialSource, + initialSource: startMode === 'source' ? latestInitialSource : undefined, name: normalizedName, onCreated: (knowledgeSpace) => { setCreatedKnowledge(knowledgeSpace) @@ -573,14 +518,12 @@ export function CreateKnowledgePage() { draft={sourceDraft} onDraftChange={(value) => { setSourceDraft(value) - updateWebsitePreviewSelection(undefined) resetUnsubmittedError() }} - onWebsitePreviewSelectionChange={updateWebsitePreviewSelection} + onInitialSourceChange={updateInitialSource} onSourceTypeChange={(value) => { - if (value !== 'websiteCrawl') return setSourceDraft(createNewKnowledgeSourceDraft(value)) - updateWebsitePreviewSelection(undefined) + updateInitialSource(undefined) resetUnsubmittedError() }} /> diff --git a/web/features/new-rag/create-knowledge-workflow.ts b/web/features/new-rag/create-knowledge-workflow.ts index 3897bad92c9..0866f99ffa2 100644 --- a/web/features/new-rag/create-knowledge-workflow.ts +++ b/web/features/new-rag/create-knowledge-workflow.ts @@ -1,6 +1,5 @@ import type { KnowledgeFsControlSpaceVisibility, - KnowledgeFsInitialWebsiteSourcePayload, KnowledgeFsModelIntent, KnowledgeFsSpaceCreatePayload, KnowledgeFsSpaceCreateResponse, @@ -21,7 +20,7 @@ type CreateKnowledgeValues = { existingKnowledge?: KnowledgeFsSpaceCreateResponse description: string idempotencyKey: string - initialSource?: KnowledgeFsInitialWebsiteSourcePayload + initialSource?: NonNullable name: string onCreated: (knowledgeSpace: KnowledgeFsSpaceCreateResponse) => void visibility: KnowledgeVisibility diff --git a/web/features/new-rag/create-source-setup.tsx b/web/features/new-rag/create-source-setup.tsx index 3e9253698a9..49fda285fc8 100644 --- a/web/features/new-rag/create-source-setup.tsx +++ b/web/features/new-rag/create-source-setup.tsx @@ -1,13 +1,12 @@ 'use client' -import type { - NewKnowledgeOnlineDocumentsProvider, - NewKnowledgeOnlineDriveProvider, - NewKnowledgeSourceDraft, - NewKnowledgeWebsiteProvider, - NewKnowledgeWebsiteSourceDraft, -} from './routes' +import type { KnowledgeFsSpaceCreatePayload } from '@dify/contracts/api/console/knowledge-fs/types.gen' +import type { NewKnowledgeSourceDraft } from './routes' import type { CrawlPreviewPage } from './source-models' +import type { + DataSourceAuth, + DataSourceCredential, +} from '@/app/components/header/account-setting/data-source-page-new/types' import type { CrawlResultItem } from '@/models/datasets' import { Button } from '@langgenius/dify-ui/button' import { Checkbox } from '@langgenius/dify-ui/checkbox' @@ -25,13 +24,29 @@ import { } from '@langgenius/dify-ui/number-field' import { useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' -import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets' +import { buildIntegrationPath } from '@/app/components/integrations/routes' +import { + checkFirecrawlTaskStatus, + checkJinaReaderTaskStatus, + checkWatercrawlTaskStatus, + createFirecrawlTask, + createJinaReaderTask, + createWatercrawlTask, +} from '@/service/datasets' import { useGetDataSourceListAuth } from '@/service/use-datasource' +import { useDataSourceList } from '@/service/use-pipeline' import { CrawlPreviewPageSelection } from './crawl-selection-form' +import { CreateConnectedSourceSetup } from './create-connected-source-setup' import { isValidWebsiteSourceDraft, NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH } from './routes' +import { + discoverSourceProviderOptions, + normalizeSourceProviderName, + sourceProviderOptionForDraft, +} from './source-provider-options' import { SourceConnectionRequiredCard, SourceNameField, + SourceProviderNotInstalledCard, SourceProviderRadioGroup, SourceSyncPolicyField, SourceTypeSelector, @@ -48,12 +63,7 @@ const CRAWL_PREVIEW_SKELETONS = [ ] as const type LocalCrawlState = 'error' | 'idle' | 'running' | 'stopped' | 'success' - -export type WebsiteCrawlPreviewSelection = { - draft: NewKnowledgeWebsiteSourceDraft - pages: CrawlPreviewPage[] - selectedPageIds: string[] -} +type InitialSource = NonNullable function crawlPages(response: Record): CrawlResultItem[] { if (!Array.isArray(response.data)) return [] @@ -92,129 +102,56 @@ function crawlPreviewPages(pages: CrawlResultItem[]): CrawlPreviewPage[] { })) } -const providers = { - onlineDocuments: [ - { available: false, icon: 'i-custom-public-common-notion', label: 'Notion' }, - { - available: false, - icon: 'i-ri-file-text-fill text-[#4d8bf5]', - label: 'Google Docs', - }, - { available: false, icon: 'i-custom-public-common-confluence', label: 'Confluence' }, - ], - onlineDrive: [ - { available: false, icon: 'i-custom-public-common-google-drive', label: 'Google Drive' }, - { available: false, icon: 'i-ri-cloud-line', label: 'OneDrive' }, - { available: false, icon: 'i-ri-box-3-line', label: 'Amazon S3' }, - ], - websiteCrawl: [ - { icon: 'i-custom-public-common-firecrawl', label: 'Firecrawl', available: true }, - { available: false, icon: 'i-custom-public-llm-jina', label: 'Jina Reader' }, - { - available: false, - icon: 'i-custom-public-knowledge-watercrawl', - label: 'WaterCrawl', - }, - { - available: false, - icon: 'i-ri-global-line text-text-accent', - label: 'FakeCrawler', - }, - ], -} as const - -function ConnectedSourceConfiguration({ - disabled, - draft, - onDraftChange, -}: { - disabled: boolean - draft: NewKnowledgeSourceDraft - onDraftChange: (draft: NewKnowledgeSourceDraft) => void -}) { - return ( -
- - -
+function datasourceAuthForProvider( + authProviders: DataSourceAuth[], + pluginId: string, + provider: string, +) { + return authProviders.find( + (candidate) => candidate.plugin_id === pluginId && candidate.provider === provider, ) } -function NotionSourceConfiguration({ - disabled, - draft, - onConnect, - onDraftChange, -}: { - disabled: boolean - draft: NewKnowledgeSourceDraft - onConnect: () => void - onDraftChange: (draft: NewKnowledgeSourceDraft) => void -}) { - const { t } = useTranslation('dataset') - const datasourceAuthQuery = useGetDataSourceListAuth() - const notionConnected = Boolean( - datasourceAuthQuery.data?.result.some( - (auth) => - auth.credentials_list.length > 0 && - [auth.name, auth.plugin_id, auth.provider].some((identity) => - identity - .toLocaleLowerCase() - .replaceAll(/[^a-z0-9]/g, '') - .includes('notion'), - ), - ), - ) - - if (notionConnected) { - return ( - - ) - } - +function preferredCredential(auth?: DataSourceAuth): DataSourceCredential | undefined { return ( - $['newKnowledge.connectNotion'])} - description={t(($) => $['newKnowledge.notionNotConnectedDescription'])} - disabled={disabled} - icon={} - title={t(($) => $['newKnowledge.notionNotConnected'])} - onConnect={onConnect} - /> + auth?.credentials_list.find((credential) => credential.is_default) ?? auth?.credentials_list[0] ) } +function providerIntegrationPath(packageId?: string) { + const base = buildIntegrationPath('data-source') + if (!packageId) return base + const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) }) + return `${base}?${query.toString()}` +} + +function websiteProviderTransport(provider: string) { + const normalized = normalizeSourceProviderName(provider) + if (normalized.includes('firecrawl')) + return { check: checkFirecrawlTaskStatus, create: createFirecrawlTask } + if (normalized.includes('jinareader') || normalized === 'jina') + return { check: checkJinaReaderTaskStatus, create: createJinaReaderTask } + if (normalized.includes('watercrawl')) + return { check: checkWatercrawlTaskStatus, create: createWatercrawlTask } +} + export function CreateSourceSetup({ disabled, draft, onDraftChange, - onWebsitePreviewSelectionChange, + onInitialSourceChange, onSourceTypeChange, }: { disabled: boolean draft: NewKnowledgeSourceDraft onDraftChange: (draft: NewKnowledgeSourceDraft) => void - onWebsitePreviewSelectionChange?: (selection?: WebsiteCrawlPreviewSelection) => void + onInitialSourceChange: (source?: InitialSource) => void onSourceTypeChange: (sourceType: NewKnowledgeSourceDraft['sourceType']) => void }) { const { t } = useTranslation('dataset') + const datasourcePluginsQuery = useDataSourceList(true) + const datasourceAuthQuery = useGetDataSourceListAuth() const [optionsExpanded, setOptionsExpanded] = useState(false) - const [backendBoundaryVisible, setBackendBoundaryVisible] = useState(false) const [crawlState, setCrawlState] = useState('idle') const [previewPages, setPreviewPages] = useState([]) const [selectedPageIds, setSelectedPageIds] = useState>(() => new Set()) @@ -222,19 +159,35 @@ export function CreateSourceSetup({ const pollResolveRef = useRef<(() => void) | undefined>(undefined) const pollTimerRef = useRef | undefined>(undefined) const sourceType = draft.sourceType - const availableProviders = providers[sourceType] - const activeProvider = availableProviders.some((provider) => provider.label === draft.provider) - ? draft.provider - : availableProviders[0].label - const websiteProviderAvailable = - draft.sourceType === 'websiteCrawl' && draft.provider === 'Firecrawl' - const previewReady = websiteProviderAvailable && isValidWebsiteSourceDraft(draft) + const providerOptions = useMemo( + () => discoverSourceProviderOptions(sourceType, datasourcePluginsQuery.data ?? []), + [datasourcePluginsQuery.data, sourceType], + ) + const providerOption = sourceProviderOptionForDraft(providerOptions, draft) + const installedProviderOption = providerOption?.installed ? providerOption : undefined + const datasourceAuth = installedProviderOption + ? datasourceAuthForProvider( + datasourceAuthQuery.data?.result ?? [], + installedProviderOption.plugin.plugin_id, + installedProviderOption.plugin.provider, + ) + : undefined + const credential = preferredCredential(datasourceAuth) + const websiteTransport = + draft.sourceType === 'websiteCrawl' && installedProviderOption + ? websiteProviderTransport(installedProviderOption.plugin.provider) + : undefined + const previewReady = Boolean( + websiteTransport && + credential && + draft.sourceType === 'websiteCrawl' && + isValidWebsiteSourceDraft(draft), + ) const previewRootUrl = draft.sourceType === 'websiteCrawl' ? draft.rootUrl : '' const selectionPages = useMemo(() => crawlPreviewPages(previewPages), [previewPages]) const crawlOptionsAreDefault = draft.sourceType !== 'websiteCrawl' || (draft.includeSubpages === DEFAULT_INCLUDE_SUBPAGES && draft.maxPages === DEFAULT_MAX_PAGES) - const showBackendBoundary = () => setBackendBoundaryVisible(true) const stopPreview = (state: LocalCrawlState = 'stopped') => { crawlAttemptRef.current += 1 globalThis.clearTimeout(pollTimerRef.current) @@ -247,23 +200,29 @@ export function CreateSourceSetup({ stopPreview('idle') setPreviewPages([]) setSelectedPageIds(new Set()) - onWebsitePreviewSelectionChange?.(undefined) + onInitialSourceChange(undefined) } const updateDraft = (nextDraft: NewKnowledgeSourceDraft) => { onDraftChange(nextDraft) - setBackendBoundaryVisible(false) resetPreview() } const updateDraftWithoutReset = (nextDraft: NewKnowledgeSourceDraft) => { onDraftChange(nextDraft) - setBackendBoundaryVisible(false) } - const selectProvider = (provider: string) => { - if (draft.sourceType === 'onlineDocuments') - updateDraft({ ...draft, provider: provider as NewKnowledgeOnlineDocumentsProvider }) - else if (draft.sourceType === 'onlineDrive') - updateDraft({ ...draft, provider: provider as NewKnowledgeOnlineDriveProvider }) - else updateDraft({ ...draft, provider: provider as NewKnowledgeWebsiteProvider }) + const selectProvider = (providerKey: string) => { + const nextProvider = providerOptions.find((option) => option.key === providerKey) + if (!nextProvider) return + updateDraft({ + ...draft, + provider: nextProvider.label, + providerKey: nextProvider.key, + sourceName: '', + ...(draft.sourceType === 'onlineDrive' && + nextProvider.label === 'Amazon S3' && + draft.syncPolicy === 'provider' + ? { syncPolicy: 'daily' as const } + : {}), + }) } useEffect( @@ -276,7 +235,14 @@ export function CreateSourceSetup({ ) const startPreview = async () => { - if (draft.sourceType !== 'websiteCrawl' || !isValidWebsiteSourceDraft(draft)) return + if ( + draft.sourceType !== 'websiteCrawl' || + !isValidWebsiteSourceDraft(draft) || + !websiteTransport || + !credential || + !installedProviderOption + ) + return const attempt = crawlAttemptRef.current + 1 crawlAttemptRef.current = attempt globalThis.clearTimeout(pollTimerRef.current) @@ -284,7 +250,7 @@ export function CreateSourceSetup({ setSelectedPageIds(new Set()) setCrawlState('running') try { - const created = (await createFirecrawlTask({ + const created = (await websiteTransport.create({ options: { crawl_sub_pages: draft.includeSubpages, excludes: '', @@ -300,7 +266,7 @@ export function CreateSourceSetup({ if (!jobId) throw new Error('Website crawl did not return a job id') while (crawlAttemptRef.current === attempt) { - const response = (await checkFirecrawlTaskStatus(jobId)) as Record + const response = (await websiteTransport.check(jobId)) as Record if (crawlAttemptRef.current !== attempt) return setPreviewPages(crawlPages(response)) if (response.status === 'completed') { @@ -326,38 +292,59 @@ export function CreateSourceSetup({ const updateSelectedPageIds = (pageIds: Set) => { setSelectedPageIds(pageIds) - if (draft.sourceType !== 'websiteCrawl' || crawlState !== 'success' || !selectionPages.length) { - onWebsitePreviewSelectionChange?.(undefined) - return - } - onWebsitePreviewSelectionChange?.({ - draft, - pages: selectionPages, - selectedPageIds: [...pageIds], - }) } useEffect(() => { - if (draft.sourceType !== 'websiteCrawl' || crawlState !== 'success' || !selectionPages.length) { - onWebsitePreviewSelectionChange?.(undefined) + if ( + draft.sourceType !== 'websiteCrawl' || + crawlState !== 'success' || + !selectionPages.length || + !installedProviderOption || + !credential + ) { + onInitialSourceChange(undefined) return } - onWebsitePreviewSelectionChange?.({ - draft, - pages: selectionPages, - selectedPageIds: [...selectedPageIds], + const selectedPages = selectionPages.filter((page) => selectedPageIds.has(page.pageId)) + if (!selectedPages.length) { + onInitialSourceChange(undefined) + return + } + onInitialSourceChange({ + crawl_options: { + include_subpages: draft.includeSubpages, + limit: draft.maxPages, + }, + credentialId: credential.id, + datasource: installedProviderOption.datasource.identity.name, + kind: 'website_crawl', + name: draft.sourceName.trim(), + pluginId: installedProviderOption.plugin.plugin_id, + provider: installedProviderOption.plugin.provider, + root_url: draft.rootUrl, + selection: selectedPages.map((page) => ({ + source_url: page.sourceUrl, + ...(page.title ? { title: page.title } : {}), + })), + sync_policy: draft.syncPolicy, }) - }, [crawlState, draft, onWebsitePreviewSelectionChange, selectedPageIds, selectionPages]) + }, [ + crawlState, + credential, + draft, + installedProviderOption, + onInitialSourceChange, + selectedPageIds, + selectionPages, + ]) return (
{ - setBackendBoundaryVisible(false) onSourceTypeChange(value) }} /> @@ -374,29 +361,92 @@ export function CreateSourceSetup({ type="button" variant="ghost-accent" size="small" - disabled + disabled={disabled} className="gap-0.5 px-2.75" - onClick={showBackendBoundary} + onClick={() => + globalThis.open(buildIntegrationPath('data-source'), '_blank', 'noopener,noreferrer') + } > {t(($) => $['newKnowledge.moreProviders'])}
({ - disabled: !provider.available, - icon: , - value: provider.label, + options={providerOptions.map((option) => ({ + icon: , + label: option.label, + value: option.key, }))} surface="default" onChange={selectProvider} /> - {draft.sourceType === 'websiteCrawl' && ( + {datasourcePluginsQuery.isPending || datasourceAuthQuery.isPending ? ( +
+ +
+ ) : datasourcePluginsQuery.error || datasourceAuthQuery.error ? ( +
+

+ {t(($) => $['newKnowledge.providerLoadFailed'])} +

+ +
+ ) : providerOption && !providerOption.installed ? ( + } + provider={providerOption.label} + onInstall={() => + globalThis.open( + providerIntegrationPath(providerOption.packageId), + '_blank', + 'noopener,noreferrer', + ) + } + /> + ) : installedProviderOption && !credential ? ( + $['newKnowledge.connectProvider'], { + provider: installedProviderOption.label, + })} + description={t(($) => $['newKnowledge.providerCredentialRequiredDescription'], { + provider: installedProviderOption.label, + })} + disabled={disabled} + icon={} + title={t(($) => $['newKnowledge.providerNotConfigured'], { + provider: installedProviderOption.label, + })} + onConnect={() => + globalThis.open( + providerIntegrationPath(installedProviderOption.packageId), + '_blank', + 'noopener,noreferrer', + ) + } + /> + ) : draft.sourceType === 'websiteCrawl' && + installedProviderOption && + credential && + !websiteTransport ? ( +
+

{installedProviderOption.label}

+

+ {t(($) => $['newKnowledge.providerUnavailable'])} +

+
+ ) : draft.sourceType === 'websiteCrawl' && installedProviderOption && credential ? (
@@ -618,35 +668,17 @@ export function CreateSourceSetup({ /> )}
- )} - - {draft.sourceType !== 'websiteCrawl' && ( - <> - {draft.sourceType === 'onlineDocuments' && activeProvider === 'Notion' ? ( - - ) : ( - - )} - - )} - - {backendBoundaryVisible && ( -

- {t(($) => $['newKnowledge.sourceSetupBackendDependency'])} -

- )} + ) : draft.sourceType !== 'websiteCrawl' && installedProviderOption && credential ? ( + + ) : null}
) }