opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-09-08 02:43:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(knowledge-fs): create spaces with initial sources

Browse Source
This commit is contained in:
Stephen Zhou 2026-08-10 22:08:01 +08:00
parent 62cebcf64d
commit e4e9271fbe
No known key found for this signature in database
17 changed files with 2467 additions and 555 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
api/controllers/console/knowledge_fs/resources.py
View File

@ -48,6 +48,7 @@ from services.knowledge_fs.control_plane_service import (
from services.knowledge_fs.credential_service import (
KnowledgeFSCredentialPolicyError,
)
from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService
from services.knowledge_fs.product_authorization import (
KnowledgeFSProductNotFoundError,
)
@ -100,6 +101,8 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSGoldenQuestionPayload,
KnowledgeFSGoldenQuestionResponse,
KnowledgeFSIdempotencyHeader,
KnowledgeFSInitialSourcePreviewPayload,
KnowledgeFSInitialSourcePreviewResponse,
KnowledgeFSJWKSResponse,
KnowledgeFSLogicalDocumentDeletePayload,
KnowledgeFSLogicalDocumentListResponse,
@ -246,6 +249,7 @@ register_schema_models(
KnowledgeFSSourceConnectionRefreshPayload,
KnowledgeFSCrawlImportPayload,
KnowledgeFSCrawlPreviewSelectionPayload,
KnowledgeFSInitialSourcePreviewPayload,
KnowledgeFSSourceDeletePayload,
KnowledgeFSSourceDeleteQuery,
KnowledgeFSSourceFilesQuery,
@ -340,6 +344,7 @@ register_response_schema_models(
KnowledgeFSOverviewQueryOutcomesResponse,
KnowledgeFSOverviewStatsResponse,
KnowledgeFSPresignedUploadResponse,
KnowledgeFSInitialSourcePreviewResponse,
KnowledgeFSUploadSessionCreateResponse,
KnowledgeFSUploadSessionMutationResponse,
)
@ -579,6 +584,28 @@ def _overview_stats_response(
)
@console_ns.route("/knowledge-fs/source-provider-preview")
class KnowledgeFSInitialSourcePreviewApi(Resource):
@console_ns.expect(console_ns.models[KnowledgeFSInitialSourcePreviewPayload.__name__])
@console_ns.response(
HTTPStatus.OK,
"Datasource resources available for an initial Source",
console_ns.models[KnowledgeFSInitialSourcePreviewResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@_knowledge_fs_errors
def post(self):
account, tenant_id = current_account_with_tenant()
result = KnowledgeFSInitialSourcePreviewService(session_factory.get_session_maker()).preview(
tenant_id=tenant_id,
account=account,
payload=_payload(KnowledgeFSInitialSourcePreviewPayload),
)
return dump_response(KnowledgeFSInitialSourcePreviewResponse, result)
@console_ns.route("/knowledge-fs/spaces")
class KnowledgeFSSpacesApi(Resource):
@console_ns.doc(params=query_params_from_model(KnowledgeFSSpaceListQuery))

177
api/services/knowledge_fs/initial_source_preview.py Normal file
View File

@ -0,0 +1,177 @@
"""Read-only datasource discovery used before a KnowledgeFS Space exists."""
from __future__ import annotations
import json
from typing import Any, cast
from sqlalchemy import select
from core.datasource.datasource_manager import DatasourceManager
from core.datasource.entities.datasource_entities import (
DatasourceProviderType,
OnlineDriveBrowseFilesRequest,
)
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from models.account import Account
from models.credential_permission import CredentialType
from models.oauth import DatasourceProvider
from services.credential_permission_service import CredentialPermissionService
from services.datasource_provider_service import DatasourceProviderService
from services.knowledge_fs.product_dto import (
KnowledgeFSInitialSourcePreviewDocumentResponse,
KnowledgeFSInitialSourcePreviewFileResponse,
KnowledgeFSInitialSourcePreviewPayload,
KnowledgeFSInitialSourcePreviewResponse,
)
_MAX_PREVIEW_ITEMS = 200
class KnowledgeFSInitialSourcePreviewService:
def __init__(self, session_maker) -> None:
self._session_maker = session_maker
def _require_visible_credential(
self,
*,
tenant_id: str,
account: Account,
payload: KnowledgeFSInitialSourcePreviewPayload,
) -> None:
query = select(DatasourceProvider).where(
DatasourceProvider.tenant_id == tenant_id,
DatasourceProvider.id == payload.credential_id,
DatasourceProvider.provider == payload.provider,
DatasourceProvider.plugin_id == payload.plugin_id,
)
query = CredentialPermissionService.apply_visibility_filter(
query,
model_id_column=DatasourceProvider.id,
model_user_id_column=DatasourceProvider.user_id,
model_visibility_column=DatasourceProvider.visibility,
credential_type=CredentialType.DATASOURCE_PROVIDER,
user=account,
)
with self._session_maker() as session:
if session.scalar(query.limit(1)) is None:
raise PermissionError("Datasource credential is unavailable")
def preview(
self,
*,
tenant_id: str,
account: Account,
payload: KnowledgeFSInitialSourcePreviewPayload,
) -> KnowledgeFSInitialSourcePreviewResponse:
self._require_visible_credential(tenant_id=tenant_id, account=account, payload=payload)
credentials = DatasourceProviderService().get_datasource_credentials(
tenant_id=tenant_id,
provider=payload.provider,
plugin_id=payload.plugin_id,
credential_id=payload.credential_id,
)
if not credentials:
raise PermissionError("Datasource credential is unavailable")
provider_type = DatasourceProviderType(payload.kind)
runtime = DatasourceManager.get_datasource_runtime(
provider_id=f"{payload.plugin_id}/{payload.provider}",
datasource_name=payload.datasource,
tenant_id=tenant_id,
datasource_type=provider_type,
)
runtime.runtime.credentials = credentials
parameters = dict(payload.parameters)
if payload.kind == "online_document":
document_runtime = cast(OnlineDocumentDatasourcePlugin, runtime)
documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = []
for document_message in document_runtime.get_online_document_pages(
user_id=account.id,
datasource_parameters=parameters,
provider_type=document_runtime.datasource_provider_type(),
):
for workspace in document_message.result:
workspace_id = workspace.workspace_id or payload.provider
for page in workspace.pages:
documents.append(
KnowledgeFSInitialSourcePreviewDocumentResponse(
last_edited_time=page.last_edited_time,
name=page.page_name,
page_id=page.page_id,
provider_item_id=json.dumps([workspace_id, page.page_id], separators=(",", ":")),
type=page.type,
workspace_id=workspace_id,
workspace_name=workspace.workspace_name,
)
)
if len(documents) >= _MAX_PREVIEW_ITEMS:
return KnowledgeFSInitialSourcePreviewResponse(
documents=documents,
kind=payload.kind,
)
return KnowledgeFSInitialSourcePreviewResponse(documents=documents, kind=payload.kind)
drive_runtime = cast(OnlineDriveDatasourcePlugin, runtime)
files: list[KnowledgeFSInitialSourcePreviewFileResponse] = []
next_page_parameters = None
max_keys = parameters.get("max_keys", _MAX_PREVIEW_ITEMS)
if not isinstance(max_keys, int) or isinstance(max_keys, bool):
max_keys = _MAX_PREVIEW_ITEMS
max_keys = min(max(max_keys, 1), _MAX_PREVIEW_ITEMS)
bucket = parameters.get("bucket")
prefix = parameters.get("prefix")
raw_next_page_parameters = parameters.get("next_page_parameters")
request = OnlineDriveBrowseFilesRequest(
bucket=bucket if isinstance(bucket, str) else None,
prefix=prefix if isinstance(prefix, str) else "",
max_keys=max_keys,
next_page_parameters=(
cast(dict[str, Any], raw_next_page_parameters) if isinstance(raw_next_page_parameters, dict) else None
),
)
for drive_message in drive_runtime.online_drive_browse_files(
user_id=account.id,
request=request,
provider_type=drive_runtime.datasource_provider_type(),
):
for group in drive_message.result:
if group.is_truncated and group.next_page_parameters:
next_page_parameters = group.next_page_parameters
if group.bucket and not group.files:
files.append(
KnowledgeFSInitialSourcePreviewFileResponse(
bucket=group.bucket,
id="",
name=group.bucket,
provider_item_id=json.dumps([group.bucket, ""], separators=(",", ":")),
size=0,
type="bucket",
)
)
for file in group.files:
files.append(
KnowledgeFSInitialSourcePreviewFileResponse(
bucket=group.bucket,
id=file.id,
mime_type=file.type if "/" in file.type else None,
name=file.name,
provider_item_id=json.dumps([group.bucket or "", file.id], separators=(",", ":")),
size=file.size,
type=file.type,
)
)
if len(files) >= _MAX_PREVIEW_ITEMS:
return KnowledgeFSInitialSourcePreviewResponse(
files=files,
kind=payload.kind,
next_page_parameters=next_page_parameters,
)
return KnowledgeFSInitialSourcePreviewResponse(
files=files,
kind=payload.kind,
next_page_parameters=next_page_parameters,
)
__all__ = ["KnowledgeFSInitialSourcePreviewService"]

6
api/services/knowledge_fs/product_application_service.py
View File

@ -111,14 +111,14 @@ class KnowledgeFSProductApplicationService:
visibility=payload.visibility,
)
if payload.initial_source is not None:
from tasks.knowledge_fs_initial_source_tasks import import_initial_website_source
from tasks.knowledge_fs_initial_source_tasks import import_initial_source
import_initial_website_source.delay(
import_initial_source.delay(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=result.control_space.id,
operation_id=operation_id,
payload=payload.initial_source.model_dump(mode="json"),
payload=payload.initial_source.model_dump(mode="json", exclude_none=True),
)
return KnowledgeFSSpaceCreateResponse(
control_space_id=result.control_space.id,

136
api/services/knowledge_fs/product_dto.py
View File

@ -113,16 +113,51 @@ class KnowledgeFSInitialWebsiteSelectionPayload(BaseModel):
return source_url.strip()
class KnowledgeFSOnlineDocumentWorkflowImportItemPayload(BaseModel):
etag: str | None = Field(default=None, max_length=1_024)
last_edited_time: str | None = Field(default=None, max_length=128, alias="lastEditedTime")
name: str | None = Field(default=None, max_length=500)
page_id: str = Field(min_length=1, max_length=1_024, alias="pageId")
provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId")
type: str = Field(min_length=1, max_length=128)
workspace_id: str = Field(min_length=1, max_length=1_024, alias="workspaceId")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSOnlineDriveWorkflowImportItemPayload(BaseModel):
bucket: str | None = Field(default=None, max_length=1_024)
etag: str | None = Field(default=None, max_length=1_024)
id: str = Field(min_length=1, max_length=1_024)
mime_type: str | None = Field(default=None, max_length=255, alias="mimeType")
name: str = Field(min_length=1, max_length=500)
provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSInitialDatasourceBindingPayload(BaseModel):
credential_id: str | None = Field(default=None, min_length=1, max_length=255, alias="credentialId")
datasource: str = Field(min_length=1, max_length=255)
plugin_id: str = Field(min_length=1, max_length=255, alias="pluginId")
provider: str = Field(min_length=1, max_length=255)
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSInitialWebsiteSourcePayload(BaseModel):
kind: Literal["website_crawl"]
name: str = Field(min_length=1, max_length=200)
provider: Literal["firecrawl"]
provider: str = Field(min_length=1, max_length=255)
plugin_id: str | None = Field(default=None, min_length=1, max_length=255, alias="pluginId")
datasource: str = Field(default="crawl", min_length=1, max_length=255)
credential_id: str | None = Field(default=None, min_length=1, max_length=255, alias="credentialId")
root_url: str = Field(min_length=1, max_length=4_096)
crawl_options: KnowledgeFSInitialWebsiteCrawlOptionsPayload
selection: list[KnowledgeFSInitialWebsiteSelectionPayload] = Field(min_length=1, max_length=200)
sync_policy: Literal["provider", "daily", "manual"] = "provider"
model_config = ConfigDict(extra="forbid")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
@model_validator(mode="after")
def validate_selection(self) -> KnowledgeFSInitialWebsiteSourcePayload:
@ -132,6 +167,68 @@ class KnowledgeFSInitialWebsiteSourcePayload(BaseModel):
return self
class KnowledgeFSInitialOnlineDocumentSourcePayload(KnowledgeFSInitialDatasourceBindingPayload):
credential_id: str = Field(min_length=1, max_length=255, alias="credentialId")
kind: Literal["online_document"]
name: str = Field(min_length=1, max_length=200)
selection: list[KnowledgeFSOnlineDocumentWorkflowImportItemPayload] = Field(min_length=1, max_length=200)
sync_policy: Literal["provider", "daily", "manual"] = "provider"
class KnowledgeFSInitialOnlineDriveSourcePayload(KnowledgeFSInitialDatasourceBindingPayload):
credential_id: str = Field(min_length=1, max_length=255, alias="credentialId")
kind: Literal["online_drive"]
name: str = Field(min_length=1, max_length=200)
selection: list[KnowledgeFSOnlineDriveWorkflowImportItemPayload] = Field(min_length=1, max_length=200)
sync_policy: Literal["provider", "daily", "manual"] = "provider"
class KnowledgeFSInitialSourcePreviewPayload(KnowledgeFSInitialDatasourceBindingPayload):
credential_id: str = Field(min_length=1, max_length=255, alias="credentialId")
kind: Literal["online_document", "online_drive"]
parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50)
class KnowledgeFSInitialSourcePreviewDocumentResponse(ResponseModel):
last_edited_time: str | None = Field(
default=None, validation_alias=AliasChoices("last_edited_time", "lastEditedTime")
)
name: str
page_id: str = Field(validation_alias=AliasChoices("page_id", "pageId"))
provider_item_id: str = Field(validation_alias=AliasChoices("provider_item_id", "providerItemId"))
type: str
workspace_id: str = Field(validation_alias=AliasChoices("workspace_id", "workspaceId"))
workspace_name: str | None = Field(default=None, validation_alias=AliasChoices("workspace_name", "workspaceName"))
class KnowledgeFSInitialSourcePreviewFileResponse(ResponseModel):
bucket: str | None = None
id: str
mime_type: str | None = Field(default=None, validation_alias=AliasChoices("mime_type", "mimeType"))
name: str
provider_item_id: str = Field(validation_alias=AliasChoices("provider_item_id", "providerItemId"))
size: int = Field(ge=0)
type: str
class KnowledgeFSInitialSourcePreviewResponse(ResponseModel):
documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = Field(default_factory=list)
files: list[KnowledgeFSInitialSourcePreviewFileResponse] = Field(default_factory=list)
kind: Literal["online_document", "online_drive"]
next_page_parameters: dict[str, JsonValue] | None = Field(
default=None,
validation_alias=AliasChoices("next_page_parameters", "nextPageParameters"),
)
KnowledgeFSInitialSourcePayload = Annotated[
KnowledgeFSInitialWebsiteSourcePayload
| KnowledgeFSInitialOnlineDocumentSourcePayload
| KnowledgeFSInitialOnlineDriveSourcePayload,
Field(discriminator="kind"),
]
class KnowledgeFSSpaceCreatePayload(BaseModel):
name: str = Field(min_length=1, max_length=40)
slug: str = Field(min_length=1, max_length=160, pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
@ -141,7 +238,7 @@ class KnowledgeFSSpaceCreatePayload(BaseModel):
embedding: KnowledgeFSModelIntent | None = None
retrieval: KnowledgeFSRetrievalProfileIntent | None = None
idempotency_key: str | None = Field(default=None, min_length=1, max_length=255)
initial_source: KnowledgeFSInitialWebsiteSourcePayload | None = None
initial_source: KnowledgeFSInitialSourcePayload | None = None
model_config = ConfigDict(extra="forbid")
@ -1573,29 +1670,6 @@ class KnowledgeFSSourceWorkflowResponse(ResponseModel):
updated_at: datetime = Field(validation_alias=AliasChoices("updated_at", "updatedAt"))
class KnowledgeFSOnlineDocumentWorkflowImportItemPayload(BaseModel):
etag: str | None = Field(default=None, max_length=1_024)
last_edited_time: str | None = Field(default=None, max_length=128, alias="lastEditedTime")
name: str | None = Field(default=None, max_length=500)
page_id: str = Field(min_length=1, max_length=1_024, alias="pageId")
provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId")
type: str = Field(min_length=1, max_length=128)
workspace_id: str = Field(min_length=1, max_length=1_024, alias="workspaceId")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSOnlineDriveWorkflowImportItemPayload(BaseModel):
bucket: str | None = Field(default=None, max_length=1_024)
etag: str | None = Field(default=None, max_length=1_024)
id: str = Field(min_length=1, max_length=1_024)
mime_type: str | None = Field(default=None, max_length=255, alias="mimeType")
name: str = Field(min_length=1, max_length=500)
provider_item_id: str = Field(min_length=1, max_length=1_024, alias="providerItemId")
model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
class KnowledgeFSOnlineDocumentWorkflowImportPayload(BaseModel):
items: list[KnowledgeFSOnlineDocumentWorkflowImportItemPayload] = Field(min_length=1, max_length=200)
kind: Literal["online-document-import"]
@ -2798,6 +2872,14 @@ __all__ = [
"KnowledgeFSExternalAccessPayload",
"KnowledgeFSExternalAccessResponse",
"KnowledgeFSIdempotencyHeader",
"KnowledgeFSInitialDatasourceBindingPayload",
"KnowledgeFSInitialOnlineDocumentSourcePayload",
"KnowledgeFSInitialOnlineDriveSourcePayload",
"KnowledgeFSInitialSourcePayload",
"KnowledgeFSInitialSourcePreviewDocumentResponse",
"KnowledgeFSInitialSourcePreviewFileResponse",
"KnowledgeFSInitialSourcePreviewPayload",
"KnowledgeFSInitialSourcePreviewResponse",
"KnowledgeFSInitialWebsiteCrawlOptionsPayload",
"KnowledgeFSInitialWebsiteSelectionPayload",
"KnowledgeFSInitialWebsiteSourcePayload",
@ -2809,6 +2891,8 @@ __all__ = [
"KnowledgeFSMemberBindingPayload",
"KnowledgeFSMembersReplacePayload",
"KnowledgeFSModelIntent",
"KnowledgeFSOnlineDocumentWorkflowImportItemPayload",
"KnowledgeFSOnlineDriveWorkflowImportItemPayload",
"KnowledgeFSOverviewBaseStatsResponse",
"KnowledgeFSOverviewCountComparisonResponse",
"KnowledgeFSOverviewHealthComponentResponse",

420
api/tasks/knowledge_fs_initial_source_tasks.py
View File

@ -1,44 +1,110 @@
"""Durable follow-up that starts the first website import after Space provisioning."""
"""Durable follow-up that imports the first Source after Space provisioning."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import Literal
from celery import shared_task
from pydantic import TypeAdapter
from sqlalchemy import select
from core.db.session_factory import session_factory
from models.account import Account
from models.credential_permission import CredentialType
from models.knowledge_fs import KnowledgeFSControlSpaceState
from models.oauth import DatasourceProvider
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
SQLAlchemyKnowledgeFSControlSpaceRepository,
)
from services.credential_permission_service import CredentialPermissionService
from services.knowledge_fs.product_dto import (
KnowledgeFSCrawlImportPayload,
KnowledgeFSInitialOnlineDocumentSourcePayload,
KnowledgeFSInitialSourcePayload,
KnowledgeFSInitialWebsiteSourcePayload,
KnowledgeFSOnlineDocumentWorkflowImportPayload,
KnowledgeFSOnlineDriveWorkflowImportPayload,
KnowledgeFSSourceConnectionCreatePayload,
KnowledgeFSSourceCreatePayload,
KnowledgeFSSourceSyncPolicyPayload,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSourceWorkflowImportPayload,
)
from services.knowledge_fs.product_remote import KnowledgeFSProductResourceNotFoundError
from services.knowledge_fs.runtime import get_knowledge_fs_runtime
_FIRECRAWL_PROVIDER_ID = "plugin-daemon-website"
_FIRECRAWL_PLUGIN_ID = "langgenius/firecrawl_datasource"
_LEGACY_WEBSITE_PLUGIN_IDS = {
"firecrawl": "langgenius/firecrawl_datasource",
"jinareader": "langgenius/jina_datasource",
"watercrawl": "watercrawl/watercrawl_datasource",
}
_PAGE_SIZE = 200
_INITIAL_SOURCE_ADAPTER: TypeAdapter[KnowledgeFSInitialSourcePayload] = TypeAdapter(KnowledgeFSInitialSourcePayload)
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class _DatasourceBinding:
credential_id: str | None
datasource: str
plugin_id: str
provider: str
provider_id: str
provider_kind: str
class KnowledgeFSInitialSourceNotReadyError(RuntimeError):
"""The Space or Source workflow is still progressing and should be retried."""
"""The Space, connection, or Source workflow is progressing and should be retried."""
def __init__(self, message: str, *, workflow_id: str | None = None) -> None:
super().__init__(message)
self.workflow_id = workflow_id
def _binding(payload: KnowledgeFSInitialSourcePayload) -> _DatasourceBinding:
if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload):
normalized_provider = "".join(character for character in payload.provider.lower() if character.isalnum())
plugin_id = payload.plugin_id or _LEGACY_WEBSITE_PLUGIN_IDS.get(normalized_provider)
if plugin_id is None:
raise RuntimeError("Website datasource plugin binding is required")
return _DatasourceBinding(
credential_id=payload.credential_id,
datasource=payload.datasource,
plugin_id=plugin_id,
provider=payload.provider,
provider_id="plugin-daemon-website",
provider_kind="website",
)
if isinstance(payload, KnowledgeFSInitialOnlineDocumentSourcePayload):
return _DatasourceBinding(
credential_id=payload.credential_id,
datasource=payload.datasource,
plugin_id=payload.plugin_id,
provider=payload.provider,
provider_id="plugin-daemon-online-document",
provider_kind="online-document",
)
return _DatasourceBinding(
credential_id=payload.credential_id,
datasource=payload.datasource,
plugin_id=payload.plugin_id,
provider=payload.provider,
provider_id="plugin-daemon-online-drive",
provider_kind="online-drive",
)
def _request_id(*, operation_id: str, payload: KnowledgeFSInitialSourcePayload) -> str:
# Preserve the original website request ID so retries from the previous rollout
# reconcile with the same provisional Source.
if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload):
return f"initial-website-source:{operation_id}"
return f"initial-source:{operation_id}"
def _find_initial_source(*, facade, tenant_id: str, account_id: str, control_space_id: str, request_id: str):
cursor: str | None = None
while True:
@ -57,29 +123,41 @@ def _find_initial_source(*, facade, tenant_id: str, account_id: str, control_spa
cursor = response.next_cursor
def _find_firecrawl_credential(*, session_maker, tenant_id: str) -> tuple[str, str]:
def _find_credential(*, session_maker, tenant_id: str, account_id: str, binding: _DatasourceBinding) -> tuple[str, str]:
query = select(DatasourceProvider).where(
DatasourceProvider.tenant_id == tenant_id,
DatasourceProvider.provider == binding.provider,
DatasourceProvider.plugin_id == binding.plugin_id,
)
with session_maker() as session:
account = session.get(Account, account_id)
if account is None:
raise RuntimeError("Initial Source account was not found")
query = CredentialPermissionService.apply_visibility_filter(
query,
model_id_column=DatasourceProvider.id,
model_user_id_column=DatasourceProvider.user_id,
model_visibility_column=DatasourceProvider.visibility,
credential_type=CredentialType.DATASOURCE_PROVIDER,
user=account,
)
if binding.credential_id is not None:
query = query.where(DatasourceProvider.id == binding.credential_id)
credential = session.scalar(
select(DatasourceProvider)
.where(
DatasourceProvider.tenant_id == tenant_id,
DatasourceProvider.provider == "firecrawl",
DatasourceProvider.plugin_id == _FIRECRAWL_PLUGIN_ID,
)
.order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc())
.limit(1)
query.order_by(DatasourceProvider.is_default.desc(), DatasourceProvider.created_at.asc()).limit(1)
)
if credential is None:
raise RuntimeError("Firecrawl credential is unavailable")
return str(credential.id), credential.name or "Firecrawl"
raise RuntimeError("Datasource credential is unavailable")
return str(credential.id), credential.name or binding.provider
def _find_or_create_firecrawl_connection(
def _find_or_create_connection(
*,
facade,
tenant_id: str,
account_id: str,
control_space_id: str,
binding: _DatasourceBinding,
credential_id: str,
credential_name: str,
):
@ -88,9 +166,16 @@ def _find_or_create_firecrawl_connection(
account_id=account_id,
control_space_id=control_space_id,
)
if not any(provider.id == _FIRECRAWL_PROVIDER_ID and provider.available for provider in providers.data):
raise RuntimeError("Firecrawl provider is unavailable")
if not any(provider.id == binding.provider_id and provider.available for provider in providers.data):
raise RuntimeError(f"{binding.provider_kind} provider is unavailable")
expected_configuration: dict[str, bool | int | str] = {
"credentialId": credential_id,
"datasource": binding.datasource,
"pluginId": binding.plugin_id,
"provider": binding.provider,
"providerKind": binding.provider_kind,
}
cursor: str | None = None
while True:
response = facade.list_source_connections(
@ -101,16 +186,15 @@ def _find_or_create_firecrawl_connection(
limit=_PAGE_SIZE,
)
for connection in response.data:
if (
connection.provider_id != _FIRECRAWL_PROVIDER_ID
or connection.configuration.get("credentialId") != credential_id
):
if connection.provider_id != binding.provider_id:
continue
if any(connection.configuration.get(key) != value for key, value in expected_configuration.items()):
continue
if connection.status == "active":
return connection
if connection.status == "provisioning":
raise KnowledgeFSInitialSourceNotReadyError("Firecrawl connection is still provisioning")
raise RuntimeError(f"Firecrawl connection is unavailable in state {connection.status}")
raise KnowledgeFSInitialSourceNotReadyError("Datasource connection is still provisioning")
raise RuntimeError(f"Datasource connection is unavailable in state {connection.status}")
if not response.next_cursor:
break
cursor = response.next_cursor
@ -121,33 +205,131 @@ def _find_or_create_firecrawl_connection(
control_space_id=control_space_id,
payload=KnowledgeFSSourceConnectionCreatePayload(
authKind="endpoint",
configuration={
"credentialId": credential_id,
"datasource": "crawl",
"pluginId": _FIRECRAWL_PLUGIN_ID,
"provider": "firecrawl",
"providerKind": "website",
},
configuration=expected_configuration,
credentials={},
name=credential_name,
providerId=_FIRECRAWL_PROVIDER_ID,
providerId=binding.provider_id,
),
)
if connection.status != "active":
raise KnowledgeFSInitialSourceNotReadyError("Firecrawl connection is still provisioning")
raise KnowledgeFSInitialSourceNotReadyError("Datasource connection is still provisioning")
return connection
def start_initial_website_source_import(
def _source_payload(
*,
payload: KnowledgeFSInitialSourcePayload,
binding: _DatasourceBinding,
connection_id: str,
request_id: str,
) -> KnowledgeFSSourceCreatePayload:
metadata: dict[str, object] = {
"clientRequestId": request_id,
"preview": True,
"providerId": binding.provider_id,
"providerKind": binding.provider_kind,
"providerName": payload.provider,
}
if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload):
metadata["crawlOptions"] = {
"includeSubpages": payload.crawl_options.include_subpages,
"limit": payload.crawl_options.limit,
}
source_type: Literal["connector", "web"] = "web"
uri = payload.root_url
else:
source_type = "connector"
uri = f"connector://{connection_id}"
return KnowledgeFSSourceCreatePayload(
connectionId=connection_id,
metadata=metadata,
name=payload.name,
status="disabled",
type=source_type,
uri=uri,
)
def _start_workflow(
*,
facade,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
request_id: str,
payload: KnowledgeFSInitialSourcePayload,
):
if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload):
return facade.import_selected_source_crawl(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
source_id=source_id,
payload=KnowledgeFSCrawlImportPayload(
sourceUrls=[selection.source_url for selection in payload.selection],
),
idempotency_key=f"{request_id}:crawl-import",
)
if isinstance(payload, KnowledgeFSInitialOnlineDocumentSourcePayload):
import_payload = KnowledgeFSSourceWorkflowImportPayload(
KnowledgeFSOnlineDocumentWorkflowImportPayload(
kind="online-document-import",
items=payload.selection,
)
)
else:
import_payload = KnowledgeFSSourceWorkflowImportPayload(
KnowledgeFSOnlineDriveWorkflowImportPayload(
kind="online-drive-import",
items=payload.selection,
)
)
return facade.import_source_workflow(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
source_id=source_id,
payload=import_payload,
idempotency_key=f"{request_id}:connector-import",
)
def _sync_policy_payload(
*, payload: KnowledgeFSInitialSourcePayload, expected_revision: int, source_version: int
) -> KnowledgeFSSourceSyncPolicyPayload:
if payload.sync_policy == "manual":
return KnowledgeFSSourceSyncPolicyPayload(
enabled=False,
mode="manual",
expectedRevision=expected_revision,
expectedSourceVersion=source_version,
)
if payload.sync_policy == "daily":
return KnowledgeFSSourceSyncPolicyPayload(
enabled=True,
mode="interval",
expectedRevision=expected_revision,
expectedSourceVersion=source_version,
)
return KnowledgeFSSourceSyncPolicyPayload(
enabled=True,
mode="provider",
expectedRevision=expected_revision,
expectedSourceVersion=source_version,
)
def start_initial_source_import(
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
payload: KnowledgeFSInitialWebsiteSourcePayload,
payload: KnowledgeFSInitialSourcePayload,
workflow_id: str | None = None,
) -> str:
"""Idempotently create the provisional Source and start its selected crawl import."""
"""Idempotently create a provisional Source, import its selection, and commit it."""
session_maker = session_factory.get_session_maker()
with session_maker() as session:
@ -173,10 +355,10 @@ def start_initial_website_source_import(
run_id=workflow_id,
)
if workflow.source_id is None:
raise RuntimeError("Initial website Source import workflow has no Source")
raise RuntimeError("Initial Source import workflow has no Source")
source_id = workflow.source_id
else:
request_id = f"initial-website-source:{operation_id}"
request_id = _request_id(operation_id=operation_id, payload=payload)
source = _find_initial_source(
facade=facade,
tenant_id=tenant_id,
@ -185,15 +367,19 @@ def start_initial_website_source_import(
request_id=request_id,
)
if source is None:
credential_id, credential_name = _find_firecrawl_credential(
binding = _binding(payload)
credential_id, credential_name = _find_credential(
session_maker=session_maker,
tenant_id=tenant_id,
account_id=account_id,
binding=binding,
)
connection = _find_or_create_firecrawl_connection(
connection = _find_or_create_connection(
facade=facade,
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
binding=binding,
credential_id=credential_id,
credential_name=credential_name,
)
@ -201,38 +387,27 @@ def start_initial_website_source_import(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
payload=KnowledgeFSSourceCreatePayload(
connectionId=connection.id,
metadata={
"clientRequestId": request_id,
"crawlOptions": {
"includeSubpages": payload.crawl_options.include_subpages,
"limit": payload.crawl_options.limit,
},
"preview": True,
"providerId": _FIRECRAWL_PROVIDER_ID,
},
name=payload.name,
status="disabled",
type="web",
uri=payload.root_url,
payload=_source_payload(
payload=payload,
binding=binding,
connection_id=connection.id,
request_id=request_id,
),
)
source_id = source.id
workflow = facade.import_selected_source_crawl(
workflow = _start_workflow(
facade=facade,
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
source_id=source_id,
payload=KnowledgeFSCrawlImportPayload(
sourceUrls=[selection.source_url for selection in payload.selection],
),
idempotency_key=f"{request_id}:crawl-import",
request_id=request_id,
payload=payload,
)
if workflow.state in {"queued", "running", "crawling", "importing", "syncing"}:
raise KnowledgeFSInitialSourceNotReadyError(
"Initial website Source import is still running",
"Initial Source import is still running",
workflow_id=workflow.id,
)
if workflow.state != "completed":
@ -269,7 +444,7 @@ def start_initial_website_source_import(
),
)
logger.error(
"Initial website Source import failed",
"Initial Source import failed",
extra={
"control_space_id": control_space_id,
"error_code": workflow.last_error_code,
@ -311,43 +486,99 @@ def start_initial_website_source_import(
expected_revision = current_policy.revision
except KnowledgeFSProductResourceNotFoundError:
expected_revision = 0
if payload.sync_policy == "manual":
sync_policy = KnowledgeFSSourceSyncPolicyPayload(
enabled=False,
mode="manual",
expectedRevision=expected_revision,
expectedSourceVersion=committed_source.version,
)
elif payload.sync_policy == "daily":
sync_policy = KnowledgeFSSourceSyncPolicyPayload(
enabled=True,
mode="interval",
expectedRevision=expected_revision,
expectedSourceVersion=committed_source.version,
)
else:
sync_policy = KnowledgeFSSourceSyncPolicyPayload(
enabled=True,
mode="provider",
expectedRevision=expected_revision,
expectedSourceVersion=committed_source.version,
)
facade.update_source_sync_policy(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
source_id=source_id,
payload=sync_policy,
payload=_sync_policy_payload(
payload=payload,
expected_revision=expected_revision,
source_version=committed_source.version,
),
)
return workflow.id
@shared_task(
bind=True,
queue="knowledge_fs_lifecycle",
max_retries=180,
default_retry_delay=2,
)
def start_initial_website_source_import(
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
payload: KnowledgeFSInitialWebsiteSourcePayload,
workflow_id: str | None = None,
) -> str:
"""Compatibility wrapper for callers using the original website-only protocol."""
return start_initial_source_import(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
payload=payload,
workflow_id=workflow_id,
)
def _run_initial_source_task(
task,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
payload: dict[str, object],
workflow_id: str | None,
) -> str:
try:
return start_initial_source_import(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
payload=_INITIAL_SOURCE_ADAPTER.validate_python(payload),
workflow_id=workflow_id,
)
except KnowledgeFSInitialSourceNotReadyError as exc:
if exc.workflow_id is not None:
raise task.retry(
exc=exc,
kwargs={
"tenant_id": tenant_id,
"account_id": account_id,
"control_space_id": control_space_id,
"operation_id": operation_id,
"payload": payload,
"workflow_id": exc.workflow_id,
},
)
raise task.retry(exc=exc)
@shared_task(bind=True, queue="knowledge_fs_lifecycle", max_retries=180, default_retry_delay=2)
def import_initial_source(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
payload: dict[str, object],
workflow_id: str | None = None,
) -> str:
return _run_initial_source_task(
self,
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
payload=payload,
workflow_id=workflow_id,
)
@shared_task(bind=True, queue="knowledge_fs_lifecycle", max_retries=180, default_retry_delay=2)
def import_initial_website_source(
self,
*,
@ -358,6 +589,8 @@ def import_initial_website_source(
payload: dict[str, object],
workflow_id: str | None = None,
) -> str:
"""Compatibility task for already-enqueued website-only messages."""
try:
return start_initial_website_source_import(
tenant_id=tenant_id,
@ -383,4 +616,9 @@ def import_initial_website_source(
raise self.retry(exc=exc)
__all__ = ["import_initial_website_source", "start_initial_website_source_import"]
__all__ = [
"import_initial_source",
"import_initial_website_source",
"start_initial_source_import",
"start_initial_website_source_import",
]

184
api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py Normal file
View File

@ -0,0 +1,184 @@
from types import SimpleNamespace
from typing import cast
from unittest.mock import MagicMock, patch
import pytest
from core.datasource.entities.datasource_entities import (
OnlineDocumentInfo,
OnlineDocumentPage,
OnlineDocumentPagesMessage,
OnlineDriveBrowseFilesResponse,
OnlineDriveFile,
OnlineDriveFileBucket,
)
from models.account import Account
from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService
from services.knowledge_fs.product_dto import KnowledgeFSInitialSourcePreviewPayload
_CREDENTIAL = object()
def _service(credential=_CREDENTIAL) -> tuple[KnowledgeFSInitialSourcePreviewService, MagicMock]:
session = MagicMock()
session.scalar.return_value = credential
context = MagicMock()
context.__enter__.return_value = session
return KnowledgeFSInitialSourcePreviewService(MagicMock(return_value=context)), session
def _payload(kind: str) -> KnowledgeFSInitialSourcePreviewPayload:
return KnowledgeFSInitialSourcePreviewPayload.model_validate(
{
"credentialId": "credential-1",
"datasource": "pages" if kind == "online_document" else "drive",
"kind": kind,
"parameters": {},
"pluginId": "langgenius/provider",
"provider": "provider",
}
)
def test_preview_lists_online_documents_with_stable_import_identity() -> None:
service, _session = _service()
runtime = MagicMock()
runtime.datasource_provider_type.return_value = "online_document"
runtime.get_online_document_pages.return_value = [
OnlineDocumentPagesMessage(
result=[
OnlineDocumentInfo(
pages=[
OnlineDocumentPage(
last_edited_time="2026-08-10T00:00:00Z",
page_id="page-1",
page_name="Roadmap",
parent_id=None,
type="page",
)
],
total=1,
workspace_id="workspace-1",
workspace_name="Product",
)
]
)
]
with (
patch(
"services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials",
return_value={"token": "secret"},
),
patch(
"services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime",
return_value=runtime,
),
):
response = service.preview(
tenant_id="tenant-1",
account=cast(Account, SimpleNamespace(id="account-1")),
payload=_payload("online_document"),
)
assert response.kind == "online_document"
assert response.documents[0].provider_item_id == '["workspace-1","page-1"]'
assert response.documents[0].name == "Roadmap"
assert response.files == []
assert runtime.runtime.credentials == {"token": "secret"}
def test_preview_browses_online_drive_and_preserves_pagination() -> None:
service, _session = _service()
runtime = MagicMock()
runtime.datasource_provider_type.return_value = "online_drive"
runtime.online_drive_browse_files.return_value = [
OnlineDriveBrowseFilesResponse(
result=[
OnlineDriveFileBucket(
bucket="manuals",
files=[
OnlineDriveFile(
id="file-1",
name="Plan.pdf",
size=128,
type="application/pdf",
)
],
is_truncated=True,
next_page_parameters={"cursor": "next"},
)
]
)
]
payload = _payload("online_drive")
payload.parameters = {"prefix": "folder-1"}
with (
patch(
"services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials",
return_value={"token": "secret"},
),
patch(
"services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime",
return_value=runtime,
),
):
response = service.preview(
tenant_id="tenant-1",
account=cast(Account, SimpleNamespace(id="account-1")),
payload=payload,
)
assert response.kind == "online_drive"
assert response.files[0].provider_item_id == '["manuals","file-1"]'
assert response.files[0].mime_type == "application/pdf"
assert response.next_page_parameters == {"cursor": "next"}
request = runtime.online_drive_browse_files.call_args.kwargs["request"]
assert request.prefix == "folder-1"
def test_preview_exposes_an_empty_drive_bucket_as_a_browsable_container() -> None:
service, _session = _service()
runtime = MagicMock()
runtime.datasource_provider_type.return_value = "online_drive"
runtime.online_drive_browse_files.return_value = [
OnlineDriveBrowseFilesResponse(
result=[OnlineDriveFileBucket(bucket="manuals", files=[], is_truncated=False)]
)
]
with (
patch(
"services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials",
return_value={"token": "secret"},
),
patch(
"services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime",
return_value=runtime,
),
):
response = service.preview(
tenant_id="tenant-1",
account=cast(Account, SimpleNamespace(id="account-1")),
payload=_payload("online_drive"),
)
assert response.files[0].bucket == "manuals"
assert response.files[0].id == ""
assert response.files[0].provider_item_id == '["manuals",""]'
assert response.files[0].type == "bucket"
def test_preview_rejects_a_credential_hidden_from_the_account() -> None:
service, _session = _service(credential=None)
with (
patch(
"services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials"
) as get_credentials,
pytest.raises(PermissionError, match="credential is unavailable"),
):
service.preview(
tenant_id="tenant-1",
account=cast(Account, SimpleNamespace(id="account-1")),
payload=_payload("online_document"),
)
get_credentials.assert_not_called()

55
api/tests/unit_tests/services/test_knowledge_fs_product_application_service.py
View File

@ -245,7 +245,7 @@ def test_product_application_create_schedules_selected_website_import() -> None:
"services.knowledge_fs.product_application_service.uuid.uuid5",
return_value="operation-1",
),
patch("tasks.knowledge_fs_initial_source_tasks.import_initial_website_source.delay") as schedule_import,
patch("tasks.knowledge_fs_initial_source_tasks.import_initial_source.delay") as schedule_import,
):
application.create_space(tenant_id="tenant-1", account_id="account-1", payload=payload)
@ -258,6 +258,7 @@ def test_product_application_create_schedules_selected_website_import() -> None:
"kind": "website_crawl",
"name": "Dify docs",
"provider": "firecrawl",
"datasource": "crawl",
"root_url": "https://docs.dify.ai",
"crawl_options": {"include_subpages": True, "limit": 25},
"selection": [
@ -271,6 +272,58 @@ def test_product_application_create_schedules_selected_website_import() -> None:
)
def test_product_application_create_schedules_selected_connector_import() -> None:
application, _product, _control_plane, _commands, _facade, _rbac = _application()
payload = _create_payload(
initial_source={
"kind": "online_document",
"name": "Product wiki",
"pluginId": "langgenius/notion_datasource",
"provider": "notion",
"datasource": "pages",
"credentialId": "credential-1",
"selection": [
{
"name": "Roadmap",
"pageId": "page-1",
"providerItemId": "notion:page-1",
"type": "page",
"workspaceId": "workspace-1",
}
],
}
)
with (
patch(
"services.knowledge_fs.product_application_service.uuid.uuid5",
return_value="operation-1",
),
patch("tasks.knowledge_fs_initial_source_tasks.import_initial_source.delay") as schedule_import,
):
application.create_space(tenant_id="tenant-1", account_id="account-1", payload=payload)
scheduled_payload = schedule_import.call_args.kwargs["payload"]
assert scheduled_payload == {
"credential_id": "credential-1",
"datasource": "pages",
"kind": "online_document",
"name": "Product wiki",
"plugin_id": "langgenius/notion_datasource",
"provider": "notion",
"selection": [
{
"name": "Roadmap",
"page_id": "page-1",
"provider_item_id": "notion:page-1",
"type": "page",
"workspace_id": "workspace-1",
}
],
"sync_policy": "provider",
}
def test_product_application_create_generates_idempotency_and_skips_default_visibility_update() -> None:
application, _product, control_plane, commands, _facade, _rbac = _application()

70
api/tests/unit_tests/services/test_knowledge_fs_product_dto.py
View File

@ -45,6 +45,7 @@ from services.knowledge_fs.product_dto import (
KnowledgeFSSourceListQuery,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSourceWorkflowImportPayload,
KnowledgeFSSpaceCreatePayload,
KnowledgeFSSpaceListItemResponse,
KnowledgeFSStatResponse,
KnowledgeFSTraceResponse,
@ -55,6 +56,75 @@ from services.knowledge_fs.product_dto import (
)
def test_space_create_initial_source_is_a_backward_compatible_discriminated_union() -> None:
website = KnowledgeFSSpaceCreatePayload.model_validate(
{
"name": "Docs",
"slug": "docs",
"initial_source": {
"kind": "website_crawl",
"name": "Website",
"provider": "firecrawl",
"root_url": "https://docs.example.com",
"crawl_options": {},
"selection": [{"source_url": "https://docs.example.com/start"}],
},
}
)
assert website.initial_source is not None
assert website.initial_source.kind == "website_crawl"
document = KnowledgeFSSpaceCreatePayload.model_validate(
{
"name": "Docs",
"slug": "docs",
"initial_source": {
"kind": "online_document",
"name": "Wiki",
"pluginId": "langgenius/notion_datasource",
"provider": "notion",
"datasource": "pages",
"credentialId": "credential-1",
"selection": [
{
"pageId": "page-1",
"providerItemId": "notion:page-1",
"type": "page",
"workspaceId": "workspace-1",
}
],
},
}
)
assert document.initial_source is not None
assert document.initial_source.kind == "online_document"
assert document.initial_source.credential_id == "credential-1"
def test_connector_initial_source_requires_an_exact_credential_binding() -> None:
with pytest.raises(ValidationError):
KnowledgeFSSpaceCreatePayload.model_validate(
{
"name": "Docs",
"slug": "docs",
"initial_source": {
"kind": "online_drive",
"name": "Drive",
"pluginId": "langgenius/google_drive",
"provider": "google_drive",
"datasource": "google_drive",
"selection": [
{
"id": "file-1",
"name": "Plan.pdf",
"providerItemId": "google-drive:file-1",
}
],
},
}
)
def test_retrieval_test_payload_uses_bounded_kfs_filters_and_resolved_modes() -> None:
payload = KnowledgeFSRetrievalTestPayload.model_validate(
{

178
api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py
View File

@ -5,11 +5,17 @@ from unittest.mock import MagicMock, patch
import pytest
from models.knowledge_fs import KnowledgeFSControlSpaceState
from services.knowledge_fs.product_dto import KnowledgeFSInitialWebsiteSourcePayload
from services.knowledge_fs.product_dto import (
KnowledgeFSInitialOnlineDocumentSourcePayload,
KnowledgeFSInitialOnlineDriveSourcePayload,
KnowledgeFSInitialWebsiteSourcePayload,
)
from services.knowledge_fs.product_remote import KnowledgeFSProductResourceNotFoundError
from tasks.knowledge_fs_initial_source_tasks import (
KnowledgeFSInitialSourceNotReadyError,
import_initial_source,
import_initial_website_source,
start_initial_source_import,
start_initial_website_source_import,
)
@ -33,6 +39,51 @@ def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePaylo
)
def _document_payload() -> KnowledgeFSInitialOnlineDocumentSourcePayload:
return KnowledgeFSInitialOnlineDocumentSourcePayload.model_validate(
{
"kind": "online_document",
"name": "Product wiki",
"pluginId": "langgenius/notion_datasource",
"provider": "notion",
"datasource": "pages",
"credentialId": "notion-credential-1",
"selection": [
{
"lastEditedTime": "2026-08-10T00:00:00Z",
"name": "Roadmap",
"pageId": "page-1",
"providerItemId": "notion:page-1",
"type": "page",
"workspaceId": "workspace-1",
}
],
}
)
def _drive_payload() -> KnowledgeFSInitialOnlineDriveSourcePayload:
return KnowledgeFSInitialOnlineDriveSourcePayload.model_validate(
{
"kind": "online_drive",
"name": "Team drive",
"pluginId": "langgenius/google_drive",
"provider": "google_drive",
"datasource": "google_drive",
"credentialId": "drive-credential-1",
"selection": [
{
"id": "file-1",
"mimeType": "application/pdf",
"name": "Plan.pdf",
"providerItemId": "google-drive:file-1",
}
],
"sync_policy": "manual",
}
)
def _facade() -> MagicMock:
facade = MagicMock()
facade.list_sources.return_value = SimpleNamespace(data=[], next_cursor=None)
@ -42,7 +93,13 @@ def _facade() -> MagicMock:
facade.list_source_connections.return_value = SimpleNamespace(
data=[
SimpleNamespace(
configuration={"credentialId": "firecrawl-credential-1"},
configuration={
"credentialId": "firecrawl-credential-1",
"datasource": "crawl",
"pluginId": "langgenius/firecrawl_datasource",
"provider": "firecrawl",
"providerKind": "website",
},
id="connection-1",
provider_id="plugin-daemon-website",
status="active",
@ -165,6 +222,85 @@ def test_initial_website_source_import_recrawls_exact_selection_and_configures_d
assert sync_payload.expected_source_version == 4
@pytest.mark.parametrize(
("payload", "provider_id", "credential_id", "workflow_kind", "expected_source_name"),
[
(
_document_payload(),
"plugin-daemon-online-document",
"notion-credential-1",
"online-document-import",
"Product wiki",
),
(
_drive_payload(),
"plugin-daemon-online-drive",
"drive-credential-1",
"online-drive-import",
"Team drive",
),
],
)
def test_initial_connector_source_import_uses_exact_binding_and_selection(
payload: KnowledgeFSInitialOnlineDocumentSourcePayload | KnowledgeFSInitialOnlineDriveSourcePayload,
provider_id: str,
credential_id: str,
workflow_kind: str,
expected_source_name: str,
) -> None:
facade = _facade()
facade.list_source_providers.return_value = SimpleNamespace(data=[SimpleNamespace(id=provider_id, available=True)])
facade.list_source_connections.return_value = SimpleNamespace(data=[], next_cursor=None)
facade.create_source_connection.return_value = SimpleNamespace(id="connector-1", status="active")
facade.import_source_workflow.return_value = SimpleNamespace(
id="connector-workflow-1",
source_id="source-1",
state="completed",
)
facade.get_source.return_value = SimpleNamespace(
metadata={"clientRequestId": "initial-source:operation-1", "preview": True},
status="disabled",
version=3,
)
facade.update_source.return_value = SimpleNamespace(
metadata={"clientRequestId": "initial-source:operation-1", "preview": False},
status="active",
version=4,
)
credential = SimpleNamespace(id=credential_id, name=expected_source_name)
with _runtime(facade, credential=credential):
result = start_initial_source_import(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=payload,
)
assert result == "connector-workflow-1"
connection_payload = facade.create_source_connection.call_args.kwargs["payload"]
assert connection_payload.provider_id == provider_id
assert connection_payload.configuration == {
"credentialId": credential_id,
"datasource": payload.datasource,
"pluginId": payload.plugin_id,
"provider": payload.provider,
"providerKind": "online-document" if workflow_kind == "online-document-import" else "online-drive",
}
source_payload = facade.create_source.call_args.kwargs["payload"]
assert source_payload.connection_id == "connector-1"
assert source_payload.name == expected_source_name
assert source_payload.type == "connector"
assert source_payload.uri == "connector://connector-1"
assert source_payload.metadata["providerId"] == provider_id
import_payload = facade.import_source_workflow.call_args.kwargs["payload"].root
assert import_payload.kind == workflow_kind
assert import_payload.items == payload.selection
sync_payload = facade.update_source_sync_policy.call_args.kwargs["payload"]
assert sync_payload.mode == ("manual" if workflow_kind == "online-drive-import" else "provider")
def test_initial_website_source_import_reuses_source_across_pages_and_preserves_failure() -> None:
facade = _facade()
existing_source = SimpleNamespace(
@ -216,7 +352,13 @@ def test_initial_website_source_import_configures_remaining_sync_modes(
SimpleNamespace(
data=[
SimpleNamespace(
configuration={"credentialId": "firecrawl-credential-1"},
configuration={
"credentialId": "firecrawl-credential-1",
"datasource": "crawl",
"pluginId": "langgenius/firecrawl_datasource",
"provider": "firecrawl",
"providerKind": "website",
},
id="connection-2",
provider_id="plugin-daemon-website",
status="active",
@ -365,7 +507,13 @@ def test_initial_website_source_import_retries_provisioning_firecrawl_connection
facade.list_source_connections.return_value = SimpleNamespace(
data=[
SimpleNamespace(
configuration={"credentialId": "firecrawl-credential-1"},
configuration={
"credentialId": "firecrawl-credential-1",
"datasource": "crawl",
"pluginId": "langgenius/firecrawl_datasource",
"provider": "firecrawl",
"providerKind": "website",
},
id="connection-1",
provider_id="plugin-daemon-website",
status="provisioning",
@ -523,3 +671,25 @@ def test_initial_website_source_task_retries_with_workflow_id() -> None:
"workflow_id": "workflow-1",
},
)
def test_initial_source_task_validates_discriminated_connector_payload() -> None:
serialized_payload = _document_payload().model_dump(mode="json", by_alias=True)
with patch(
"tasks.knowledge_fs_initial_source_tasks.start_initial_source_import",
return_value="workflow-1",
) as start_import:
assert (
import_initial_source.run(
tenant_id="tenant-1",
account_id="account-1",
control_space_id="control-1",
operation_id="operation-1",
payload=serialized_payload,
)
== "workflow-1"
)
parsed_payload = start_import.call_args.kwargs["payload"]
assert isinstance(parsed_payload, KnowledgeFSInitialOnlineDocumentSourcePayload)
assert parsed_payload.credential_id == "notion-credential-1"

166
packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts
View File

@ -179,6 +179,8 @@ import {
zPatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponse,
zPostKnowledgeFsQueryStreamBody,
zPostKnowledgeFsQueryStreamResponse,
zPostKnowledgeFsSourceProviderPreviewBody,
zPostKnowledgeFsSourceProviderPreviewResponse,
zPostKnowledgeFsSpacesBody,
zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelPath,
zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse,
@ -363,6 +365,21 @@ export const researchTasks = {
byTaskId,
}
export const post2 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
operationId: 'postKnowledgeFsSourceProviderPreview',
path: '/knowledge-fs/source-provider-preview',
tags: ['console'],
})
.input(z.object({ body: zPostKnowledgeFsSourceProviderPreviewBody }))
.output(zPostKnowledgeFsSourceProviderPreviewResponse)
export const sourceProviderPreview = {
post: post2,
}
export const delete_ = oc
.route({
inputStructure: 'detailed',
@ -420,7 +437,7 @@ export const appBindings = {
byCallerKind,
}
export const post2 = oc
export const post3 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -436,10 +453,10 @@ export const post2 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse)
export const cancel = {
post: post2,
post: post3,
}
export const post3 = oc
export const post4 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -455,7 +472,7 @@ export const post3 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponse)
export const retry = {
post: post3,
post: post4,
}
export const byTaskId2 = {
@ -536,7 +553,7 @@ export const get6 = oc
.input(z.object({ params: zGetKnowledgeFsSpacesByControlSpaceIdCredentialsPath }))
.output(zGetKnowledgeFsSpacesByControlSpaceIdCredentialsResponse)
export const post4 = oc
export const post5 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -555,7 +572,7 @@ export const post4 = oc
export const credentials = {
get: get6,
post: post4,
post: post5,
byCredentialId,
}
@ -581,7 +598,7 @@ export const bulk = {
delete: delete3,
}
export const post5 = oc
export const post6 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -598,7 +615,7 @@ export const post5 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdDocumentsReindexResponse)
export const reindex = {
post: post5,
post: post6,
}
export const get7 = oc
@ -761,7 +778,7 @@ export const get12 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdDocumentsResponse)
export const post6 = oc
export const post7 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -780,7 +797,7 @@ export const post6 = oc
export const documents = {
get: get12,
post: post6,
post: post7,
bulk,
reindex,
byDocumentId,
@ -818,7 +835,7 @@ export const externalAccess = {
put: put2,
}
export const post7 = oc
export const post8 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -836,10 +853,10 @@ export const post7 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsBulkImportResponse)
export const bulkImport = {
post: post7,
post: post8,
}
export const post8 = oc
export const post9 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -856,7 +873,7 @@ export const post8 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsEvidenceMatchesResponse)
export const evidenceMatches = {
post: post8,
post: post9,
}
export const delete5 = oc
@ -910,7 +927,7 @@ export const get14 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsResponse)
export const post9 = oc
export const post10 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -929,13 +946,13 @@ export const post9 = oc
export const goldenQuestions = {
get: get14,
post: post9,
post: post10,
bulkImport,
evidenceMatches,
byQuestionId,
}
export const post10 = oc
export const post11 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -947,7 +964,7 @@ export const post10 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponse)
export const retry2 = {
post: post10,
post: post11,
}
export const delete6 = oc
@ -1112,7 +1129,7 @@ export const get18 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdMetadataResponse)
export const post11 = oc
export const post12 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1131,7 +1148,7 @@ export const post11 = oc
export const metadata = {
get: get18,
post: post11,
post: post12,
byFieldId,
}
@ -1337,7 +1354,7 @@ export const get28 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdQualityBadCasesResponse)
export const post12 = oc
export const post13 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1356,11 +1373,11 @@ export const post12 = oc
export const badCases = {
get: get28,
post: post12,
post: post13,
byBadCaseId,
}
export const post13 = oc
export const post14 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1379,7 +1396,7 @@ export const post13 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdQualityReplayRunsResponse)
export const replayRuns = {
post: post13,
post: post14,
}
export const quality = {
@ -1387,7 +1404,7 @@ export const quality = {
replayRuns,
}
export const post14 = oc
export const post15 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1404,13 +1421,13 @@ export const post14 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdQueriesAdmissionResponse)
export const admission = {
post: post14,
post: post15,
}
/**
* @deprecated
*/
export const post15 = oc
export const post16 = oc
.route({
deprecated: true,
inputStructure: 'detailed',
@ -1429,14 +1446,14 @@ export const post15 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdQueriesResponse)
export const queries = {
post: post15,
post: post16,
admission,
}
/**
* @deprecated
*/
export const post16 = oc
export const post17 = oc
.route({
deprecated: true,
inputStructure: 'detailed',
@ -1449,10 +1466,10 @@ export const post16 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdQueryStreamCapabilityResponse)
export const queryStreamCapability = {
post: post16,
post: post17,
}
export const post17 = oc
export const post18 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1469,7 +1486,7 @@ export const post17 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdResearchTasksPlanResponse)
export const plan = {
post: post17,
post: post18,
}
export const get29 = oc
@ -1536,7 +1553,7 @@ export const get31 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdResearchTasksResponse)
export const post18 = oc
export const post19 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1555,7 +1572,7 @@ export const post18 = oc
export const researchTasks2 = {
get: get31,
post: post18,
post: post19,
plan,
byTaskId: byTaskId3,
}
@ -1614,7 +1631,7 @@ export const settings = {
migrations,
}
export const post19 = oc
export const post20 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1631,7 +1648,7 @@ export const post19 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponse)
export const refresh = {
post: post19,
post: post20,
}
export const byConnectionId = {
@ -1654,7 +1671,7 @@ export const get34 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse)
export const post20 = oc
export const post21 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1673,7 +1690,7 @@ export const post20 = oc
export const sourceConnections = {
get: get34,
post: post20,
post: post21,
byConnectionId,
}
@ -1692,7 +1709,7 @@ export const sourceProviders = {
get: get35,
}
export const post21 = oc
export const post22 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1709,7 +1726,7 @@ export const post21 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponse)
export const cancel2 = {
post: post21,
post: post22,
}
export const get36 = oc
@ -1732,7 +1749,7 @@ export const pages = {
get: get36,
}
export const post22 = oc
export const post23 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1746,10 +1763,10 @@ export const post22 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponse)
export const retry3 = {
post: post22,
post: post23,
}
export const post23 = oc
export const post24 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1768,7 +1785,7 @@ export const post23 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponse)
export const selection = {
post: post23,
post: post24,
}
export const get37 = oc
@ -1794,7 +1811,7 @@ export const sourceWorkflows = {
byRunId,
}
export const post24 = oc
export const post25 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1813,10 +1830,10 @@ export const post24 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlImportResponse)
export const crawlImport = {
post: post24,
post: post25,
}
export const post25 = oc
export const post26 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1834,7 +1851,7 @@ export const post25 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponse)
export const crawlPreview = {
post: post25,
post: post26,
}
export const get38 = oc
@ -1857,7 +1874,7 @@ export const files = {
get: get38,
}
export const post26 = oc
export const post27 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1874,10 +1891,10 @@ export const post26 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdImportResponse)
export const import_ = {
post: post26,
post: post27,
}
export const post27 = oc
export const post28 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1894,7 +1911,7 @@ export const post27 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdImportFilesResponse)
export const importFiles = {
post: post27,
post: post28,
}
export const get39 = oc
@ -1917,7 +1934,7 @@ export const pages2 = {
get: get39,
}
export const post28 = oc
export const post29 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1935,7 +1952,7 @@ export const post28 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponse)
export const sync = {
post: post28,
post: post29,
}
export const get40 = oc
@ -1970,7 +1987,7 @@ export const syncPolicy = {
put: put4,
}
export const post29 = oc
export const post30 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -1982,10 +1999,10 @@ export const post29 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdTestResponse)
export const test = {
post: post29,
post: post30,
}
export const post30 = oc
export const post31 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2004,7 +2021,7 @@ export const post30 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdWorkflowImportsResponse)
export const workflowImports = {
post: post30,
post: post31,
}
export const delete10 = oc
@ -2085,7 +2102,7 @@ export const get42 = oc
)
.output(zGetKnowledgeFsSpacesByControlSpaceIdSourcesResponse)
export const post31 = oc
export const post32 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2104,7 +2121,7 @@ export const post31 = oc
export const sources = {
get: get42,
post: post31,
post: post32,
bySourceId,
}
@ -2207,7 +2224,7 @@ export const traces = {
byTraceId,
}
export const post32 = oc
export const post33 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2224,10 +2241,10 @@ export const post32 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdAbortResponse)
export const abort = {
post: post32,
post: post33,
}
export const post33 = oc
export const post34 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2244,10 +2261,10 @@ export const post33 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdCompleteResponse)
export const complete = {
post: post33,
post: post34,
}
export const post34 = oc
export const post35 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2268,7 +2285,7 @@ export const post34 = oc
)
export const presign = {
post: post34,
post: post35,
}
export const byPartNumber = {
@ -2279,7 +2296,7 @@ export const parts = {
byPartNumber,
}
export const post35 = oc
export const post36 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2296,7 +2313,7 @@ export const post35 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdSmallFileResponse)
export const smallFile = {
post: post35,
post: post36,
}
export const byUploadSessionId = {
@ -2306,7 +2323,7 @@ export const byUploadSessionId = {
smallFile,
}
export const post36 = oc
export const post37 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2325,7 +2342,7 @@ export const post36 = oc
.output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsResponse)
export const uploadSessions = {
post: post36,
post: post37,
byUploadSessionId,
}
@ -2409,7 +2426,7 @@ export const get49 = oc
.input(z.object({ query: zGetKnowledgeFsSpacesQuery.optional() }))
.output(zGetKnowledgeFsSpacesResponse)
export const post37 = oc
export const post38 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2423,11 +2440,11 @@ export const post37 = oc
export const spaces = {
get: get49,
post: post37,
post: post38,
byControlSpaceId,
}
export const post38 = oc
export const post39 = oc
.route({
inputStructure: 'detailed',
method: 'POST',
@ -2444,7 +2461,7 @@ export const post38 = oc
.output(zPostKnowledgeFsTasksByTaskIdStreamCapabilityResponse)
export const streamCapability = {
post: post38,
post: post39,
}
export const byTaskId4 = {
@ -2459,6 +2476,7 @@ export const knowledgeFs = {
wellKnown,
queryStream,
researchTasks,
sourceProviderPreview,
spaces,
tasks,
}

133
packages/contracts/generated/api/console/knowledge-fs/types.gen.ts
View File

@ -18,6 +18,26 @@ export type KnowledgeFsAdmittedQueryRequest = {
sessionId?: string | null
}
export type KnowledgeFsInitialSourcePreviewPayload = {
credentialId: string
datasource: string
kind: 'online_document' | 'online_drive'
parameters?: {
[key: string]: JsonValue
}
pluginId: string
provider: string
}
export type KnowledgeFsInitialSourcePreviewResponse = {
documents?: Array<KnowledgeFsInitialSourcePreviewDocumentResponse>
files?: Array<KnowledgeFsInitialSourcePreviewFileResponse>
kind: 'online_document' | 'online_drive'
next_page_parameters?: {
[key: string]: JsonValue
} | null
}
export type KnowledgeFsSpaceListResponse = {
data: Array<KnowledgeFsSpaceListItemResponse>
has_more: boolean
@ -30,7 +50,17 @@ export type KnowledgeFsSpaceCreatePayload = {
embedding?: KnowledgeFsModelIntent | null
icon?: string | null
idempotency_key?: string | null
initial_source?: KnowledgeFsInitialWebsiteSourcePayload | null
initial_source?:
| ({
kind: 'website_crawl'
} & KnowledgeFsInitialWebsiteSourcePayload)
| ({
kind: 'online_document'
} & KnowledgeFsInitialOnlineDocumentSourcePayload)
| ({
kind: 'online_drive'
} & KnowledgeFsInitialOnlineDriveSourcePayload)
| null
name: string
retrieval?: KnowledgeFsRetrievalProfileIntent | null
slug: string
@ -947,6 +977,28 @@ export type KnowledgeFsQueryImageReference = {
uploadFileId: string
}
export type JsonValue = unknown
export type KnowledgeFsInitialSourcePreviewDocumentResponse = {
last_edited_time?: string | null
name: string
page_id: string
provider_item_id: string
type: string
workspace_id: string
workspace_name?: string | null
}
export type KnowledgeFsInitialSourcePreviewFileResponse = {
bucket?: string | null
id: string
mime_type?: string | null
name: string
provider_item_id: string
size: number
type: string
}
export type KnowledgeFsSpaceListItemResponse = {
control_space_id: string
created_at: string
@ -970,14 +1022,39 @@ export type KnowledgeFsModelIntent = {
export type KnowledgeFsInitialWebsiteSourcePayload = {
crawl_options: KnowledgeFsInitialWebsiteCrawlOptionsPayload
credentialId?: string | null
datasource?: string
kind: 'website_crawl'
name: string
provider: 'firecrawl'
pluginId?: string | null
provider: string
root_url: string
selection: Array<KnowledgeFsInitialWebsiteSelectionPayload>
sync_policy?: 'daily' | 'manual' | 'provider'
}
export type KnowledgeFsInitialOnlineDocumentSourcePayload = {
credentialId: string
datasource: string
kind: 'online_document'
name: string
pluginId: string
provider: string
selection: Array<KnowledgeFsOnlineDocumentWorkflowImportItemPayload>
sync_policy?: 'daily' | 'manual' | 'provider'
}
export type KnowledgeFsInitialOnlineDriveSourcePayload = {
credentialId: string
datasource: string
kind: 'online_drive'
name: string
pluginId: string
provider: string
selection: Array<KnowledgeFsOnlineDriveWorkflowImportItemPayload>
sync_policy?: 'daily' | 'manual' | 'provider'
}
export type KnowledgeFsRetrievalProfileIntent = {
default_mode: 'deep' | 'fast' | 'research'
reasoning_model: KnowledgeFsModelIntent
@ -1478,6 +1555,25 @@ export type KnowledgeFsInitialWebsiteSelectionPayload = {
title?: string | null
}
export type KnowledgeFsOnlineDocumentWorkflowImportItemPayload = {
etag?: string | null
lastEditedTime?: string | null
name?: string | null
pageId: string
providerItemId: string
type: string
workspaceId: string
}
export type KnowledgeFsOnlineDriveWorkflowImportItemPayload = {
bucket?: string | null
etag?: string | null
id: string
mimeType?: string | null
name: string
providerItemId: string
}
export type KnowledgeFsRerankIntent = {
enabled: boolean
model?: KnowledgeFsModelIntent | null
@ -1578,25 +1674,6 @@ export type KnowledgeFsSourcePageResponse = {
type: string
}
export type KnowledgeFsOnlineDocumentWorkflowImportItemPayload = {
etag?: string | null
lastEditedTime?: string | null
name?: string | null
pageId: string
providerItemId: string
type: string
workspaceId: string
}
export type KnowledgeFsOnlineDriveWorkflowImportItemPayload = {
bucket?: string | null
etag?: string | null
id: string
mimeType?: string | null
name: string
providerItemId: string
}
export type KnowledgeFsTraceProfileResponse = {
embedding_model?: string | null
embedding_vector_space_id?: string | null
@ -1671,6 +1748,20 @@ export type GetKnowledgeFsResearchTasksByTaskIdEventsResponses = {
export type GetKnowledgeFsResearchTasksByTaskIdEventsResponse =
GetKnowledgeFsResearchTasksByTaskIdEventsResponses[keyof GetKnowledgeFsResearchTasksByTaskIdEventsResponses]
export type PostKnowledgeFsSourceProviderPreviewData = {
body: KnowledgeFsInitialSourcePreviewPayload
path?: never
query?: never
url: '/knowledge-fs/source-provider-preview'
}
export type PostKnowledgeFsSourceProviderPreviewResponses = {
200: KnowledgeFsInitialSourcePreviewResponse
}
export type PostKnowledgeFsSourceProviderPreviewResponse =
PostKnowledgeFsSourceProviderPreviewResponses[keyof PostKnowledgeFsSourceProviderPreviewResponses]
export type GetKnowledgeFsSpacesData = {
body?: never
path?: never

201
packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts
View File

@ -733,6 +733,56 @@ export const zKnowledgeFsResearchTaskPlanPayload = z.object({
topK: z.int().gte(1).lte(50).nullish(),
})
export const zJsonValue = z.unknown()
/**
* KnowledgeFSInitialSourcePreviewPayload
*/
export const zKnowledgeFsInitialSourcePreviewPayload = z.object({
credentialId: z.string().min(1).max(255),
datasource: z.string().min(1).max(255),
kind: z.enum(['online_document', 'online_drive']),
parameters: z.record(z.string(), zJsonValue).optional(),
pluginId: z.string().min(1).max(255),
provider: z.string().min(1).max(255),
})
/**
* KnowledgeFSInitialSourcePreviewDocumentResponse
*/
export const zKnowledgeFsInitialSourcePreviewDocumentResponse = z.object({
last_edited_time: z.string().nullish(),
name: z.string(),
page_id: z.string(),
provider_item_id: z.string(),
type: z.string(),
workspace_id: z.string(),
workspace_name: z.string().nullish(),
})
/**
* KnowledgeFSInitialSourcePreviewFileResponse
*/
export const zKnowledgeFsInitialSourcePreviewFileResponse = z.object({
bucket: z.string().nullish(),
id: z.string(),
mime_type: z.string().nullish(),
name: z.string(),
provider_item_id: z.string(),
size: z.int().gte(0),
type: z.string(),
})
/**
* KnowledgeFSInitialSourcePreviewResponse
*/
export const zKnowledgeFsInitialSourcePreviewResponse = z.object({
documents: z.array(zKnowledgeFsInitialSourcePreviewDocumentResponse).optional(),
files: z.array(zKnowledgeFsInitialSourcePreviewFileResponse).optional(),
kind: z.enum(['online_document', 'online_drive']),
next_page_parameters: z.record(z.string(), zJsonValue).nullish(),
})
/**
* KnowledgeFSModelIntent
*/
@ -1599,14 +1649,96 @@ export const zKnowledgeFsInitialWebsiteSelectionPayload = z.object({
*/
export const zKnowledgeFsInitialWebsiteSourcePayload = z.object({
crawl_options: zKnowledgeFsInitialWebsiteCrawlOptionsPayload,
credentialId: z.string().min(1).max(255).nullish(),
datasource: z.string().min(1).max(255).optional().default('crawl'),
kind: z.literal('website_crawl'),
name: z.string().min(1).max(200),
provider: z.literal('firecrawl'),
pluginId: z.string().min(1).max(255).nullish(),
provider: z.string().min(1).max(255),
root_url: z.string().min(1).max(4096),
selection: z.array(zKnowledgeFsInitialWebsiteSelectionPayload).min(1).max(200),
sync_policy: z.enum(['daily', 'manual', 'provider']).optional().default('provider'),
})
/**
* KnowledgeFSOnlineDocumentWorkflowImportItemPayload
*/
export const zKnowledgeFsOnlineDocumentWorkflowImportItemPayload = z.object({
etag: z.string().max(1024).nullish(),
lastEditedTime: z.string().max(128).nullish(),
name: z.string().max(500).nullish(),
pageId: z.string().min(1).max(1024),
providerItemId: z.string().min(1).max(1024),
type: z.string().min(1).max(128),
workspaceId: z.string().min(1).max(1024),
})
/**
* KnowledgeFSInitialOnlineDocumentSourcePayload
*/
export const zKnowledgeFsInitialOnlineDocumentSourcePayload = z.object({
credentialId: z.string().min(1).max(255),
datasource: z.string().min(1).max(255),
kind: z.literal('online_document'),
name: z.string().min(1).max(200),
pluginId: z.string().min(1).max(255),
provider: z.string().min(1).max(255),
selection: z.array(zKnowledgeFsOnlineDocumentWorkflowImportItemPayload).min(1).max(200),
sync_policy: z.enum(['daily', 'manual', 'provider']).optional().default('provider'),
})
/**
* KnowledgeFSOnlineDocumentWorkflowImportPayload
*/
export const zKnowledgeFsOnlineDocumentWorkflowImportPayload = z.object({
items: z.array(zKnowledgeFsOnlineDocumentWorkflowImportItemPayload).min(1).max(200),
kind: z.literal('online-document-import'),
})
/**
* KnowledgeFSOnlineDriveWorkflowImportItemPayload
*/
export const zKnowledgeFsOnlineDriveWorkflowImportItemPayload = z.object({
bucket: z.string().max(1024).nullish(),
etag: z.string().max(1024).nullish(),
id: z.string().min(1).max(1024),
mimeType: z.string().max(255).nullish(),
name: z.string().min(1).max(500),
providerItemId: z.string().min(1).max(1024),
})
/**
* KnowledgeFSInitialOnlineDriveSourcePayload
*/
export const zKnowledgeFsInitialOnlineDriveSourcePayload = z.object({
credentialId: z.string().min(1).max(255),
datasource: z.string().min(1).max(255),
kind: z.literal('online_drive'),
name: z.string().min(1).max(200),
pluginId: z.string().min(1).max(255),
provider: z.string().min(1).max(255),
selection: z.array(zKnowledgeFsOnlineDriveWorkflowImportItemPayload).min(1).max(200),
sync_policy: z.enum(['daily', 'manual', 'provider']).optional().default('provider'),
})
/**
* KnowledgeFSOnlineDriveWorkflowImportPayload
*/
export const zKnowledgeFsOnlineDriveWorkflowImportPayload = z.object({
items: z.array(zKnowledgeFsOnlineDriveWorkflowImportItemPayload).min(1).max(200),
kind: z.literal('online-drive-import'),
})
/**
* KnowledgeFSSourceWorkflowImportPayload
*/
export const zKnowledgeFsSourceWorkflowImportPayload = z.discriminatedUnion('kind', [
zKnowledgeFsOnlineDocumentWorkflowImportPayload.extend({
kind: z.literal('online-document-import'),
}),
zKnowledgeFsOnlineDriveWorkflowImportPayload.extend({ kind: z.literal('online-drive-import') }),
])
/**
* KnowledgeFSRerankIntent
*/
@ -1647,7 +1779,13 @@ export const zKnowledgeFsSpaceCreatePayload = z.object({
.regex(/^(?:builtin:)?[+a-z0-9_-]{1,64}$/)
.nullish(),
idempotency_key: z.string().min(1).max(255).nullish(),
initial_source: zKnowledgeFsInitialWebsiteSourcePayload.nullish(),
initial_source: z
.discriminatedUnion('kind', [
zKnowledgeFsInitialWebsiteSourcePayload.extend({ kind: z.literal('website_crawl') }),
zKnowledgeFsInitialOnlineDocumentSourcePayload.extend({ kind: z.literal('online_document') }),
zKnowledgeFsInitialOnlineDriveSourcePayload.extend({ kind: z.literal('online_drive') }),
])
.nullish(),
name: z.string().min(1).max(40),
retrieval: zKnowledgeFsRetrievalProfileIntent.nullish(),
slug: z
@ -2077,57 +2215,6 @@ export const zKnowledgeFsSourcePagesResponse = z.object({
workspaces: z.array(zKnowledgeFsSourceWorkspacePagesResponse),
})
/**
* KnowledgeFSOnlineDocumentWorkflowImportItemPayload
*/
export const zKnowledgeFsOnlineDocumentWorkflowImportItemPayload = z.object({
etag: z.string().max(1024).nullish(),
lastEditedTime: z.string().max(128).nullish(),
name: z.string().max(500).nullish(),
pageId: z.string().min(1).max(1024),
providerItemId: z.string().min(1).max(1024),
type: z.string().min(1).max(128),
workspaceId: z.string().min(1).max(1024),
})
/**
* KnowledgeFSOnlineDocumentWorkflowImportPayload
*/
export const zKnowledgeFsOnlineDocumentWorkflowImportPayload = z.object({
items: z.array(zKnowledgeFsOnlineDocumentWorkflowImportItemPayload).min(1).max(200),
kind: z.literal('online-document-import'),
})
/**
* KnowledgeFSOnlineDriveWorkflowImportItemPayload
*/
export const zKnowledgeFsOnlineDriveWorkflowImportItemPayload = z.object({
bucket: z.string().max(1024).nullish(),
etag: z.string().max(1024).nullish(),
id: z.string().min(1).max(1024),
mimeType: z.string().max(255).nullish(),
name: z.string().min(1).max(500),
providerItemId: z.string().min(1).max(1024),
})
/**
* KnowledgeFSOnlineDriveWorkflowImportPayload
*/
export const zKnowledgeFsOnlineDriveWorkflowImportPayload = z.object({
items: z.array(zKnowledgeFsOnlineDriveWorkflowImportItemPayload).min(1).max(200),
kind: z.literal('online-drive-import'),
})
/**
* KnowledgeFSSourceWorkflowImportPayload
*/
export const zKnowledgeFsSourceWorkflowImportPayload = z.discriminatedUnion('kind', [
zKnowledgeFsOnlineDocumentWorkflowImportPayload.extend({
kind: z.literal('online-document-import'),
}),
zKnowledgeFsOnlineDriveWorkflowImportPayload.extend({ kind: z.literal('online-drive-import') }),
])
/**
* KnowledgeFSTraceProfileResponse
*/
@ -2213,6 +2300,14 @@ export const zGetKnowledgeFsResearchTasksByTaskIdEventsQuery = z.object({
*/
export const zGetKnowledgeFsResearchTasksByTaskIdEventsResponse = z.record(z.string(), z.unknown())
export const zPostKnowledgeFsSourceProviderPreviewBody = zKnowledgeFsInitialSourcePreviewPayload
/**
* Datasource resources available for an initial Source
*/
export const zPostKnowledgeFsSourceProviderPreviewResponse =
zKnowledgeFsInitialSourcePreviewResponse
export const zGetKnowledgeFsSpacesQuery = z.object({
creator_ids: z.array(z.string().min(1).max(255)).max(100).optional(),
limit: z.int().gte(1).lte(100).optional().default(20),

355
web/features/new-rag/__tests__/create-knowledge-page.spec.tsx
View File

@ -10,6 +10,7 @@ const serviceMock = vi.hoisted(() => ({
createKfsSource: vi.fn(),
getKfsSource: vi.fn(),
getCrawlStatus: vi.fn(),
previewInitialSource: vi.fn(),
getDefaultModel: vi.fn(),
getSpace: vi.fn(),
getSyncPolicy: vi.fn(),
@ -27,6 +28,21 @@ const serviceMock = vi.hoisted(() => ({
documentsKey: vi.fn(() => ['console', 'knowledgeFs', 'documents']),
}))
const datasourceQueryMock = vi.hoisted(() => ({
auth: {
data: { result: [] as Array<Record<string, unknown>> },
error: null as unknown,
isPending: false,
refetch: vi.fn(),
},
plugins: {
data: [] as Array<Record<string, unknown>>,
error: null as unknown,
isPending: false,
refetch: vi.fn(),
},
}))
const routerMock = vi.hoisted(() => ({
back: vi.fn(),
replace: vi.fn(),
@ -82,6 +98,9 @@ vi.mock('jotai', async (importOriginal) => {
vi.mock('@/service/client', () => ({
consoleClient: {
knowledgeFs: {
sourceProviderPreview: {
post: serviceMock.previewInitialSource,
},
spaces: {
byControlSpaceId: {
get: serviceMock.getSpace,
@ -153,9 +172,161 @@ vi.mock('@/service/client', () => ({
vi.mock('@/service/datasets', () => ({
checkFirecrawlTaskStatus: serviceMock.getCrawlStatus,
checkJinaReaderTaskStatus: serviceMock.getCrawlStatus,
checkWatercrawlTaskStatus: serviceMock.getCrawlStatus,
createFirecrawlTask: serviceMock.createCrawl,
createJinaReaderTask: serviceMock.createCrawl,
createWatercrawlTask: serviceMock.createCrawl,
}))
vi.mock('@/service/use-pipeline', () => ({
useDataSourceList: () => datasourceQueryMock.plugins,
}))
vi.mock('@/service/use-datasource', () => ({
useGetDataSourceListAuth: () => datasourceQueryMock.auth,
}))
const firecrawlDatasourcePlugin = {
declaration: {
credentials_schema: [],
datasources: [
{
description: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' },
identity: {
author: 'langgenius',
label: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' },
name: 'crawl',
provider: 'firecrawl',
},
parameters: [],
},
],
identity: {
author: 'langgenius',
description: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' },
icon: 'icon.svg',
label: { en_US: 'Firecrawl', zh_Hans: 'Firecrawl' },
name: 'firecrawl',
tags: [],
},
provider_type: 'website_crawl',
},
is_authorized: true,
plugin_id: 'langgenius/firecrawl_datasource',
plugin_unique_identifier: 'langgenius/firecrawl_datasource:1.0.0@local',
provider: 'firecrawl',
}
const firecrawlDatasourceAuth = {
author: 'langgenius',
credentials_list: [
{
avatar_url: '',
credential: {},
id: 'firecrawl-credential-1',
is_default: true,
name: 'Default Firecrawl',
type: 'api-key',
},
],
description: { en_US: 'Firecrawl' },
icon: 'icon.svg',
label: { en_US: 'Firecrawl' },
name: 'firecrawl',
plugin_id: 'langgenius/firecrawl_datasource',
plugin_unique_identifier: 'langgenius/firecrawl_datasource:1.0.0@local',
provider: 'firecrawl',
}
const notionDatasourcePlugin = {
...firecrawlDatasourcePlugin,
declaration: {
...firecrawlDatasourcePlugin.declaration,
datasources: [
{
description: { en_US: 'Notion', zh_Hans: 'Notion' },
identity: {
author: 'langgenius',
label: { en_US: 'Notion', zh_Hans: 'Notion' },
name: 'notion',
provider: 'notion',
},
parameters: [],
},
],
identity: {
...firecrawlDatasourcePlugin.declaration.identity,
label: { en_US: 'Notion', zh_Hans: 'Notion' },
name: 'notion',
},
provider_type: 'online_document',
},
plugin_id: 'langgenius/notion_datasource',
plugin_unique_identifier: 'langgenius/notion_datasource:1.0.0@local',
provider: 'notion',
}
const notionDatasourceAuth = {
...firecrawlDatasourceAuth,
credentials_list: [
{
...firecrawlDatasourceAuth.credentials_list[0],
id: 'notion-credential-1',
name: 'Default Notion',
},
],
label: { en_US: 'Notion' },
name: 'notion',
plugin_id: 'langgenius/notion_datasource',
plugin_unique_identifier: 'langgenius/notion_datasource:1.0.0@local',
provider: 'notion',
}
const googleDriveDatasourcePlugin = {
...firecrawlDatasourcePlugin,
declaration: {
...firecrawlDatasourcePlugin.declaration,
datasources: [
{
description: { en_US: 'Google Drive', zh_Hans: 'Google Drive' },
identity: {
author: 'langgenius',
label: { en_US: 'Google Drive', zh_Hans: 'Google Drive' },
name: 'google_drive',
provider: 'google_drive',
},
parameters: [],
},
],
identity: {
...firecrawlDatasourcePlugin.declaration.identity,
label: { en_US: 'Google Drive', zh_Hans: 'Google Drive' },
name: 'google_drive',
},
provider_type: 'online_drive',
},
plugin_id: 'langgenius/google_drive',
plugin_unique_identifier: 'langgenius/google_drive:1.0.0@local',
provider: 'google_drive',
}
const googleDriveDatasourceAuth = {
...firecrawlDatasourceAuth,
credentials_list: [
{
...firecrawlDatasourceAuth.credentials_list[0],
id: 'google-drive-credential-1',
name: 'Default Google Drive',
},
],
label: { en_US: 'Google Drive' },
name: 'google_drive',
plugin_id: 'langgenius/google_drive',
plugin_unique_identifier: 'langgenius/google_drive:1.0.0@local',
provider: 'google_drive',
}
const createdKnowledge = {
control_space_id: 'e735c1dc-d2b8-4dc4-86dc-abaf2fb7d084',
model_setup_required: false,
@ -249,6 +420,12 @@ describe('CreateKnowledgePage', () => {
beforeEach(() => {
vi.clearAllMocks()
globalThis.sessionStorage.clear()
datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin]
datasourceQueryMock.plugins.error = null
datasourceQueryMock.plugins.isPending = false
datasourceQueryMock.auth.data = { result: [firecrawlDatasourceAuth] }
datasourceQueryMock.auth.error = null
datasourceQueryMock.auth.isPending = false
serviceMock.create.mockResolvedValue(createdKnowledge)
serviceMock.createCrawl.mockResolvedValue({ job_id: 'crawl-job-1' })
serviceMock.createKfsSource.mockResolvedValue(kfsSourceResponse())
@ -735,7 +912,7 @@ describe('CreateKnowledgePage', () => {
)
})
it('enables only source options supported by atomic creation', async () => {
it('enables every atomic source type and distinguishes installed providers', async () => {
const user = userEvent.setup()
renderPage()
@ -756,24 +933,17 @@ describe('CreateKnowledgePage', () => {
const onlineDocuments = screen.getByRole('radio', {
name: 'dataset.newKnowledge.onlineDocuments',
})
expect(onlineDocuments).toHaveAttribute('aria-disabled', 'true')
expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDrive' })).toHaveAttribute(
'aria-disabled',
'true',
)
expect(onlineDocuments).toBeEnabled()
expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDrive' })).toBeEnabled()
expect(screen.getByRole('radio', { name: 'Firecrawl' })).toBeChecked()
for (const unavailableProvider of ['Jina Reader', 'WaterCrawl', 'FakeCrawler']) {
expect(screen.getByRole('radio', { name: unavailableProvider })).toHaveAttribute(
'aria-disabled',
'true',
)
}
expect(screen.getByRole('radio', { name: 'Jina Reader' })).toBeEnabled()
expect(screen.getByRole('radio', { name: 'WaterCrawl' })).toBeEnabled()
await user.click(onlineDocuments)
expect(onlineDocuments).not.toBeChecked()
expect(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' })).toBeChecked()
expect(
screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' }),
).toBeDisabled()
expect(onlineDocuments).toBeChecked()
expect(screen.getByRole('radio', { name: 'Notion' })).toBeChecked()
expect(screen.getByText('workflow.nodes.common.pluginNotInstalled')).toBeInTheDocument()
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' }))
expect(screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' })).toBeEnabled()
expect(screen.queryByRole('alert')).not.toBeInTheDocument()
expect(screen.getByText('dataset.newKnowledge.crawlOptions')).toBeInTheDocument()
expect(
@ -998,8 +1168,11 @@ describe('CreateKnowledgePage', () => {
include_subpages: true,
limit: 100,
},
credentialId: 'firecrawl-credential-1',
datasource: 'crawl',
kind: 'website_crawl',
name: 'Dify docs',
pluginId: 'langgenius/firecrawl_datasource',
provider: 'firecrawl',
root_url: 'https://docs.dify.ai',
selection: [
@ -1017,6 +1190,154 @@ describe('CreateKnowledgePage', () => {
expect(serviceMock.selectWorkflowPages).not.toHaveBeenCalled()
})
it('distinguishes an installed document provider with no credential', async () => {
const user = userEvent.setup()
navigationMock.startMode = 'source'
datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, notionDatasourcePlugin]
renderPage()
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' }))
expect(
screen.getByText('dataset.newKnowledge.providerNotConfigured:{"provider":"Notion"}'),
).toBeInTheDocument()
expect(
screen.getByRole('button', {
name: 'dataset.newKnowledge.connectProvider:{"provider":"Notion"}',
}),
).toBeEnabled()
expect(screen.queryByText('workflow.nodes.common.pluginNotInstalled')).not.toBeInTheDocument()
})
it('creates a knowledge space atomically with selected online documents', async () => {
const user = userEvent.setup()
navigationMock.startMode = 'source'
datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, notionDatasourcePlugin]
datasourceQueryMock.auth.data = {
result: [firecrawlDatasourceAuth, notionDatasourceAuth],
}
serviceMock.previewInitialSource.mockResolvedValue({
documents: [
{
last_edited_time: '2026-08-10T08:00:00Z',
name: 'Product handbook',
page_id: 'page-1',
provider_item_id: '["workspace-1","page-1"]',
type: 'page',
workspace_id: 'workspace-1',
workspace_name: 'Dify',
},
],
kind: 'online_document',
next_page_parameters: null,
})
renderPage()
await fillRequiredFields(user)
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' }))
await user.type(
screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
'Notion handbook',
)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' }))
await user.click(await screen.findByRole('checkbox', { name: 'Product handbook' }))
await waitFor(() =>
expect(
screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }),
).toBeEnabled(),
)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce())
expect(serviceMock.create).toHaveBeenCalledWith({
body: expect.objectContaining({
initial_source: {
credentialId: 'notion-credential-1',
datasource: 'notion',
kind: 'online_document',
name: 'Notion handbook',
pluginId: 'langgenius/notion_datasource',
provider: 'notion',
selection: [
{
lastEditedTime: '2026-08-10T08:00:00Z',
name: 'Product handbook',
pageId: 'page-1',
providerItemId: '["workspace-1","page-1"]',
type: 'page',
workspaceId: 'workspace-1',
},
],
sync_policy: 'provider',
},
}),
})
})
it('creates a knowledge space atomically with a selected drive file', async () => {
const user = userEvent.setup()
navigationMock.startMode = 'source'
datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, googleDriveDatasourcePlugin]
datasourceQueryMock.auth.data = {
result: [firecrawlDatasourceAuth, googleDriveDatasourceAuth],
}
serviceMock.previewInitialSource.mockResolvedValue({
files: [
{
bucket: null,
id: 'file-1',
mime_type: 'application/pdf',
name: 'Runbook.pdf',
provider_item_id: '["","file-1"]',
size: 2048,
type: 'application/pdf',
},
],
kind: 'online_drive',
next_page_parameters: null,
})
renderPage()
await fillRequiredFields(user)
await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDrive' }))
await user.type(
screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
'Drive runbook',
)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' }))
await user.click(await screen.findByRole('checkbox', { name: 'Runbook.pdf' }))
await waitFor(() =>
expect(
screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }),
).toBeEnabled(),
)
await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce())
expect(serviceMock.create).toHaveBeenCalledWith({
body: expect.objectContaining({
initial_source: {
credentialId: 'google-drive-credential-1',
datasource: 'google_drive',
kind: 'online_drive',
name: 'Drive runbook',
pluginId: 'langgenius/google_drive',
provider: 'google_drive',
selection: [
{
bucket: undefined,
id: 'file-1',
mimeType: 'application/pdf',
name: 'Runbook.pdf',
providerItemId: '["","file-1"]',
},
],
sync_policy: 'provider',
},
}),
})
})
it('requires a selected website preview page before creating with an initial source', async () => {
const user = userEvent.setup()
navigationMock.startMode = 'source'

410
web/features/new-rag/create-connected-source-setup.tsx Normal file
View File

@ -0,0 +1,410 @@
'use client'
import type {
KnowledgeFsInitialSourcePreviewDocumentResponse,
KnowledgeFsInitialSourcePreviewFileResponse,
KnowledgeFsSpaceCreatePayload,
} from '@dify/contracts/api/console/knowledge-fs/types.gen'
import type {
NewKnowledgeOnlineDocumentsSourceDraft,
NewKnowledgeOnlineDriveSourceDraft,
NewKnowledgeSourceDraft,
} from './routes'
import type { InstalledSourceProviderOption } from './source-provider-options'
import type { DataSourceCredential } from '@/app/components/header/account-setting/data-source-page-new/types'
import { Button } from '@langgenius/dify-ui/button'
import { Checkbox } from '@langgenius/dify-ui/checkbox'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
import Loading from '@/app/components/base/loading'
import { consoleClient } from '@/service/client'
import { SourceNameField, SourceSyncPolicyField } from './source-setup-fields'
type ConnectedDraft = NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft
type InitialSource = NonNullable<KnowledgeFsSpaceCreatePayload['initial_source']>
type PreviewDocument = KnowledgeFsInitialSourcePreviewDocumentResponse
type PreviewFile = KnowledgeFsInitialSourcePreviewFileResponse
type PreviewResource =
| {
depth: number
document: PreviewDocument
key: string
kind: 'document'
parentKey?: string
}
| { depth: number; file: PreviewFile; key: string; kind: 'file'; parentKey?: string }
type NextPageRequest = {
bucket?: string
depth: number
nextPage: Record<string, unknown>
parentKey?: string
prefix?: string
}
function isDriveContainer(file: PreviewFile) {
return /bucket|directory|folder|workspace/i.test(file.type)
}
function resourceName(resource: PreviewResource) {
return resource.kind === 'document' ? resource.document.name : resource.file.name
}
function resourceIcon(resource: PreviewResource) {
if (resource.kind === 'file' && isDriveContainer(resource.file))
return 'i-ri-folder-3-fill text-text-warning'
return resource.kind === 'document'
? 'i-ri-file-text-line text-text-tertiary'
: 'i-ri-file-3-line text-text-tertiary'
}
export function CreateConnectedSourceSetup({
credential,
disabled,
draft,
providerOption,
onDraftChange,
onInitialSourceChange,
}: {
credential: DataSourceCredential
disabled: boolean
draft: ConnectedDraft
providerOption: InstalledSourceProviderOption
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
onInitialSourceChange: (source?: InitialSource) => void
}) {
const { t } = useTranslation('dataset')
const [resources, setResources] = useState<PreviewResource[]>([])
const [selected, setSelected] = useState<Set<string>>(() => new Set())
const [expanded, setExpanded] = useState<Set<string>>(() => new Set())
const [loading, setLoading] = useState(false)
const [loadingMore, setLoadingMore] = useState(false)
const [error, setError] = useState(false)
const [previewed, setPreviewed] = useState(false)
const [nextPageRequest, setNextPageRequest] = useState<NextPageRequest | null>()
const selectableResources = useMemo(
() =>
resources.filter(
(resource) => resource.kind === 'document' || !isDriveContainer(resource.file),
),
[resources],
)
const visibleResources = useMemo(() => {
const byKey = new Map(resources.map((resource) => [resource.key, resource]))
return resources.filter((resource) => {
let parentKey = resource.parentKey
while (parentKey) {
if (!expanded.has(parentKey)) return false
parentKey = byKey.get(parentKey)?.parentKey
}
return true
})
}, [expanded, resources])
useEffect(() => {
const selectedResources = selectableResources.filter((resource) => selected.has(resource.key))
const name = draft.sourceName.trim()
if (!name || !selectedResources.length) {
onInitialSourceChange(undefined)
return
}
const binding = {
credentialId: credential.id,
datasource: providerOption.datasource.identity.name,
pluginId: providerOption.plugin.plugin_id,
provider: providerOption.plugin.provider,
}
if (draft.sourceType === 'onlineDocuments') {
onInitialSourceChange({
...binding,
kind: 'online_document',
name,
selection: selectedResources.flatMap((resource) =>
resource.kind === 'document'
? [
{
lastEditedTime: resource.document.last_edited_time ?? undefined,
name: resource.document.name,
pageId: resource.document.page_id,
providerItemId: resource.document.provider_item_id,
type: resource.document.type,
workspaceId: resource.document.workspace_id,
},
]
: [],
),
sync_policy: draft.syncPolicy,
})
return
}
onInitialSourceChange({
...binding,
kind: 'online_drive',
name,
selection: selectedResources.flatMap((resource) =>
resource.kind === 'file'
? [
{
bucket: resource.file.bucket ?? undefined,
id: resource.file.id,
mimeType: resource.file.mime_type ?? undefined,
name: resource.file.name,
providerItemId: resource.file.provider_item_id,
},
]
: [],
),
sync_policy: draft.syncPolicy,
})
}, [
credential.id,
draft,
onInitialSourceChange,
providerOption.datasource.identity.name,
providerOption.plugin.plugin_id,
providerOption.plugin.provider,
selectableResources,
selected,
])
const requestPreview = useCallback(
async ({
append = false,
bucket,
depth = 0,
nextPage,
parentKey,
prefix,
}: {
append?: boolean
bucket?: string
depth?: number
nextPage?: Record<string, unknown>
parentKey?: string
prefix?: string
} = {}) => {
append ? setLoadingMore(true) : setLoading(true)
setError(false)
try {
const response = await consoleClient.knowledgeFs.sourceProviderPreview.post({
body: {
credentialId: credential.id,
datasource: providerOption.datasource.identity.name,
kind: draft.sourceType === 'onlineDocuments' ? 'online_document' : 'online_drive',
parameters: {
...(bucket ? { bucket } : {}),
...(prefix ? { prefix } : {}),
...(nextPage ? { next_page_parameters: nextPage } : {}),
},
pluginId: providerOption.plugin.plugin_id,
provider: providerOption.plugin.provider,
},
})
const nextResources: PreviewResource[] =
draft.sourceType === 'onlineDocuments'
? (response.documents ?? []).map((document) => ({
depth,
document,
key: `document:${document.provider_item_id}`,
kind: 'document' as const,
parentKey,
}))
: (response.files ?? []).map((file) => ({
depth,
file,
key: `file:${file.provider_item_id}`,
kind: 'file' as const,
parentKey,
}))
setResources((current) => {
const next = new Map((append ? current : []).map((resource) => [resource.key, resource]))
for (const resource of nextResources) next.set(resource.key, resource)
return [...next.values()]
})
if (parentKey) setExpanded((current) => new Set(current).add(parentKey))
setNextPageRequest(
response.next_page_parameters
? {
bucket,
depth,
nextPage: response.next_page_parameters,
parentKey,
prefix,
}
: null,
)
setPreviewed(true)
} catch {
setError(true)
} finally {
setLoading(false)
setLoadingMore(false)
}
},
[credential.id, draft.sourceType, providerOption],
)
const toggle = (key: string) => {
setSelected((current) => {
const next = new Set(current)
if (next.has(key)) next.delete(key)
else next.add(key)
return next
})
}
const toggleAll = () => {
setSelected((current) => {
const next = new Set(current)
const allSelected =
selectableResources.length > 0 &&
selectableResources.every((resource) => current.has(resource.key))
for (const resource of selectableResources) {
if (allSelected) next.delete(resource.key)
else next.add(resource.key)
}
return next
})
}
const expandContainer = (resource: Extract<PreviewResource, { kind: 'file' }>) => {
if (expanded.has(resource.key)) {
setExpanded((current) => {
const next = new Set(current)
next.delete(resource.key)
return next
})
return
}
void requestPreview({
append: true,
bucket: resource.file.bucket ?? undefined,
depth: resource.depth + 1,
parentKey: resource.key,
prefix: resource.file.id || undefined,
})
}
return (
<div className="space-y-4">
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<SourceNameField
disabled={disabled}
draft={draft}
preventSubmitOnEnter
size="medium"
onDraftChange={onDraftChange}
/>
<SourceSyncPolicyField
availablePolicies={draft.provider === 'Amazon S3' ? ['daily', 'manual'] : undefined}
disabled={disabled}
draft={draft}
size="medium"
onDraftChange={onDraftChange}
/>
</div>
{!previewed && (
<Button
type="button"
variant="primary"
className="w-full"
loading={loading}
disabled={disabled || loading}
onClick={() => void requestPreview()}
>
{t(($) => $['newKnowledge.preview'])}
</Button>
)}
{loading && (
<div className="flex min-h-40 items-center justify-center rounded-lg border border-divider-subtle">
<Loading />
</div>
)}
{error && (
<div role="alert" className="rounded-lg bg-background-section p-4">
<p className="system-sm-semibold text-text-primary">
{t(($) => $['newKnowledge.providerLoadFailed'])}
</p>
<Button className="mt-3" onClick={() => void requestPreview()}>
{t(($) => $['newKnowledge.retryProviderLoad'])}
</Button>
</div>
)}
{previewed && !loading && (
<section className="overflow-hidden rounded-lg border border-divider-subtle bg-background-default">
<div className="flex items-center gap-2 border-b border-divider-subtle px-3 py-2">
<Checkbox
aria-label={t(($) => $['newKnowledge.selectAll'])}
checked={
selectableResources.length > 0 &&
selectableResources.every((resource) => selected.has(resource.key))
}
disabled={disabled || !selectableResources.length}
onCheckedChange={toggleAll}
/>
<span className="system-xs-medium text-text-secondary">
{draft.sourceType === 'onlineDocuments'
? t(($) => $['newKnowledge.selectPagesToSync'])
: t(($) => $['newKnowledge.selectFilesAndFolders'])}
</span>
<span className="ml-auto system-xs-regular text-text-tertiary">
{t(($) => $['newKnowledge.pagesSelected'], { count: selected.size })}
</span>
</div>
<ul className="max-h-64 overflow-y-auto p-1.5">
{visibleResources.map((resource) => {
const container = resource.kind === 'file' && isDriveContainer(resource.file)
return (
<li
key={resource.key}
className="flex min-h-8 items-center gap-2 rounded-md px-2 hover:bg-state-base-hover"
style={{ paddingLeft: `${8 + resource.depth * 20}px` }}
>
{container ? (
<Button
type="button"
variant="ghost"
size="small"
className="size-5 px-0"
aria-label={resourceName(resource)}
disabled={disabled || loadingMore}
onClick={() => expandContainer(resource)}
>
<span
aria-hidden
className={`i-ri-arrow-right-s-line size-4 transition-transform ${
expanded.has(resource.key) ? 'rotate-90' : ''
}`}
/>
</Button>
) : (
<Checkbox
aria-label={resourceName(resource)}
checked={selected.has(resource.key)}
disabled={disabled}
onCheckedChange={() => toggle(resource.key)}
/>
)}
<span aria-hidden className={`${resourceIcon(resource)} size-4 shrink-0`} />
<span className="min-w-0 flex-1 truncate system-xs-regular text-text-primary">
{resourceName(resource)}
</span>
</li>
)
})}
</ul>
{nextPageRequest && (
<div className="border-t border-divider-subtle px-3 py-2 text-center">
<Button
type="button"
size="small"
variant="ghost"
loading={loadingMore}
onClick={() => void requestPreview({ append: true, ...nextPageRequest })}
>
{t(($) => $['newKnowledge.loadMore'])}
</Button>
</div>
)}
</section>
)}
</div>
)
}

85
web/features/new-rag/create-knowledge-page.tsx
View File

@ -1,12 +1,11 @@
'use client'
import type {
KnowledgeFsInitialWebsiteSourcePayload,
KnowledgeFsSpaceCreatePayload,
KnowledgeFsSpaceCreateResponse,
} from '@dify/contracts/api/console/knowledge-fs/types.gen'
import type { CreateKnowledgeExitReason } from './components/create-knowledge-exit-dialog'
import type { KnowledgeVisibility } from './create-knowledge-workflow'
import type { WebsiteCrawlPreviewSelection } from './create-source-setup'
import type { QueuedUpload } from './create-upload-queue'
import type { KnowledgeFsUploadPhase, KnowledgeFsUploadProgress } from './knowledge-fs-upload'
import type { NewKnowledgeSourceDraft, NewKnowledgeStartMode } from './routes'
@ -64,58 +63,19 @@ import { uploadKnowledgeFsDocuments } from './knowledge-fs-upload'
import { createRequestId } from './request-id'
import {
createNewKnowledgeSourceDraft,
isValidWebsiteSourceDraft,
newKnowledgeDetailPath,
newKnowledgeDocumentsPath,
newKnowledgeListPath,
newKnowledgeSettingsPath,
} from './routes'
type InitialSource = NonNullable<KnowledgeFsSpaceCreatePayload['initial_source']>
function normalizeStartMode(value: string | null): NewKnowledgeStartMode {
if (value === 'source' || value === 'upload') return value
return 'empty'
}
function initialWebsiteSourceFromSelection(
draft: NewKnowledgeSourceDraft,
selection?: WebsiteCrawlPreviewSelection,
): KnowledgeFsInitialWebsiteSourcePayload | undefined {
if (
draft.sourceType !== 'websiteCrawl' ||
draft.provider !== 'Firecrawl' ||
!selection ||
selection.draft.sourceType !== 'websiteCrawl' ||
selection.draft.provider !== draft.provider ||
selection.draft.rootUrl !== draft.rootUrl ||
selection.draft.sourceName !== draft.sourceName ||
selection.draft.includeSubpages !== draft.includeSubpages ||
selection.draft.maxPages !== draft.maxPages ||
selection.draft.syncPolicy !== draft.syncPolicy
)
return undefined
const selectedPages = selection.pages.filter((page) =>
selection.selectedPageIds.includes(page.pageId),
)
if (!selectedPages.length) return undefined
return {
crawl_options: {
include_subpages: draft.includeSubpages,
limit: draft.maxPages,
},
kind: 'website_crawl',
name: draft.sourceName.trim(),
provider: 'firecrawl',
root_url: draft.rootUrl,
selection: selectedPages.map((page) => ({
source_url: page.sourceUrl,
...(page.title ? { title: page.title } : {}),
})),
sync_policy: draft.syncPolicy,
}
}
export function CreateKnowledgePage() {
const { t } = useTranslation('dataset')
const { t: tCommon } = useTranslation('common')
@ -141,9 +101,8 @@ export function CreateKnowledgePage() {
const [sourceDraft, setSourceDraft] = useState<NewKnowledgeSourceDraft>(() =>
createNewKnowledgeSourceDraft('websiteCrawl'),
)
const [websitePreviewSelection, setWebsitePreviewSelection] =
useState<WebsiteCrawlPreviewSelection>()
const websitePreviewSelectionRef = useRef<WebsiteCrawlPreviewSelection | undefined>(undefined)
const [initialSource, setInitialSource] = useState<InitialSource>()
const initialSourceRef = useRef<InitialSource | undefined>(undefined)
const [uploads, setUploads] = useState<QueuedUpload[]>([])
const [createdKnowledge, setCreatedKnowledge] = useState<KnowledgeFsSpaceCreateResponse>()
const [modelSetupDialogOpen, setModelSetupDialogOpen] = useState(false)
@ -167,15 +126,7 @@ export function CreateKnowledgePage() {
const uploadSubmissionBlocked =
startMode === 'upload' &&
(!uploadAvailable || !uploads.length || uploads.some((upload) => upload.issue))
const initialWebsiteSource = initialWebsiteSourceFromSelection(
sourceDraft,
websitePreviewSelection,
)
const sourceSubmissionBlocked =
startMode === 'source' &&
(sourceDraft.sourceType === 'websiteCrawl'
? !isValidWebsiteSourceDraft(sourceDraft) || !initialWebsiteSource
: !sourceDraft.sourceName.trim())
const sourceSubmissionBlocked = startMode === 'source' && !initialSource
const sourceDraftChanged =
JSON.stringify(sourceDraft) !==
JSON.stringify(createNewKnowledgeSourceDraft(sourceDraft.sourceType))
@ -189,9 +140,9 @@ export function CreateKnowledgePage() {
createdKnowledge,
)
const updateWebsitePreviewSelection = useCallback((selection?: WebsiteCrawlPreviewSelection) => {
websitePreviewSelectionRef.current = selection
setWebsitePreviewSelection(selection)
const updateInitialSource = useCallback((source?: InitialSource) => {
initialSourceRef.current = source
setInitialSource(source)
}, [])
const armHistoryGuard = useCallback(() => {
@ -321,14 +272,8 @@ export function CreateKnowledgePage() {
const normalizedDescription = description.trim()
if (!normalizedName) return
const latestWebsitePreviewSelection =
websitePreviewSelectionRef.current ?? websitePreviewSelection
const initialSource = initialWebsiteSourceFromSelection(
sourceDraft,
latestWebsitePreviewSelection,
)
if (startMode === 'source' && sourceDraft.sourceType === 'websiteCrawl' && !initialSource)
return
const latestInitialSource = initialSourceRef.current ?? initialSource
if (startMode === 'source' && !latestInitialSource) return
idempotencyKeyRef.current ??= createRequestId()
setSubmissionLocked(true)
@ -337,7 +282,7 @@ export function CreateKnowledgePage() {
existingKnowledge: createdKnowledge,
description: normalizedDescription,
idempotencyKey: idempotencyKeyRef.current,
initialSource,
initialSource: startMode === 'source' ? latestInitialSource : undefined,
name: normalizedName,
onCreated: (knowledgeSpace) => {
setCreatedKnowledge(knowledgeSpace)
@ -573,14 +518,12 @@ export function CreateKnowledgePage() {
draft={sourceDraft}
onDraftChange={(value) => {
setSourceDraft(value)
updateWebsitePreviewSelection(undefined)
resetUnsubmittedError()
}}
onWebsitePreviewSelectionChange={updateWebsitePreviewSelection}
onInitialSourceChange={updateInitialSource}
onSourceTypeChange={(value) => {
if (value !== 'websiteCrawl') return
setSourceDraft(createNewKnowledgeSourceDraft(value))
updateWebsitePreviewSelection(undefined)
updateInitialSource(undefined)
resetUnsubmittedError()
}}
/>

3
web/features/new-rag/create-knowledge-workflow.ts
View File

@ -1,6 +1,5 @@
import type {
KnowledgeFsControlSpaceVisibility,
KnowledgeFsInitialWebsiteSourcePayload,
KnowledgeFsModelIntent,
KnowledgeFsSpaceCreatePayload,
KnowledgeFsSpaceCreateResponse,
@ -21,7 +20,7 @@ type CreateKnowledgeValues = {
existingKnowledge?: KnowledgeFsSpaceCreateResponse
description: string
idempotencyKey: string
initialSource?: KnowledgeFsInitialWebsiteSourcePayload
initialSource?: NonNullable<KnowledgeFsSpaceCreatePayload['initial_source']>
name: string
onCreated: (knowledgeSpace: KnowledgeFsSpaceCreateResponse) => void
visibility: KnowledgeVisibility

416
web/features/new-rag/create-source-setup.tsx
View File

@ -1,13 +1,12 @@
'use client'
import type {
NewKnowledgeOnlineDocumentsProvider,
NewKnowledgeOnlineDriveProvider,
NewKnowledgeSourceDraft,
NewKnowledgeWebsiteProvider,
NewKnowledgeWebsiteSourceDraft,
} from './routes'
import type { KnowledgeFsSpaceCreatePayload } from '@dify/contracts/api/console/knowledge-fs/types.gen'
import type { NewKnowledgeSourceDraft } from './routes'
import type { CrawlPreviewPage } from './source-models'
import type {
DataSourceAuth,
DataSourceCredential,
} from '@/app/components/header/account-setting/data-source-page-new/types'
import type { CrawlResultItem } from '@/models/datasets'
import { Button } from '@langgenius/dify-ui/button'
import { Checkbox } from '@langgenius/dify-ui/checkbox'
@ -25,13 +24,29 @@ import {
} from '@langgenius/dify-ui/number-field'
import { useEffect, useMemo, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
import { buildIntegrationPath } from '@/app/components/integrations/routes'
import {
checkFirecrawlTaskStatus,
checkJinaReaderTaskStatus,
checkWatercrawlTaskStatus,
createFirecrawlTask,
createJinaReaderTask,
createWatercrawlTask,
} from '@/service/datasets'
import { useGetDataSourceListAuth } from '@/service/use-datasource'
import { useDataSourceList } from '@/service/use-pipeline'
import { CrawlPreviewPageSelection } from './crawl-selection-form'
import { CreateConnectedSourceSetup } from './create-connected-source-setup'
import { isValidWebsiteSourceDraft, NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH } from './routes'
import {
discoverSourceProviderOptions,
normalizeSourceProviderName,
sourceProviderOptionForDraft,
} from './source-provider-options'
import {
SourceConnectionRequiredCard,
SourceNameField,
SourceProviderNotInstalledCard,
SourceProviderRadioGroup,
SourceSyncPolicyField,
SourceTypeSelector,
@ -48,12 +63,7 @@ const CRAWL_PREVIEW_SKELETONS = [
] as const
type LocalCrawlState = 'error' | 'idle' | 'running' | 'stopped' | 'success'
export type WebsiteCrawlPreviewSelection = {
draft: NewKnowledgeWebsiteSourceDraft
pages: CrawlPreviewPage[]
selectedPageIds: string[]
}
type InitialSource = NonNullable<KnowledgeFsSpaceCreatePayload['initial_source']>
function crawlPages(response: Record<string, unknown>): CrawlResultItem[] {
if (!Array.isArray(response.data)) return []
@ -92,129 +102,56 @@ function crawlPreviewPages(pages: CrawlResultItem[]): CrawlPreviewPage[] {
}))
}
const providers = {
onlineDocuments: [
{ available: false, icon: 'i-custom-public-common-notion', label: 'Notion' },
{
available: false,
icon: 'i-ri-file-text-fill text-[#4d8bf5]',
label: 'Google Docs',
},
{ available: false, icon: 'i-custom-public-common-confluence', label: 'Confluence' },
],
onlineDrive: [
{ available: false, icon: 'i-custom-public-common-google-drive', label: 'Google Drive' },
{ available: false, icon: 'i-ri-cloud-line', label: 'OneDrive' },
{ available: false, icon: 'i-ri-box-3-line', label: 'Amazon S3' },
],
websiteCrawl: [
{ icon: 'i-custom-public-common-firecrawl', label: 'Firecrawl', available: true },
{ available: false, icon: 'i-custom-public-llm-jina', label: 'Jina Reader' },
{
available: false,
icon: 'i-custom-public-knowledge-watercrawl',
label: 'WaterCrawl',
},
{
available: false,
icon: 'i-ri-global-line text-text-accent',
label: 'FakeCrawler',
},
],
} as const
function ConnectedSourceConfiguration({
disabled,
draft,
onDraftChange,
}: {
disabled: boolean
draft: NewKnowledgeSourceDraft
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
}) {
return (
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<SourceNameField
disabled={disabled}
draft={draft}
preventSubmitOnEnter
size="medium"
onDraftChange={onDraftChange}
/>
<SourceSyncPolicyField
disabled={disabled}
draft={draft}
size="medium"
onDraftChange={onDraftChange}
/>
</div>
function datasourceAuthForProvider(
authProviders: DataSourceAuth[],
pluginId: string,
provider: string,
) {
return authProviders.find(
(candidate) => candidate.plugin_id === pluginId && candidate.provider === provider,
)
}
function NotionSourceConfiguration({
disabled,
draft,
onConnect,
onDraftChange,
}: {
disabled: boolean
draft: NewKnowledgeSourceDraft
onConnect: () => void
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
}) {
const { t } = useTranslation('dataset')
const datasourceAuthQuery = useGetDataSourceListAuth()
const notionConnected = Boolean(
datasourceAuthQuery.data?.result.some(
(auth) =>
auth.credentials_list.length > 0 &&
[auth.name, auth.plugin_id, auth.provider].some((identity) =>
identity
.toLocaleLowerCase()
.replaceAll(/[^a-z0-9]/g, '')
.includes('notion'),
),
),
)
if (notionConnected) {
return (
<ConnectedSourceConfiguration
disabled={disabled}
draft={draft}
onDraftChange={onDraftChange}
/>
)
}
function preferredCredential(auth?: DataSourceAuth): DataSourceCredential | undefined {
return (
<SourceConnectionRequiredCard
actionLabel={t(($) => $['newKnowledge.connectNotion'])}
description={t(($) => $['newKnowledge.notionNotConnectedDescription'])}
disabled={disabled}
icon={<span aria-hidden className="i-custom-public-common-notion size-4.5" />}
title={t(($) => $['newKnowledge.notionNotConnected'])}
onConnect={onConnect}
/>
auth?.credentials_list.find((credential) => credential.is_default) ?? auth?.credentials_list[0]
)
}
function providerIntegrationPath(packageId?: string) {
const base = buildIntegrationPath('data-source')
if (!packageId) return base
const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) })
return `${base}?${query.toString()}`
}
function websiteProviderTransport(provider: string) {
const normalized = normalizeSourceProviderName(provider)
if (normalized.includes('firecrawl'))
return { check: checkFirecrawlTaskStatus, create: createFirecrawlTask }
if (normalized.includes('jinareader') || normalized === 'jina')
return { check: checkJinaReaderTaskStatus, create: createJinaReaderTask }
if (normalized.includes('watercrawl'))
return { check: checkWatercrawlTaskStatus, create: createWatercrawlTask }
}
export function CreateSourceSetup({
disabled,
draft,
onDraftChange,
onWebsitePreviewSelectionChange,
onInitialSourceChange,
onSourceTypeChange,
}: {
disabled: boolean
draft: NewKnowledgeSourceDraft
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
onWebsitePreviewSelectionChange?: (selection?: WebsiteCrawlPreviewSelection) => void
onInitialSourceChange: (source?: InitialSource) => void
onSourceTypeChange: (sourceType: NewKnowledgeSourceDraft['sourceType']) => void
}) {
const { t } = useTranslation('dataset')
const datasourcePluginsQuery = useDataSourceList(true)
const datasourceAuthQuery = useGetDataSourceListAuth()
const [optionsExpanded, setOptionsExpanded] = useState(false)
const [backendBoundaryVisible, setBackendBoundaryVisible] = useState(false)
const [crawlState, setCrawlState] = useState<LocalCrawlState>('idle')
const [previewPages, setPreviewPages] = useState<CrawlResultItem[]>([])
const [selectedPageIds, setSelectedPageIds] = useState<Set<string>>(() => new Set())
@ -222,19 +159,35 @@ export function CreateSourceSetup({
const pollResolveRef = useRef<(() => void) | undefined>(undefined)
const pollTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined)
const sourceType = draft.sourceType
const availableProviders = providers[sourceType]
const activeProvider = availableProviders.some((provider) => provider.label === draft.provider)
? draft.provider
: availableProviders[0].label
const websiteProviderAvailable =
draft.sourceType === 'websiteCrawl' && draft.provider === 'Firecrawl'
const previewReady = websiteProviderAvailable && isValidWebsiteSourceDraft(draft)
const providerOptions = useMemo(
() => discoverSourceProviderOptions(sourceType, datasourcePluginsQuery.data ?? []),
[datasourcePluginsQuery.data, sourceType],
)
const providerOption = sourceProviderOptionForDraft(providerOptions, draft)
const installedProviderOption = providerOption?.installed ? providerOption : undefined
const datasourceAuth = installedProviderOption
? datasourceAuthForProvider(
datasourceAuthQuery.data?.result ?? [],
installedProviderOption.plugin.plugin_id,
installedProviderOption.plugin.provider,
)
: undefined
const credential = preferredCredential(datasourceAuth)
const websiteTransport =
draft.sourceType === 'websiteCrawl' && installedProviderOption
? websiteProviderTransport(installedProviderOption.plugin.provider)
: undefined
const previewReady = Boolean(
websiteTransport &&
credential &&
draft.sourceType === 'websiteCrawl' &&
isValidWebsiteSourceDraft(draft),
)
const previewRootUrl = draft.sourceType === 'websiteCrawl' ? draft.rootUrl : ''
const selectionPages = useMemo(() => crawlPreviewPages(previewPages), [previewPages])
const crawlOptionsAreDefault =
draft.sourceType !== 'websiteCrawl' ||
(draft.includeSubpages === DEFAULT_INCLUDE_SUBPAGES && draft.maxPages === DEFAULT_MAX_PAGES)
const showBackendBoundary = () => setBackendBoundaryVisible(true)
const stopPreview = (state: LocalCrawlState = 'stopped') => {
crawlAttemptRef.current += 1
globalThis.clearTimeout(pollTimerRef.current)
@ -247,23 +200,29 @@ export function CreateSourceSetup({
stopPreview('idle')
setPreviewPages([])
setSelectedPageIds(new Set())
onWebsitePreviewSelectionChange?.(undefined)
onInitialSourceChange(undefined)
}
const updateDraft = (nextDraft: NewKnowledgeSourceDraft) => {
onDraftChange(nextDraft)
setBackendBoundaryVisible(false)
resetPreview()
}
const updateDraftWithoutReset = (nextDraft: NewKnowledgeSourceDraft) => {
onDraftChange(nextDraft)
setBackendBoundaryVisible(false)
}
const selectProvider = (provider: string) => {
if (draft.sourceType === 'onlineDocuments')
updateDraft({ ...draft, provider: provider as NewKnowledgeOnlineDocumentsProvider })
else if (draft.sourceType === 'onlineDrive')
updateDraft({ ...draft, provider: provider as NewKnowledgeOnlineDriveProvider })
else updateDraft({ ...draft, provider: provider as NewKnowledgeWebsiteProvider })
const selectProvider = (providerKey: string) => {
const nextProvider = providerOptions.find((option) => option.key === providerKey)
if (!nextProvider) return
updateDraft({
...draft,
provider: nextProvider.label,
providerKey: nextProvider.key,
sourceName: '',
...(draft.sourceType === 'onlineDrive' &&
nextProvider.label === 'Amazon S3' &&
draft.syncPolicy === 'provider'
? { syncPolicy: 'daily' as const }
: {}),
})
}
useEffect(
@ -276,7 +235,14 @@ export function CreateSourceSetup({
)
const startPreview = async () => {
if (draft.sourceType !== 'websiteCrawl' || !isValidWebsiteSourceDraft(draft)) return
if (
draft.sourceType !== 'websiteCrawl' ||
!isValidWebsiteSourceDraft(draft) ||
!websiteTransport ||
!credential ||
!installedProviderOption
)
return
const attempt = crawlAttemptRef.current + 1
crawlAttemptRef.current = attempt
globalThis.clearTimeout(pollTimerRef.current)
@ -284,7 +250,7 @@ export function CreateSourceSetup({
setSelectedPageIds(new Set())
setCrawlState('running')
try {
const created = (await createFirecrawlTask({
const created = (await websiteTransport.create({
options: {
crawl_sub_pages: draft.includeSubpages,
excludes: '',
@ -300,7 +266,7 @@ export function CreateSourceSetup({
if (!jobId) throw new Error('Website crawl did not return a job id')
while (crawlAttemptRef.current === attempt) {
const response = (await checkFirecrawlTaskStatus(jobId)) as Record<string, unknown>
const response = (await websiteTransport.check(jobId)) as Record<string, unknown>
if (crawlAttemptRef.current !== attempt) return
setPreviewPages(crawlPages(response))
if (response.status === 'completed') {
@ -326,38 +292,59 @@ export function CreateSourceSetup({
const updateSelectedPageIds = (pageIds: Set<string>) => {
setSelectedPageIds(pageIds)
if (draft.sourceType !== 'websiteCrawl' || crawlState !== 'success' || !selectionPages.length) {
onWebsitePreviewSelectionChange?.(undefined)
return
}
onWebsitePreviewSelectionChange?.({
draft,
pages: selectionPages,
selectedPageIds: [...pageIds],
})
}
useEffect(() => {
if (draft.sourceType !== 'websiteCrawl' || crawlState !== 'success' || !selectionPages.length) {
onWebsitePreviewSelectionChange?.(undefined)
if (
draft.sourceType !== 'websiteCrawl' ||
crawlState !== 'success' ||
!selectionPages.length ||
!installedProviderOption ||
!credential
) {
onInitialSourceChange(undefined)
return
}
onWebsitePreviewSelectionChange?.({
draft,
pages: selectionPages,
selectedPageIds: [...selectedPageIds],
const selectedPages = selectionPages.filter((page) => selectedPageIds.has(page.pageId))
if (!selectedPages.length) {
onInitialSourceChange(undefined)
return
}
onInitialSourceChange({
crawl_options: {
include_subpages: draft.includeSubpages,
limit: draft.maxPages,
},
credentialId: credential.id,
datasource: installedProviderOption.datasource.identity.name,
kind: 'website_crawl',
name: draft.sourceName.trim(),
pluginId: installedProviderOption.plugin.plugin_id,
provider: installedProviderOption.plugin.provider,
root_url: draft.rootUrl,
selection: selectedPages.map((page) => ({
source_url: page.sourceUrl,
...(page.title ? { title: page.title } : {}),
})),
sync_policy: draft.syncPolicy,
})
}, [crawlState, draft, onWebsitePreviewSelectionChange, selectedPageIds, selectionPages])
}, [
crawlState,
credential,
draft,
installedProviderOption,
onInitialSourceChange,
selectedPageIds,
selectionPages,
])
return (
<div className="mx-4 -mt-1 mb-3.75 flex flex-col gap-4">
<SourceTypeSelector
appearance="embedded"
disabled={disabled}
disabledValues={['onlineDocuments', 'onlineDrive']}
value={sourceType}
onChange={(value) => {
setBackendBoundaryVisible(false)
onSourceTypeChange(value)
}}
/>
@ -374,29 +361,92 @@ export function CreateSourceSetup({
type="button"
variant="ghost-accent"
size="small"
disabled
disabled={disabled}
className="gap-0.5 px-2.75"
onClick={showBackendBoundary}
onClick={() =>
globalThis.open(buildIntegrationPath('data-source'), '_blank', 'noopener,noreferrer')
}
>
{t(($) => $['newKnowledge.moreProviders'])}
<span aria-hidden className="i-ri-arrow-right-up-line size-3.5" />
</Button>
</div>
<SourceProviderRadioGroup
value={activeProvider}
value={providerOption?.key ?? ''}
disabled={disabled}
layout={sourceType === 'websiteCrawl' ? 'grid-four' : 'grid-three'}
options={providers[sourceType].map((provider) => ({
disabled: !provider.available,
icon: <span aria-hidden className={`${provider.icon} size-4 shrink-0`} />,
value: provider.label,
options={providerOptions.map((option) => ({
icon: <span aria-hidden className={`${option.fallbackIcon} size-4 shrink-0`} />,
label: option.label,
value: option.key,
}))}
surface="default"
onChange={selectProvider}
/>
</Fieldset>
{draft.sourceType === 'websiteCrawl' && (
{datasourcePluginsQuery.isPending || datasourceAuthQuery.isPending ? (
<div className="flex min-h-44 items-center justify-center">
<span aria-hidden className="i-ri-loader-4-line size-5 animate-spin text-text-tertiary" />
</div>
) : datasourcePluginsQuery.error || datasourceAuthQuery.error ? (
<div className="rounded-xl bg-background-section p-4">
<p className="system-sm-semibold text-text-primary">
{t(($) => $['newKnowledge.providerLoadFailed'])}
</p>
<Button
className="mt-3"
onClick={() =>
void Promise.all([datasourcePluginsQuery.refetch(), datasourceAuthQuery.refetch()])
}
>
{t(($) => $['newKnowledge.retryProviderLoad'])}
</Button>
</div>
) : providerOption && !providerOption.installed ? (
<SourceProviderNotInstalledCard
icon={<span aria-hidden className={`${providerOption.fallbackIcon} size-4.5`} />}
provider={providerOption.label}
onInstall={() =>
globalThis.open(
providerIntegrationPath(providerOption.packageId),
'_blank',
'noopener,noreferrer',
)
}
/>
) : installedProviderOption && !credential ? (
<SourceConnectionRequiredCard
actionLabel={t(($) => $['newKnowledge.connectProvider'], {
provider: installedProviderOption.label,
})}
description={t(($) => $['newKnowledge.providerCredentialRequiredDescription'], {
provider: installedProviderOption.label,
})}
disabled={disabled}
icon={<span aria-hidden className={`${installedProviderOption.fallbackIcon} size-4.5`} />}
title={t(($) => $['newKnowledge.providerNotConfigured'], {
provider: installedProviderOption.label,
})}
onConnect={() =>
globalThis.open(
providerIntegrationPath(installedProviderOption.packageId),
'_blank',
'noopener,noreferrer',
)
}
/>
) : draft.sourceType === 'websiteCrawl' &&
installedProviderOption &&
credential &&
!websiteTransport ? (
<div className="rounded-xl bg-background-section p-4">
<p className="system-sm-semibold text-text-primary">{installedProviderOption.label}</p>
<p className="mt-1 system-xs-regular text-text-tertiary">
{t(($) => $['newKnowledge.providerUnavailable'])}
</p>
</div>
) : draft.sourceType === 'websiteCrawl' && installedProviderOption && credential ? (
<div className="space-y-4">
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<Field name="rootUrl" className="gap-1.5">
@ -618,35 +668,17 @@ export function CreateSourceSetup({
/>
)}
</div>
)}
{draft.sourceType !== 'websiteCrawl' && (
<>
{draft.sourceType === 'onlineDocuments' && activeProvider === 'Notion' ? (
<NotionSourceConfiguration
disabled={disabled || !websiteProviderAvailable}
draft={draft}
onConnect={showBackendBoundary}
onDraftChange={updateDraft}
/>
) : (
<ConnectedSourceConfiguration
disabled={disabled || !websiteProviderAvailable}
draft={draft}
onDraftChange={updateDraft}
/>
)}
</>
)}
{backendBoundaryVisible && (
<p
role="alert"
className="rounded-md bg-components-badge-status-light-warning-bg px-3 py-2 system-xs-regular text-text-warning"
>
{t(($) => $['newKnowledge.sourceSetupBackendDependency'])}
</p>
)}
) : draft.sourceType !== 'websiteCrawl' && installedProviderOption && credential ? (
<CreateConnectedSourceSetup
key={`${draft.sourceType}:${installedProviderOption.key}:${credential.id}`}
credential={credential}
disabled={disabled}
draft={draft}
providerOption={installedProviderOption}
onDraftChange={updateDraftWithoutReset}
onInitialSourceChange={onInitialSourceChange}
/>
) : null}
</div>
)
}