diff --git a/api/controllers/console/knowledge_fs/resources.py b/api/controllers/console/knowledge_fs/resources.py index 86d515f41b5..2bdfaf595c1 100644 --- a/api/controllers/console/knowledge_fs/resources.py +++ b/api/controllers/console/knowledge_fs/resources.py @@ -60,6 +60,11 @@ from services.knowledge_fs.download_service import ( KnowledgeFSDownloadTooLargeError, ) from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService +from services.knowledge_fs.initial_source_preview_job import ( + KnowledgeFSInitialSourcePreviewJobAlreadyRunningError, + KnowledgeFSInitialSourcePreviewJobNotFoundError, + KnowledgeFSInitialSourcePreviewJobService, +) from services.knowledge_fs.product_authorization import ( KnowledgeFSProductNotFoundError, ) @@ -119,8 +124,11 @@ from services.knowledge_fs.product_dto import ( KnowledgeFSGoldenQuestionPayload, KnowledgeFSGoldenQuestionResponse, KnowledgeFSIdempotencyHeader, + KnowledgeFSInitialSourcePreviewJobCreateResponse, + KnowledgeFSInitialSourcePreviewJobResponse, KnowledgeFSInitialSourcePreviewPayload, KnowledgeFSInitialSourcePreviewResponse, + KnowledgeFSInitialWebsiteSourcePreviewPayload, KnowledgeFSJWKSResponse, KnowledgeFSLogicalDocumentDeletePayload, KnowledgeFSLogicalDocumentListResponse, @@ -281,6 +289,7 @@ register_schema_models( KnowledgeFSCrawlImportPayload, KnowledgeFSCrawlPreviewSelectionPayload, KnowledgeFSInitialSourcePreviewPayload, + KnowledgeFSInitialWebsiteSourcePreviewPayload, KnowledgeFSSourceDeletePayload, KnowledgeFSSourceDeleteQuery, KnowledgeFSSourceFilesQuery, @@ -380,6 +389,8 @@ register_response_schema_models( KnowledgeFSOverviewStatsResponse, KnowledgeFSPresignedUploadResponse, KnowledgeFSInitialSourcePreviewResponse, + KnowledgeFSInitialSourcePreviewJobCreateResponse, + KnowledgeFSInitialSourcePreviewJobResponse, KnowledgeFSUploadSessionCreateResponse, KnowledgeFSUploadSessionMutationResponse, ) @@ -401,6 +412,8 @@ def _knowledge_fs_errors[**P, R](view: Callable[P, R]) -> Callable[P, R]: raise KnowledgeFSSpaceNotFoundHTTPError() from exc except KnowledgeFSOperationUnavailableError as exc: raise KnowledgeFSOperationUnavailableHTTPError() from exc + except KnowledgeFSInitialSourcePreviewJobAlreadyRunningError as exc: + raise Conflict() from exc except KnowledgeFSProductResourceNotFoundError as exc: raise NotFound() from exc except KnowledgeFSStagedUploadNotFoundError as exc: @@ -666,6 +679,73 @@ class KnowledgeFSInitialSourcePreviewApi(Resource): return dump_response(KnowledgeFSInitialSourcePreviewResponse, result) +@console_ns.route("/knowledge-fs/source-provider-preview/jobs") +class KnowledgeFSInitialSourcePreviewJobsApi(Resource): + @console_ns.expect(console_ns.models[KnowledgeFSInitialWebsiteSourcePreviewPayload.__name__]) + @console_ns.response( + HTTPStatus.ACCEPTED, + "Website datasource preview queued", + console_ns.models[KnowledgeFSInitialSourcePreviewJobCreateResponse.__name__], + ) + @setup_required + @login_required + @account_initialization_required + @_knowledge_fs_errors + def post(self): + account, tenant_id = current_account_with_tenant() + result = KnowledgeFSInitialSourcePreviewJobService(session_factory.get_session_maker()).start( + tenant_id=tenant_id, + account=account, + payload=_payload(KnowledgeFSInitialWebsiteSourcePreviewPayload), + ) + return dump_response(KnowledgeFSInitialSourcePreviewJobCreateResponse, result), HTTPStatus.ACCEPTED + + +@console_ns.route("/knowledge-fs/source-provider-preview/jobs/") +class KnowledgeFSInitialSourcePreviewJobApi(Resource): + @console_ns.response( + HTTPStatus.OK, + "Website datasource preview status", + console_ns.models[KnowledgeFSInitialSourcePreviewJobResponse.__name__], + ) + @setup_required + @login_required + @account_initialization_required + @_knowledge_fs_errors + def get(self, job_id: str): + account, tenant_id = current_account_with_tenant() + try: + result = KnowledgeFSInitialSourcePreviewJobService.get( + tenant_id=tenant_id, + account_id=account.id, + job_id=job_id, + ) + except KnowledgeFSInitialSourcePreviewJobNotFoundError as exc: + raise NotFound() from exc + return dump_response(KnowledgeFSInitialSourcePreviewJobResponse, result) + + @console_ns.response( + HTTPStatus.OK, + "Website datasource preview canceled", + console_ns.models[KnowledgeFSInitialSourcePreviewJobResponse.__name__], + ) + @setup_required + @login_required + @account_initialization_required + @_knowledge_fs_errors + def delete(self, job_id: str): + account, tenant_id = current_account_with_tenant() + try: + result = KnowledgeFSInitialSourcePreviewJobService.cancel( + tenant_id=tenant_id, + account_id=account.id, + job_id=job_id, + ) + except KnowledgeFSInitialSourcePreviewJobNotFoundError as exc: + raise NotFound() from exc + return dump_response(KnowledgeFSInitialSourcePreviewJobResponse, result) + + @console_ns.route("/knowledge-fs/uploads") class KnowledgeFSStagedUploadsApi(Resource): @console_ns.doc(consumes=["multipart/form-data"], params=_STAGED_UPLOAD_PARAMS) diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.py index 4b0ab407dfe..680a1a59aaf 100644 --- a/api/extensions/ext_celery.py +++ b/api/extensions/ext_celery.py @@ -175,6 +175,7 @@ def init_app(app: DifyApp) -> Celery: "tasks.refresh_billing_vector_space_task", # billing vector-space cache refresh "tasks.app_generate.resume_agent_app_task", # ENG-635: Agent v2 chat ask_human resume "tasks.workflow_run_archive_download_tasks", # workflow-run archive download preparation + "tasks.knowledge_fs_initial_source_preview_tasks", # datasource previews use the standard dataset queue ] day = dify_config.CELERY_BEAT_SCHEDULER_TIME diff --git a/api/services/datasource_provider_service.py b/api/services/datasource_provider_service.py index 7ab68247771..61fc74ec50d 100644 --- a/api/services/datasource_provider_service.py +++ b/api/services/datasource_provider_service.py @@ -197,6 +197,7 @@ class DatasourceProviderService: provider: str, plugin_id: str, credential_id: str | None = None, + current_user: Any | None = None, ) -> dict[str, Any]: """ Return decrypted datasource credentials. @@ -230,7 +231,8 @@ class DatasourceProviderService: if not datasource_provider: return {} if self._should_refresh_credentials(datasource_provider): - current_user = get_current_user() + if current_user is None: + current_user = get_current_user() encrypted_credentials, expires_at = self._refresh_datasource_credentials( tenant_id=tenant_id, provider=provider, diff --git a/api/services/knowledge_fs/initial_source_preview.py b/api/services/knowledge_fs/initial_source_preview.py index 53282822564..b4460e25a65 100644 --- a/api/services/knowledge_fs/initial_source_preview.py +++ b/api/services/knowledge_fs/initial_source_preview.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from collections.abc import Callable from typing import Any, cast from sqlalchemy import select @@ -14,6 +15,7 @@ from core.datasource.entities.datasource_entities import ( ) from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin +from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin from models.account import Account from models.credential_permission import CredentialType from models.oauth import DatasourceProvider @@ -22,23 +24,34 @@ from services.datasource_provider_service import DatasourceProviderService from services.knowledge_fs.product_dto import ( KnowledgeFSInitialSourcePreviewDocumentResponse, KnowledgeFSInitialSourcePreviewFileResponse, + KnowledgeFSInitialSourcePreviewPageResponse, KnowledgeFSInitialSourcePreviewPayload, KnowledgeFSInitialSourcePreviewResponse, + KnowledgeFSInitialWebsiteSourcePreviewPayload, ) _MAX_PREVIEW_ITEMS = 200 +class KnowledgeFSInitialSourcePreviewCanceledError(RuntimeError): + pass + + +def _raise_if_canceled(is_canceled: Callable[[], bool] | None) -> None: + if is_canceled is not None and is_canceled(): + raise KnowledgeFSInitialSourcePreviewCanceledError("Datasource preview was canceled") + + class KnowledgeFSInitialSourcePreviewService: def __init__(self, session_maker) -> None: self._session_maker = session_maker - def _require_visible_credential( + def require_visible_credential( self, *, tenant_id: str, account: Account, - payload: KnowledgeFSInitialSourcePreviewPayload, + payload: KnowledgeFSInitialSourcePreviewPayload | KnowledgeFSInitialWebsiteSourcePreviewPayload, ) -> None: query = select(DatasourceProvider).where( DatasourceProvider.tenant_id == tenant_id, @@ -63,14 +76,17 @@ class KnowledgeFSInitialSourcePreviewService: *, tenant_id: str, account: Account, - payload: KnowledgeFSInitialSourcePreviewPayload, + payload: KnowledgeFSInitialSourcePreviewPayload | KnowledgeFSInitialWebsiteSourcePreviewPayload, + is_canceled: Callable[[], bool] | None = None, ) -> KnowledgeFSInitialSourcePreviewResponse: - self._require_visible_credential(tenant_id=tenant_id, account=account, payload=payload) + _raise_if_canceled(is_canceled) + self.require_visible_credential(tenant_id=tenant_id, account=account, payload=payload) credentials = DatasourceProviderService().get_datasource_credentials( tenant_id=tenant_id, provider=payload.provider, plugin_id=payload.plugin_id, credential_id=payload.credential_id, + current_user=account, ) if not credentials: raise PermissionError("Datasource credential is unavailable") @@ -83,6 +99,30 @@ class KnowledgeFSInitialSourcePreviewService: ) runtime.runtime.credentials = credentials parameters = dict(payload.parameters) + if payload.kind == "website_crawl": + website_runtime = cast(WebsiteCrawlDatasourcePlugin, runtime) + pages_by_url: dict[str, KnowledgeFSInitialSourcePreviewPageResponse] = {} + for website_message in website_runtime.get_website_crawl( + user_id=account.id, + datasource_parameters=parameters, + provider_type=website_runtime.datasource_provider_type(), + ): + _raise_if_canceled(is_canceled) + for page in website_message.result.web_info_list or []: + pages_by_url[page.source_url] = KnowledgeFSInitialSourcePreviewPageResponse( + description=page.description or None, + source_url=page.source_url, + title=page.title or None, + ) + if len(pages_by_url) >= _MAX_PREVIEW_ITEMS: + return KnowledgeFSInitialSourcePreviewResponse( + kind=payload.kind, + pages=list(pages_by_url.values()), + ) + return KnowledgeFSInitialSourcePreviewResponse( + kind=payload.kind, + pages=list(pages_by_url.values()), + ) if payload.kind == "online_document": document_runtime = cast(OnlineDocumentDatasourcePlugin, runtime) documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = [] @@ -174,4 +214,4 @@ class KnowledgeFSInitialSourcePreviewService: ) -__all__ = ["KnowledgeFSInitialSourcePreviewService"] +__all__ = ["KnowledgeFSInitialSourcePreviewCanceledError", "KnowledgeFSInitialSourcePreviewService"] diff --git a/api/services/knowledge_fs/initial_source_preview_job.py b/api/services/knowledge_fs/initial_source_preview_job.py new file mode 100644 index 00000000000..55297d24e96 --- /dev/null +++ b/api/services/knowledge_fs/initial_source_preview_job.py @@ -0,0 +1,190 @@ +"""Asynchronous website datasource previews that do not hold Console HTTP workers.""" + +from __future__ import annotations + +import json +import uuid +from typing import Literal + +from celery import current_app as celery_app + +from extensions.ext_redis import redis_client +from models.account import Account +from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService +from services.knowledge_fs.product_dto import ( + KnowledgeFSInitialSourcePreviewJobCreateResponse, + KnowledgeFSInitialSourcePreviewJobResponse, + KnowledgeFSInitialSourcePreviewResponse, + KnowledgeFSInitialWebsiteSourcePreviewPayload, +) + +_JOB_TTL_SECONDS = 60 * 60 +_RELEASE_ACTIVE_JOB_SCRIPT = """ +local current = redis.call('GET', KEYS[1]) +if current == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" +_TRANSITION_STATUS_SCRIPT = """ +local current = redis.call('GET', KEYS[1]) +if not current then + return 0 +end +local decoded = cjson.decode(current) +for index = 3, #ARGV do + if decoded.status == ARGV[index] then + redis.call('SETEX', KEYS[1], ARGV[2], ARGV[1]) + return 1 + end +end +return 0 +""" + + +class KnowledgeFSInitialSourcePreviewJobNotFoundError(LookupError): + pass + + +class KnowledgeFSInitialSourcePreviewJobAlreadyRunningError(RuntimeError): + pass + + +def _job_key(*, tenant_id: str, account_id: str, job_id: str) -> str: + return f"knowledge_fs:initial_source_preview:{tenant_id}:{account_id}:{job_id}" + + +def _active_job_key(*, tenant_id: str, account_id: str) -> str: + return f"knowledge_fs:initial_source_preview:{tenant_id}:{account_id}:active" + + +class KnowledgeFSInitialSourcePreviewJobService: + def __init__(self, session_maker) -> None: + self._preview_service = KnowledgeFSInitialSourcePreviewService(session_maker) + + def start( + self, + *, + tenant_id: str, + account: Account, + payload: KnowledgeFSInitialWebsiteSourcePreviewPayload, + ) -> KnowledgeFSInitialSourcePreviewJobCreateResponse: + self._preview_service.require_visible_credential(tenant_id=tenant_id, account=account, payload=payload) + job_id = str(uuid.uuid4()) + active_job_key = _active_job_key(tenant_id=tenant_id, account_id=account.id) + if not redis_client.set(active_job_key, job_id, ex=_JOB_TTL_SECONDS, nx=True): + raise KnowledgeFSInitialSourcePreviewJobAlreadyRunningError( + "A datasource preview is already running for this account" + ) + try: + self.set_status( + tenant_id=tenant_id, + account_id=account.id, + job_id=job_id, + status="pending", + ) + from tasks.knowledge_fs_initial_source_preview_tasks import run_knowledge_fs_initial_source_preview + + run_knowledge_fs_initial_source_preview.apply_async( + kwargs={ + "account_id": account.id, + "job_id": job_id, + "payload": payload.model_dump(mode="json", by_alias=True), + "tenant_id": tenant_id, + }, + task_id=job_id, + ) + except Exception: + redis_client.delete(_job_key(tenant_id=tenant_id, account_id=account.id, job_id=job_id)) + self.release_active_job(tenant_id=tenant_id, account_id=account.id, job_id=job_id) + raise + return KnowledgeFSInitialSourcePreviewJobCreateResponse(job_id=job_id) + + @staticmethod + def get(*, tenant_id: str, account_id: str, job_id: str) -> KnowledgeFSInitialSourcePreviewJobResponse: + raw = redis_client.get(_job_key(tenant_id=tenant_id, account_id=account_id, job_id=job_id)) + if raw is None: + raise KnowledgeFSInitialSourcePreviewJobNotFoundError(job_id) + return KnowledgeFSInitialSourcePreviewJobResponse.model_validate_json(raw) + + @classmethod + def cancel(cls, *, tenant_id: str, account_id: str, job_id: str) -> KnowledgeFSInitialSourcePreviewJobResponse: + current = cls.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id) + if current.status in {"completed", "failed"}: + cls.release_active_job(tenant_id=tenant_id, account_id=account_id, job_id=job_id) + return current + + if current.status != "canceled": + transitioned = cls.transition_status( + tenant_id=tenant_id, + account_id=account_id, + job_id=job_id, + status="canceled", + allowed_from=("pending", "running"), + ) + if not transitioned: + current = cls.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id) + if current.status != "canceled": + return current + + # Reissue termination for an already-canceled job so a client can safely retry + # after a lost response or broker publication failure. + celery_app.control.revoke(job_id, terminate=True, signal="SIGTERM") + response = cls.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id) + cls.release_active_job(tenant_id=tenant_id, account_id=account_id, job_id=job_id) + return response + + @staticmethod + def release_active_job(*, tenant_id: str, account_id: str, job_id: str) -> bool: + released = redis_client.eval( + _RELEASE_ACTIVE_JOB_SCRIPT, + 1, + _active_job_key(tenant_id=tenant_id, account_id=account_id), + job_id, + ) + return bool(released) + + @staticmethod + def set_status( + *, + tenant_id: str, + account_id: str, + job_id: str, + status: Literal["pending", "running", "completed", "failed", "canceled"], + result: KnowledgeFSInitialSourcePreviewResponse | None = None, + ) -> None: + response = KnowledgeFSInitialSourcePreviewJobResponse(job_id=job_id, result=result, status=status) + redis_client.setex( + _job_key(tenant_id=tenant_id, account_id=account_id, job_id=job_id), + _JOB_TTL_SECONDS, + json.dumps(response.model_dump(mode="json", by_alias=True), separators=(",", ":")), + ) + + @staticmethod + def transition_status( + *, + tenant_id: str, + account_id: str, + job_id: str, + status: Literal["pending", "running", "completed", "failed", "canceled"], + allowed_from: tuple[Literal["pending", "running", "completed", "failed", "canceled"], ...], + result: KnowledgeFSInitialSourcePreviewResponse | None = None, + ) -> bool: + response = KnowledgeFSInitialSourcePreviewJobResponse(job_id=job_id, result=result, status=status) + serialized = json.dumps(response.model_dump(mode="json", by_alias=True), separators=(",", ":")) + transitioned = redis_client.eval( + _TRANSITION_STATUS_SCRIPT, + 1, + _job_key(tenant_id=tenant_id, account_id=account_id, job_id=job_id), + serialized, + _JOB_TTL_SECONDS, + *allowed_from, + ) + return bool(transitioned) + + +__all__ = [ + "KnowledgeFSInitialSourcePreviewJobAlreadyRunningError", + "KnowledgeFSInitialSourcePreviewJobNotFoundError", + "KnowledgeFSInitialSourcePreviewJobService", +] diff --git a/api/services/knowledge_fs/product_dto.py b/api/services/knowledge_fs/product_dto.py index 13938b39d52..194881c309c 100644 --- a/api/services/knowledge_fs/product_dto.py +++ b/api/services/knowledge_fs/product_dto.py @@ -142,6 +142,7 @@ class KnowledgeFSInitialDatasourceBindingPayload(BaseModel): plugin_id: str = Field(min_length=1, max_length=255, alias="pluginId") provider: str = Field(min_length=1, max_length=255) provider_display_name: str | None = Field(default=None, min_length=1, max_length=255, alias="providerDisplayName") + parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50) model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True) @@ -154,6 +155,7 @@ class KnowledgeFSInitialWebsiteSourcePayload(BaseModel): datasource: str = Field(default="crawl", min_length=1, max_length=255) credential_id: str | None = Field(default=None, min_length=1, max_length=255, alias="credentialId") provider_display_name: str | None = Field(default=None, min_length=1, max_length=255, alias="providerDisplayName") + parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50) root_url: str = Field(min_length=1, max_length=4_096) crawl_options: KnowledgeFSInitialWebsiteCrawlOptionsPayload selection: list[KnowledgeFSInitialWebsiteSelectionPayload] = Field(min_length=1, max_length=200) @@ -185,7 +187,16 @@ class KnowledgeFSInitialOnlineDriveSourcePayload(KnowledgeFSInitialDatasourceBin class KnowledgeFSInitialSourcePreviewPayload(KnowledgeFSInitialDatasourceBindingPayload): kind: Literal["online_document", "online_drive"] - parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50) + + +class KnowledgeFSInitialWebsiteSourcePreviewPayload(KnowledgeFSInitialDatasourceBindingPayload): + kind: Literal["website_crawl"] + + +class KnowledgeFSInitialSourcePreviewPageResponse(ResponseModel): + description: str | None = None + source_url: str = Field(validation_alias=AliasChoices("source_url", "sourceUrl")) + title: str | None = None class KnowledgeFSInitialSourcePreviewDocumentResponse(ResponseModel): @@ -213,11 +224,23 @@ class KnowledgeFSInitialSourcePreviewFileResponse(ResponseModel): class KnowledgeFSInitialSourcePreviewResponse(ResponseModel): documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = Field(default_factory=list) files: list[KnowledgeFSInitialSourcePreviewFileResponse] = Field(default_factory=list) - kind: Literal["online_document", "online_drive"] + kind: Literal["online_document", "online_drive", "website_crawl"] next_page_parameters: dict[str, JsonValue] | None = Field( default=None, validation_alias=AliasChoices("next_page_parameters", "nextPageParameters"), ) + pages: list[KnowledgeFSInitialSourcePreviewPageResponse] = Field(default_factory=list) + + +class KnowledgeFSInitialSourcePreviewJobCreateResponse(ResponseModel): + job_id: str = Field(validation_alias=AliasChoices("job_id", "jobId")) + status: Literal["pending"] = "pending" + + +class KnowledgeFSInitialSourcePreviewJobResponse(ResponseModel): + job_id: str = Field(validation_alias=AliasChoices("job_id", "jobId")) + result: KnowledgeFSInitialSourcePreviewResponse | None = None + status: Literal["pending", "running", "completed", "failed", "canceled"] KnowledgeFSInitialSourcePayload = Annotated[ @@ -2993,11 +3016,15 @@ __all__ = [ "KnowledgeFSInitialSourcePayload", "KnowledgeFSInitialSourcePreviewDocumentResponse", "KnowledgeFSInitialSourcePreviewFileResponse", + "KnowledgeFSInitialSourcePreviewJobCreateResponse", + "KnowledgeFSInitialSourcePreviewJobResponse", + "KnowledgeFSInitialSourcePreviewPageResponse", "KnowledgeFSInitialSourcePreviewPayload", "KnowledgeFSInitialSourcePreviewResponse", "KnowledgeFSInitialWebsiteCrawlOptionsPayload", "KnowledgeFSInitialWebsiteSelectionPayload", "KnowledgeFSInitialWebsiteSourcePayload", + "KnowledgeFSInitialWebsiteSourcePreviewPayload", "KnowledgeFSJWKResponse", "KnowledgeFSJWKSResponse", "KnowledgeFSLogicalDocumentDeletePayload", diff --git a/api/tasks/knowledge_fs_initial_source_preview_tasks.py b/api/tasks/knowledge_fs_initial_source_preview_tasks.py new file mode 100644 index 00000000000..0cf6bfd6352 --- /dev/null +++ b/api/tasks/knowledge_fs_initial_source_preview_tasks.py @@ -0,0 +1,119 @@ +"""Runs website datasource preview jobs outside Console request workers.""" + +from __future__ import annotations + +import logging + +from celery import shared_task +from sqlalchemy import select + +from core.db.session_factory import session_factory +from models.account import Account, AccountStatus, TenantAccountJoin +from services.knowledge_fs.initial_source_preview import ( + KnowledgeFSInitialSourcePreviewCanceledError, + KnowledgeFSInitialSourcePreviewService, +) +from services.knowledge_fs.initial_source_preview_job import ( + KnowledgeFSInitialSourcePreviewJobNotFoundError, + KnowledgeFSInitialSourcePreviewJobService, +) +from services.knowledge_fs.product_dto import KnowledgeFSInitialWebsiteSourcePreviewPayload + +logger = logging.getLogger(__name__) + + +@shared_task(queue="dataset") +def run_knowledge_fs_initial_source_preview( + *, + tenant_id: str, + account_id: str, + job_id: str, + payload: dict[str, object], +) -> None: + job_service = KnowledgeFSInitialSourcePreviewJobService(session_factory.get_session_maker()) + try: + if not job_service.transition_status( + tenant_id=tenant_id, + account_id=account_id, + job_id=job_id, + status="running", + allowed_from=("pending",), + ): + return + with session_factory.create_session() as session: + account = session.get(Account, account_id) + if account is None or account.status != AccountStatus.ACTIVE: + raise LookupError("Datasource preview account is unavailable") + membership = session.scalar( + select(TenantAccountJoin.id).where( + TenantAccountJoin.tenant_id == tenant_id, + TenantAccountJoin.account_id == account_id, + ) + ) + if membership is None: + raise PermissionError("Datasource preview account is not a tenant member") + account.set_tenant_id_with_session(tenant_id, session=session) + if account.current_tenant_id != tenant_id: + raise PermissionError("Datasource preview account is not a tenant member") + session.expunge(account) + result = KnowledgeFSInitialSourcePreviewService(session_factory.get_session_maker()).preview( + tenant_id=tenant_id, + account=account, + payload=KnowledgeFSInitialWebsiteSourcePreviewPayload.model_validate(payload), + is_canceled=lambda: _preview_was_canceled( + job_service=job_service, + tenant_id=tenant_id, + account_id=account_id, + job_id=job_id, + ), + ) + job_service.transition_status( + tenant_id=tenant_id, + account_id=account_id, + job_id=job_id, + status="completed", + allowed_from=("running",), + result=result, + ) + except KnowledgeFSInitialSourcePreviewCanceledError: + logger.info( + "KnowledgeFS initial source preview canceled", + extra={"account_id": account_id, "job_id": job_id, "tenant_id": tenant_id}, + ) + except Exception: + logger.exception( + "KnowledgeFS initial source preview failed", + extra={"account_id": account_id, "job_id": job_id, "tenant_id": tenant_id}, + ) + job_service.transition_status( + tenant_id=tenant_id, + account_id=account_id, + job_id=job_id, + status="failed", + allowed_from=("pending", "running"), + ) + finally: + job_service.release_active_job( + tenant_id=tenant_id, + account_id=account_id, + job_id=job_id, + ) + + +def _preview_was_canceled( + *, + job_service: KnowledgeFSInitialSourcePreviewJobService, + tenant_id: str, + account_id: str, + job_id: str, +) -> bool: + try: + return ( + job_service.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id).status + == "canceled" + ) + except KnowledgeFSInitialSourcePreviewJobNotFoundError: + return True + + +__all__ = ["run_knowledge_fs_initial_source_preview"] diff --git a/api/tasks/knowledge_fs_initial_source_tasks.py b/api/tasks/knowledge_fs_initial_source_tasks.py index 3cd029696d2..fa65f0d19c0 100644 --- a/api/tasks/knowledge_fs_initial_source_tasks.py +++ b/api/tasks/knowledge_fs_initial_source_tasks.py @@ -230,6 +230,9 @@ def _source_payload( "providerKind": binding.provider_kind, "providerName": payload.provider_display_name or payload.provider, } + if "parameters" in payload.model_fields_set: + metadata["datasourceParameterMode"] = "exact" + metadata["parameters"] = dict(payload.parameters) if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload): metadata["crawlOptions"] = { "includeSubpages": payload.crawl_options.include_subpages, diff --git a/api/tests/unit_tests/extensions/test_knowledge_fs_celery.py b/api/tests/unit_tests/extensions/test_knowledge_fs_celery.py index 8119308a9a4..89c55d5610d 100644 --- a/api/tests/unit_tests/extensions/test_knowledge_fs_celery.py +++ b/api/tests/unit_tests/extensions/test_knowledge_fs_celery.py @@ -52,8 +52,22 @@ def test_celery_registers_initial_source_task_when_knowledge_fs_lifecycle_is_rea celery_app = init_app(DifyApp(__name__)) assert "tasks.knowledge_fs_initial_source_tasks" in celery_app.conf["imports"] + assert "tasks.knowledge_fs_initial_source_preview_tasks" in celery_app.conf["imports"] assert "tasks.knowledge_fs_lifecycle_tasks" in celery_app.conf["imports"] assert celery_app.conf["beat_schedule"]["knowledge_fs_staged_upload_cleanup"] == { "task": "tasks.knowledge_fs_lifecycle_tasks.cleanup_knowledge_fs_staged_uploads", "schedule": timedelta(seconds=2), } + + with ( + patch("extensions.ext_celery.dify_config", config), + patch( + "services.knowledge_fs.lifecycle_readiness.get_configured_knowledge_fs_lifecycle_worker_readiness", + return_value=SimpleNamespace(ready=False), + ), + ): + preview_only_app = init_app(DifyApp(f"{__name__}.preview_only")) + + assert "tasks.knowledge_fs_initial_source_preview_tasks" in preview_only_app.conf["imports"] + assert "tasks.knowledge_fs_initial_source_tasks" not in preview_only_app.conf["imports"] + assert "tasks.knowledge_fs_lifecycle_tasks" not in preview_only_app.conf["imports"] diff --git a/api/tests/unit_tests/services/test_datasource_provider_service.py b/api/tests/unit_tests/services/test_datasource_provider_service.py index 22514839d9d..7e6bebe789e 100644 --- a/api/tests/unit_tests/services/test_datasource_provider_service.py +++ b/api/tests/unit_tests/services/test_datasource_provider_service.py @@ -345,7 +345,7 @@ class TestDatasourceProviderService: def test_should_fetch_by_credential_id_when_provided(self, service, sqlite_session, mock_user): """When credential_id is passed, the credential_id filter path (line 113) is taken.""" - p = make_provider(credential_id="cred-id", provider="other-provider", plugin_id="other-plugin") + p = make_provider(credential_id="cred-id") persist(sqlite_session, p) with ( patch("services.datasource_provider_service.get_current_user", return_value=mock_user), @@ -354,13 +354,6 @@ class TestDatasourceProviderService: result = service.get_datasource_credentials("t1", "prov", "org/plug", credential_id="cred-id") assert result == {"k": "v"} - statement = mock_db_session.scalar.call_args.args[0] - sql = str(statement.compile(compile_kwargs={"literal_binds": True})) - assert "datasource_providers.tenant_id = 't1'" in sql - assert "datasource_providers.id = 'cred-id'" in sql - assert "datasource_providers.provider = 'prov'" in sql - assert "datasource_providers.plugin_id = 'org/plug'" in sql - # ----------------------------------------------------------------------- # get_all_datasource_credentials_by_provider (lines 176-228) # ----------------------------------------------------------------------- diff --git a/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py b/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py index de25f1bf310..56403b0c977 100644 --- a/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py +++ b/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py @@ -11,10 +11,19 @@ from core.datasource.entities.datasource_entities import ( OnlineDriveBrowseFilesResponse, OnlineDriveFile, OnlineDriveFileBucket, + WebsiteCrawlMessage, + WebSiteInfo, + WebSiteInfoDetail, ) from models.account import Account -from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService -from services.knowledge_fs.product_dto import KnowledgeFSInitialSourcePreviewPayload +from services.knowledge_fs.initial_source_preview import ( + KnowledgeFSInitialSourcePreviewCanceledError, + KnowledgeFSInitialSourcePreviewService, +) +from services.knowledge_fs.product_dto import ( + KnowledgeFSInitialSourcePreviewPayload, + KnowledgeFSInitialWebsiteSourcePreviewPayload, +) _CREDENTIAL = object() @@ -27,8 +36,13 @@ def _service(credential=_CREDENTIAL) -> tuple[KnowledgeFSInitialSourcePreviewSer return KnowledgeFSInitialSourcePreviewService(MagicMock(return_value=context)), session -def _payload(kind: str) -> KnowledgeFSInitialSourcePreviewPayload: - return KnowledgeFSInitialSourcePreviewPayload.model_validate( +def _payload(kind: str) -> KnowledgeFSInitialSourcePreviewPayload | KnowledgeFSInitialWebsiteSourcePreviewPayload: + payload_type = ( + KnowledgeFSInitialWebsiteSourcePreviewPayload + if kind == "website_crawl" + else KnowledgeFSInitialSourcePreviewPayload + ) + return payload_type.model_validate( { "credentialId": "credential-1", "datasource": "pages" if kind == "online_document" else "drive", @@ -136,14 +150,95 @@ def test_preview_browses_online_drive_and_preserves_pagination() -> None: assert request.prefix == "folder-1" +def test_preview_crawls_a_website_with_exact_declared_parameters() -> None: + service, _session = _service() + runtime = MagicMock() + runtime.datasource_provider_type.return_value = "website_crawl" + runtime.get_website_crawl.return_value = [ + WebsiteCrawlMessage( + result=WebSiteInfo( + status="completed", + web_info_list=[ + WebSiteInfoDetail( + content="# Result", + description="Search result", + source_url="https://example.com/result", + title="Result", + ) + ], + ) + ) + ] + payload = _payload("website_crawl") + payload.parameters = {"query": "agentic RAG", "search_depth": "advanced"} + account = cast(Account, SimpleNamespace(id="account-1")) + with ( + patch( + "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials", + return_value={"api_key": "secret"}, + ) as get_credentials, + patch( + "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime", + return_value=runtime, + ), + ): + response = service.preview( + tenant_id="tenant-1", + account=account, + payload=payload, + ) + + assert response.kind == "website_crawl" + assert response.pages[0].source_url == "https://example.com/result" + get_credentials.assert_called_once_with( + tenant_id="tenant-1", + provider="provider", + plugin_id="langgenius/provider", + credential_id="credential-1", + current_user=account, + ) + runtime.get_website_crawl.assert_called_once_with( + user_id="account-1", + datasource_parameters={"query": "agentic RAG", "search_depth": "advanced"}, + provider_type="website_crawl", + ) + + +def test_preview_stops_consuming_website_results_after_cancellation() -> None: + service, _session = _service() + runtime = MagicMock() + runtime.datasource_provider_type.return_value = "website_crawl" + runtime.get_website_crawl.return_value = [ + WebsiteCrawlMessage(result=WebSiteInfo(status="running", web_info_list=[])) + ] + is_canceled = MagicMock(side_effect=[False, True]) + with ( + patch( + "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials", + return_value={"api_key": "secret"}, + ), + patch( + "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime", + return_value=runtime, + ), + pytest.raises(KnowledgeFSInitialSourcePreviewCanceledError), + ): + service.preview( + tenant_id="tenant-1", + account=cast(Account, SimpleNamespace(id="account-1")), + payload=_payload("website_crawl"), + is_canceled=is_canceled, + ) + + assert is_canceled.call_count == 2 + + def test_preview_exposes_an_empty_drive_bucket_as_a_browsable_container() -> None: service, _session = _service() runtime = MagicMock() runtime.datasource_provider_type.return_value = "online_drive" runtime.online_drive_browse_files.return_value = [ - OnlineDriveBrowseFilesResponse( - result=[OnlineDriveFileBucket(bucket="manuals", files=[], is_truncated=False)] - ) + OnlineDriveBrowseFilesResponse(result=[OnlineDriveFileBucket(bucket="manuals", files=[], is_truncated=False)]) ] with ( patch( diff --git a/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview_job.py b/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview_job.py new file mode 100644 index 00000000000..c235323ab55 --- /dev/null +++ b/api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview_job.py @@ -0,0 +1,203 @@ +import json +from types import SimpleNamespace +from typing import cast +from unittest.mock import MagicMock, patch + +import pytest + +from models.account import Account +from services.knowledge_fs.initial_source_preview_job import ( + KnowledgeFSInitialSourcePreviewJobAlreadyRunningError, + KnowledgeFSInitialSourcePreviewJobNotFoundError, + KnowledgeFSInitialSourcePreviewJobService, +) +from services.knowledge_fs.product_dto import KnowledgeFSInitialWebsiteSourcePreviewPayload + + +def _payload() -> KnowledgeFSInitialWebsiteSourcePreviewPayload: + return KnowledgeFSInitialWebsiteSourcePreviewPayload.model_validate( + { + "credentialId": "credential-1", + "datasource": "crawl", + "kind": "website_crawl", + "parameters": {"url": "https://docs.dify.ai"}, + "pluginId": "langgenius/firecrawl_datasource", + "provider": "firecrawl", + } + ) + + +def test_start_validates_binding_and_enqueues_preview_without_running_it_inline() -> None: + service = KnowledgeFSInitialSourcePreviewJobService(MagicMock()) + account = cast(Account, SimpleNamespace(id="account-1")) + + with ( + patch( + "services.knowledge_fs.initial_source_preview_job.KnowledgeFSInitialSourcePreviewService.require_visible_credential" + ) as require_credential, + patch("services.knowledge_fs.initial_source_preview_job.redis_client.set", return_value=True) as acquire, + patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex") as setex, + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.run_knowledge_fs_initial_source_preview.apply_async" + ) as apply_async, + ): + response = service.start(tenant_id="tenant-1", account=account, payload=_payload()) + + require_credential.assert_called_once() + acquire.assert_called_once_with( + "knowledge_fs:initial_source_preview:tenant-1:account-1:active", + response.job_id, + ex=3600, + nx=True, + ) + setex.assert_called_once() + assert '"status":"pending"' in setex.call_args.args[2] + apply_async.assert_called_once_with( + kwargs={ + "account_id": "account-1", + "job_id": response.job_id, + "payload": { + "credentialId": "credential-1", + "datasource": "crawl", + "kind": "website_crawl", + "parameters": {"url": "https://docs.dify.ai"}, + "pluginId": "langgenius/firecrawl_datasource", + "provider": "firecrawl", + "providerDisplayName": None, + }, + "tenant_id": "tenant-1", + }, + task_id=response.job_id, + ) + + +def test_start_rejects_a_second_active_preview_for_the_same_account() -> None: + service = KnowledgeFSInitialSourcePreviewJobService(MagicMock()) + account = cast(Account, SimpleNamespace(id="account-1")) + + with ( + patch( + "services.knowledge_fs.initial_source_preview_job.KnowledgeFSInitialSourcePreviewService.require_visible_credential" + ), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.set", return_value=False), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex") as setex, + pytest.raises(KnowledgeFSInitialSourcePreviewJobAlreadyRunningError), + ): + service.start(tenant_id="tenant-1", account=account, payload=_payload()) + + setex.assert_not_called() + + +def test_start_releases_the_active_preview_slot_when_enqueue_fails() -> None: + service = KnowledgeFSInitialSourcePreviewJobService(MagicMock()) + account = cast(Account, SimpleNamespace(id="account-1")) + + with ( + patch( + "services.knowledge_fs.initial_source_preview_job.KnowledgeFSInitialSourcePreviewService.require_visible_credential" + ), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.set", return_value=True), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex"), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.delete") as delete, + patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval", return_value=1) as redis_eval, + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.run_knowledge_fs_initial_source_preview.apply_async", + side_effect=RuntimeError("broker unavailable"), + ), + pytest.raises(RuntimeError, match="broker unavailable"), + ): + service.start(tenant_id="tenant-1", account=account, payload=_payload()) + + job_id = redis_eval.call_args.args[3] + delete.assert_called_once_with( + f"knowledge_fs:initial_source_preview:tenant-1:account-1:{job_id}" + ) + assert redis_eval.call_args.args[2:] == ( + "knowledge_fs:initial_source_preview:tenant-1:account-1:active", + job_id, + ) + + +def test_cancel_is_scoped_to_the_requesting_tenant_and_account() -> None: + cache: dict[str, str] = {} + + def setex(key: str, _ttl: int, value: str) -> None: + cache[key] = value + + def transition( + _script: str, + _key_count: int, + key: str, + value: str, + _ttl: int | None = None, + *allowed_from: str, + ) -> int: + if _ttl is None: + return 0 + current = cache.get(key) + if current is None or json.loads(current)["status"] not in allowed_from: + return 0 + cache[key] = value + return 1 + + with ( + patch("services.knowledge_fs.initial_source_preview_job.redis_client.get", side_effect=cache.get), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex", side_effect=setex), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval", side_effect=transition), + patch("services.knowledge_fs.initial_source_preview_job.celery_app.control.revoke") as revoke, + ): + KnowledgeFSInitialSourcePreviewJobService.set_status( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="running", + ) + response = KnowledgeFSInitialSourcePreviewJobService.cancel( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + ) + with pytest.raises(KnowledgeFSInitialSourcePreviewJobNotFoundError): + KnowledgeFSInitialSourcePreviewJobService.get( + tenant_id="tenant-1", + account_id="account-2", + job_id="job-1", + ) + + assert response.status == "canceled" + revoke.assert_called_once_with("job-1", terminate=True, signal="SIGTERM") + + +def test_cancel_reissues_termination_for_an_already_canceled_job() -> None: + canceled = json.dumps({"jobId": "job-1", "result": None, "status": "canceled"}) + + with ( + patch("services.knowledge_fs.initial_source_preview_job.redis_client.get", return_value=canceled), + patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval") as redis_eval, + patch("services.knowledge_fs.initial_source_preview_job.celery_app.control.revoke") as revoke, + ): + response = KnowledgeFSInitialSourcePreviewJobService.cancel( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + ) + + assert response.status == "canceled" + redis_eval.assert_called_once() + revoke.assert_called_once_with("job-1", terminate=True, signal="SIGTERM") + + +def test_terminal_canceled_status_cannot_be_overwritten() -> None: + with patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval", return_value=0) as eval_status: + transitioned = KnowledgeFSInitialSourcePreviewJobService.transition_status( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="completed", + allowed_from=("running",), + result=None, + ) + + assert transitioned is False + assert '"status":"completed"' in eval_status.call_args.args[3] + assert eval_status.call_args.args[5:] == ("running",) diff --git a/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_preview_tasks.py b/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_preview_tasks.py new file mode 100644 index 00000000000..78139098c31 --- /dev/null +++ b/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_preview_tasks.py @@ -0,0 +1,224 @@ +from types import SimpleNamespace +from unittest.mock import MagicMock, call, patch + +from models.account import AccountStatus +from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewCanceledError +from services.knowledge_fs.product_dto import KnowledgeFSInitialSourcePreviewResponse +from tasks.knowledge_fs_initial_source_preview_tasks import run_knowledge_fs_initial_source_preview + + +def _payload() -> dict[str, object]: + return { + "credentialId": "credential-1", + "datasource": "crawl", + "kind": "website_crawl", + "parameters": {"url": "https://docs.dify.ai"}, + "pluginId": "langgenius/firecrawl_datasource", + "provider": "firecrawl", + } + + +def test_preview_task_uses_the_dataset_queue() -> None: + assert run_knowledge_fs_initial_source_preview.queue == "dataset" + + +def test_preview_task_persists_running_and_completed_states() -> None: + account = SimpleNamespace( + current_tenant_id="tenant-1", + id="account-1", + set_tenant_id_with_session=MagicMock(), + status=AccountStatus.ACTIVE, + ) + session = MagicMock() + session.get.return_value = account + session.scalar.return_value = "membership-1" + session_context = MagicMock() + session_context.__enter__.return_value = session + job_service = MagicMock() + job_service.transition_status.side_effect = [True, True] + preview_service = MagicMock() + result = KnowledgeFSInitialSourcePreviewResponse(kind="website_crawl") + preview_service.preview.return_value = result + + with ( + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService", + return_value=job_service, + ), + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService", + return_value=preview_service, + ), + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.session_factory.create_session", + return_value=session_context, + ), + patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"), + ): + run_knowledge_fs_initial_source_preview.run( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + payload=_payload(), + ) + + assert job_service.transition_status.call_args_list == [ + call( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="running", + allowed_from=("pending",), + ), + call( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="completed", + allowed_from=("running",), + result=result, + ), + ] + job_service.release_active_job.assert_called_once_with( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + ) + account.set_tenant_id_with_session.assert_called_once_with("tenant-1", session=session) + session.expunge.assert_called_once_with(account) + preview_service.preview.assert_called_once() + + +def test_preview_task_does_not_start_a_canceled_job() -> None: + job_service = MagicMock() + job_service.transition_status.return_value = False + + with ( + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService", + return_value=job_service, + ), + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService" + ) as preview_service, + patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"), + ): + run_knowledge_fs_initial_source_preview.run( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + payload=_payload(), + ) + + job_service.transition_status.assert_called_once_with( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="running", + allowed_from=("pending",), + ) + preview_service.assert_not_called() + job_service.release_active_job.assert_called_once_with( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + ) + + +def test_preview_task_keeps_a_cooperatively_canceled_job_terminal() -> None: + account = SimpleNamespace( + current_tenant_id="tenant-1", + id="account-1", + set_tenant_id_with_session=MagicMock(), + status=AccountStatus.ACTIVE, + ) + session = MagicMock() + session.get.return_value = account + session.scalar.return_value = "membership-1" + session_context = MagicMock() + session_context.__enter__.return_value = session + job_service = MagicMock() + job_service.transition_status.return_value = True + job_service.get.return_value = SimpleNamespace(status="canceled") + preview_service = MagicMock() + + def preview(**kwargs): + assert kwargs["is_canceled"]() is True + raise KnowledgeFSInitialSourcePreviewCanceledError("canceled") + + preview_service.preview.side_effect = preview + with ( + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService", + return_value=job_service, + ), + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService", + return_value=preview_service, + ), + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.session_factory.create_session", + return_value=session_context, + ), + patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"), + ): + run_knowledge_fs_initial_source_preview.run( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + payload=_payload(), + ) + + job_service.transition_status.assert_called_once_with( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="running", + allowed_from=("pending",), + ) + + +def test_preview_task_rejects_an_account_removed_from_the_tenant() -> None: + account = SimpleNamespace( + current_tenant_id=None, + id="account-1", + set_tenant_id_with_session=MagicMock(), + status=AccountStatus.ACTIVE, + ) + session = MagicMock() + session.get.return_value = account + session.scalar.return_value = None + session_context = MagicMock() + session_context.__enter__.return_value = session + job_service = MagicMock() + job_service.transition_status.side_effect = [True, True] + + with ( + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService", + return_value=job_service, + ), + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService" + ) as preview_service, + patch( + "tasks.knowledge_fs_initial_source_preview_tasks.session_factory.create_session", + return_value=session_context, + ), + patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"), + ): + run_knowledge_fs_initial_source_preview.run( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + payload=_payload(), + ) + + preview_service.assert_not_called() + assert job_service.transition_status.call_args_list[-1] == call( + tenant_id="tenant-1", + account_id="account-1", + job_id="job-1", + status="failed", + allowed_from=("pending", "running"), + ) diff --git a/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py b/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py index ba764aee3a0..2bd8f59f690 100644 --- a/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py +++ b/api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py @@ -29,6 +29,7 @@ def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePaylo "name": "Dify docs", "provider": "firecrawl", "providerDisplayName": "Firecrawl", + "parameters": {"url": "https://docs.dify.ai", "limit": 25}, "root_url": "https://docs.dify.ai", "crawl_options": {"include_subpages": True, "limit": 25}, "selection": [ @@ -40,6 +41,21 @@ def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePaylo ) +def _legacy_payload() -> KnowledgeFSInitialWebsiteSourcePayload: + return KnowledgeFSInitialWebsiteSourcePayload.model_validate( + { + "kind": "website_crawl", + "name": "Dify docs", + "provider": "firecrawl", + "providerDisplayName": "Firecrawl", + "root_url": "https://docs.dify.ai", + "crawl_options": {"include_subpages": False, "limit": 25}, + "selection": [{"source_url": "https://docs.dify.ai/a", "title": "A"}], + "sync_policy": "manual", + } + ) + + def _document_payload() -> KnowledgeFSInitialOnlineDocumentSourcePayload: return KnowledgeFSInitialOnlineDocumentSourcePayload.model_validate( { @@ -205,7 +221,12 @@ def test_initial_website_source_import_recrawls_exact_selection_and_configures_d source_payload = facade.create_source.call_args.kwargs["payload"] assert source_payload.status == "disabled" assert source_payload.connection_id == "connection-1" + assert source_payload.metadata["datasourceParameterMode"] == "exact" assert source_payload.metadata["preview"] is True + assert source_payload.metadata["parameters"] == { + "limit": 25, + "url": "https://docs.dify.ai", + } import_payload = facade.import_selected_source_crawl.call_args.kwargs["payload"] assert import_payload.source_urls == [ "https://docs.dify.ai/a", @@ -225,6 +246,21 @@ def test_initial_website_source_import_recrawls_exact_selection_and_configures_d assert sync_payload.expected_source_version == 4 +def test_initial_website_source_import_preserves_legacy_crawl_option_projection() -> None: + facade = _facade() + + assert _start(facade, _legacy_payload()) == "workflow-1" + + source_payload = facade.create_source.call_args.kwargs["payload"] + assert "datasourceParameterMode" not in source_payload.metadata + assert "parameters" not in source_payload.metadata + assert source_payload.metadata["crawlOptions"] == { + "includeSubpages": False, + "limit": 25, + } + assert source_payload.uri == "https://docs.dify.ai" + + @pytest.mark.parametrize( ( "payload", diff --git a/knowledge-fs/apps/api/src/dify-datasource-invocation-client.test.ts b/knowledge-fs/apps/api/src/dify-datasource-invocation-client.test.ts index 307f8c82acc..a32593ff569 100644 --- a/knowledge-fs/apps/api/src/dify-datasource-invocation-client.test.ts +++ b/knowledge-fs/apps/api/src/dify-datasource-invocation-client.test.ts @@ -3,6 +3,7 @@ import type { DifyDatasourceRuntimeClient } from "@knowledge/dify-datasource-run import { describe, expect, it, vi } from "vitest"; import { createDifyDatasourceInvocationClient } from "./dify-datasource-invocation-client"; +import { createApiOnlineDriveConnector } from "./online-drive-options"; const SOURCE: Source = { createdAt: "2026-07-03T00:00:00.000Z", @@ -61,7 +62,7 @@ describe("createDifyDatasourceInvocationClient", () => { expect(JSON.stringify(getOnlineDocumentPages.mock.calls)).not.toContain("credentials"); }); - it("maps product crawl options to the Firecrawl datasource parameters", async () => { + it("maps legacy product crawl options to the Firecrawl datasource parameters", async () => { const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } })); const adapter = createDifyDatasourceInvocationClient({ client: difyClient({ getWebsiteCrawl }), @@ -103,6 +104,153 @@ describe("createDifyDatasourceInvocationClient", () => { }); }); + it("preserves exact datasource parameters for declaration-driven website sources", async () => { + const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } })); + const adapter = createDifyDatasourceInvocationClient({ + client: difyClient({ getWebsiteCrawl }), + }); + const source: Source = { + ...SOURCE, + metadata: { + credentialId: "dify-credential-1", + crawlOptions: { includeSubpages: false, limit: 1 }, + datasource: "search_extract", + datasourceParameterMode: "exact", + parameters: { query: "dify knowledge", search_depth: "advanced" }, + pluginId: "langgenius/tavily_datasource", + provider: "tavily", + }, + type: "web", + uri: "datasource://tavily", + }; + + await collect( + adapter.dispatch({ + operation: "get_website_crawl", + source, + tenantId: "tenant-1", + }), + ); + + expect(getWebsiteCrawl).toHaveBeenCalledWith({ + credentialId: "dify-credential-1", + datasource: "search_extract", + datasourceParameters: { query: "dify knowledge", search_depth: "advanced" }, + pluginId: "langgenius/tavily_datasource", + provider: "tavily", + tenantId: "tenant-1", + }); + }); + + it("preserves an explicitly empty declaration-driven parameter set", async () => { + const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } })); + const adapter = createDifyDatasourceInvocationClient({ + client: difyClient({ getWebsiteCrawl }), + }); + const source: Source = { + ...SOURCE, + metadata: { + credentialId: "dify-credential-1", + crawlOptions: { includeSubpages: true, limit: 200 }, + datasource: "optional_search", + datasourceParameterMode: "exact", + parameters: {}, + pluginId: "langgenius/optional_search_datasource", + provider: "optional_search", + }, + type: "web", + uri: "datasource://optional-search", + }; + + await collect( + adapter.dispatch({ + operation: "get_website_crawl", + source, + tenantId: "tenant-1", + }), + ); + + expect(getWebsiteCrawl).toHaveBeenCalledWith( + expect.objectContaining({ datasourceParameters: {} }), + ); + }); + + it("maps legacy Jina crawl options to crawl_sub_pages", async () => { + const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } })); + const adapter = createDifyDatasourceInvocationClient({ + client: difyClient({ getWebsiteCrawl }), + }); + const source: Source = { + ...SOURCE, + metadata: { + credentialId: "jina-credential-1", + crawlOptions: { includeSubpages: true, limit: 10 }, + datasource: "jina_reader", + parameters: {}, + pluginId: "langgenius/jina_datasource", + provider: "jinareader", + }, + type: "web", + uri: "https://example.com", + }; + + await collect( + adapter.dispatch({ + operation: "get_website_crawl", + source, + tenantId: "tenant-1", + }), + ); + + expect(getWebsiteCrawl).toHaveBeenCalledWith( + expect.objectContaining({ + datasourceParameters: { + crawl_sub_pages: true, + limit: 10, + url: "https://example.com", + }, + }), + ); + }); + + it("uses declaration-driven online drive parameters as the browse root", async () => { + const browseOnlineDrive = vi.fn(() => chunks({ result: [] })); + const adapter = createDifyDatasourceInvocationClient({ + client: difyClient({ browseOnlineDrive }), + }); + const source: Source = { + ...SOURCE, + metadata: { + ...SOURCE.metadata, + datasource: "shared_drive", + parameters: { + bucket: "manuals", + max_keys: 50, + next_page_parameters: { cursor: "saved" }, + prefix: "products/", + }, + providerKind: "online-drive", + }, + uri: "gdrive://shared-drive", + }; + + const connector = createApiOnlineDriveConnector({ client: adapter }); + + await connector.browse({ source, tenantId: "tenant-1" }); + + expect(browseOnlineDrive).toHaveBeenCalledWith({ + bucket: "manuals", + credentialId: "dify-credential-1", + datasource: "shared_drive", + maxKeys: 50, + nextPageParameters: { cursor: "saved" }, + pluginId: "langgenius/notion_datasource", + prefix: "products/", + provider: "notion_datasource", + tenantId: "tenant-1", + }); + }); + it("rejects inline credentials in integrated mode", async () => { const adapter = createDifyDatasourceInvocationClient({ client: difyClient() }); const source = { diff --git a/knowledge-fs/apps/api/src/dify-datasource-invocation-client.ts b/knowledge-fs/apps/api/src/dify-datasource-invocation-client.ts index 2de37e17e41..345b59c712c 100644 --- a/knowledge-fs/apps/api/src/dify-datasource-invocation-client.ts +++ b/knowledge-fs/apps/api/src/dify-datasource-invocation-client.ts @@ -9,6 +9,8 @@ import type { ApiDatasourceInvocationInput, } from "./datasource-invocation-client"; +const DEFAULT_ONLINE_DRIVE_MAX_KEYS = 20; + interface DifyDatasourceSourceConfig { readonly credentialId: string; readonly datasource: string; @@ -39,7 +41,7 @@ export function createDifyDatasourceInvocationClient(input: { yield* input.client.getWebsiteCrawl({ ...common, datasourceParameters: withCrawlOptions( - config.parameters, + config, invocation.source, invocation.source.uri, ), @@ -61,17 +63,26 @@ export function createDifyDatasourceInvocationClient(input: { page: invocation.page, }); return; - case "online_drive_browse_files": + case "online_drive_browse_files": { + const sourceBucket = optionalString(config.parameters.bucket); + const sourcePrefix = optionalString(config.parameters.prefix); + const sourceMaxKeys = positiveSafeInteger(config.parameters.max_keys); + const sourceNextPageParameters = plainObject(config.parameters.next_page_parameters); yield* input.client.browseOnlineDrive({ ...common, - ...(invocation.bucket === undefined ? {} : { bucket: invocation.bucket }), - ...(invocation.continuationToken === undefined + ...(invocation.bucket === undefined && sourceBucket === undefined ? {} + : { bucket: invocation.bucket ?? sourceBucket }), + ...(invocation.continuationToken === undefined + ? Object.keys(sourceNextPageParameters).length + ? { nextPageParameters: sourceNextPageParameters } + : {} : { nextPageParameters: decodeNextPageParameters(invocation.continuationToken) }), - ...(invocation.maxKeys === undefined ? {} : { maxKeys: invocation.maxKeys }), - prefix: invocation.prefix ?? "", + maxKeys: invocation.maxKeys ?? sourceMaxKeys ?? DEFAULT_ONLINE_DRIVE_MAX_KEYS, + prefix: invocation.prefix || sourcePrefix || "", }); return; + } case "online_drive_download_file": yield* input.client.downloadOnlineDriveFile({ ...common, file: invocation.file }); return; @@ -149,18 +160,37 @@ function withCrawlUrl(parameters: Record, uri: string): Record< : { ...parameters, url: uri }; } +function optionalString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +function positiveSafeInteger(value: unknown): number | undefined { + return Number.isSafeInteger(value) && Number(value) > 0 ? Number(value) : undefined; +} + function withCrawlOptions( - parameters: Record, + config: DifyDatasourceSourceConfig, source: Source, uri: string, ): Record { + const parameters = config.parameters; + // New datasource-driven Sources persist the exact plugin parameters. Preserve the legacy + // URL, provider-specific subpage flag, and limit projection only for older Sources. + if (source.metadata.datasourceParameterMode === "exact") { + return parameters; + } const crawlOptions = plainObject(source.metadata.crawlOptions); const includeSubpages = crawlOptions.includeSubpages; const limit = crawlOptions.limit; + const subpagesParameter = /jina|watercrawl/u.test( + `${config.pluginId} ${config.provider} ${config.datasource}`.toLowerCase(), + ) + ? "crawl_sub_pages" + : "crawl_subpages"; return withCrawlUrl( { ...parameters, - ...(typeof includeSubpages === "boolean" ? { crawl_subpages: includeSubpages } : {}), + ...(typeof includeSubpages === "boolean" ? { [subpagesParameter]: includeSubpages } : {}), ...(Number.isSafeInteger(limit) && Number(limit) > 0 ? { limit } : {}), }, uri, diff --git a/knowledge-fs/apps/api/src/online-drive-options.test.ts b/knowledge-fs/apps/api/src/online-drive-options.test.ts index 254dcdecb4c..61ccae3777c 100644 --- a/knowledge-fs/apps/api/src/online-drive-options.test.ts +++ b/knowledge-fs/apps/api/src/online-drive-options.test.ts @@ -91,12 +91,12 @@ describe("createApiOnlineDriveConnector", () => { ]); expect(calls[0]).toMatchObject({ bucket: "b1", - maxKeys: 20, operation: "online_drive_browse_files", prefix: "docs/", source: SOURCE, tenantId: "tenant-1", }); + expect(calls[0]).not.toHaveProperty("maxKeys"); expect(JSON.stringify(calls[0])).not.toContain("credentials"); }); diff --git a/knowledge-fs/apps/api/src/online-drive-options.ts b/knowledge-fs/apps/api/src/online-drive-options.ts index aae45d14f34..b0122e35af4 100644 --- a/knowledge-fs/apps/api/src/online-drive-options.ts +++ b/knowledge-fs/apps/api/src/online-drive-options.ts @@ -6,8 +6,6 @@ import type { import type { ApiDatasourceInvocationClient } from "./datasource-invocation-client"; -const DEFAULT_MAX_KEYS = 20; - /** * Online-drive connector backed by the deployment-selected datasource runtime. Downloaded file bytes arrive as * ToolInvokeMessage @@ -33,7 +31,7 @@ export function createApiOnlineDriveConnector(input: { for await (const raw of input.client.dispatch({ ...(bucket === undefined ? {} : { bucket }), ...(continuationToken === undefined ? {} : { continuationToken }), - maxKeys: maxKeys ?? DEFAULT_MAX_KEYS, + ...(maxKeys === undefined ? {} : { maxKeys }), operation: "online_drive_browse_files", prefix: prefix ?? "", source, diff --git a/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts index b3dce69f054..26b6f7ddb7b 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/orpc.gen.ts @@ -3,6 +3,8 @@ import { oc } from '@orpc/contract' import * as z from 'zod' import { + zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdPath, + zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponse, zDeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdPath, zDeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdResponse, zDeleteKnowledgeFsSpacesByControlSpaceIdCredentialsByCredentialIdPath, @@ -44,6 +46,8 @@ import { zGetKnowledgeFsResearchTasksByTaskIdEventsPath, zGetKnowledgeFsResearchTasksByTaskIdEventsQuery, zGetKnowledgeFsResearchTasksByTaskIdEventsResponse, + zGetKnowledgeFsSourceProviderPreviewJobsByJobIdPath, + zGetKnowledgeFsSourceProviderPreviewJobsByJobIdResponse, zGetKnowledgeFsSpacesByControlSpaceIdAppBindingsPath, zGetKnowledgeFsSpacesByControlSpaceIdAppBindingsResponse, zGetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksPath, @@ -194,6 +198,8 @@ import { zPostKnowledgeFsQueryStreamBody, zPostKnowledgeFsQueryStreamResponse, zPostKnowledgeFsSourceProviderPreviewBody, + zPostKnowledgeFsSourceProviderPreviewJobsBody, + zPostKnowledgeFsSourceProviderPreviewJobsResponse, zPostKnowledgeFsSourceProviderPreviewResponse, zPostKnowledgeFsSpacesBody, zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelPath, @@ -384,7 +390,51 @@ export const researchTasks = { byTaskId, } +export const delete_ = oc + .route({ + inputStructure: 'detailed', + method: 'DELETE', + operationId: 'deleteKnowledgeFsSourceProviderPreviewJobsByJobId', + path: '/knowledge-fs/source-provider-preview/jobs/{job_id}', + tags: ['console'], + }) + .input(z.object({ params: zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdPath })) + .output(zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponse) + +export const get3 = oc + .route({ + inputStructure: 'detailed', + method: 'GET', + operationId: 'getKnowledgeFsSourceProviderPreviewJobsByJobId', + path: '/knowledge-fs/source-provider-preview/jobs/{job_id}', + tags: ['console'], + }) + .input(z.object({ params: zGetKnowledgeFsSourceProviderPreviewJobsByJobIdPath })) + .output(zGetKnowledgeFsSourceProviderPreviewJobsByJobIdResponse) + +export const byJobId = { + delete: delete_, + get: get3, +} + export const post2 = oc + .route({ + inputStructure: 'detailed', + method: 'POST', + operationId: 'postKnowledgeFsSourceProviderPreviewJobs', + path: '/knowledge-fs/source-provider-preview/jobs', + successStatus: 202, + tags: ['console'], + }) + .input(z.object({ body: zPostKnowledgeFsSourceProviderPreviewJobsBody })) + .output(zPostKnowledgeFsSourceProviderPreviewJobsResponse) + +export const jobs = { + post: post2, + byJobId, +} + +export const post3 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -396,10 +446,11 @@ export const post2 = oc .output(zPostKnowledgeFsSourceProviderPreviewResponse) export const sourceProviderPreview = { - post: post2, + post: post3, + jobs, } -export const delete_ = oc +export const delete2 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -416,14 +467,14 @@ export const delete_ = oc .output(zDeleteKnowledgeFsSpacesByControlSpaceIdAppBindingsByCallerKindByAppIdResponse) export const byAppId = { - delete: delete_, + delete: delete2, } export const byCallerKind = { byAppId, } -export const get3 = oc +export const get4 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -451,12 +502,12 @@ export const put = oc .output(zPutKnowledgeFsSpacesByControlSpaceIdAppBindingsResponse) export const appBindings = { - get: get3, + get: get4, put, byCallerKind, } -export const post3 = oc +export const post4 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -472,10 +523,10 @@ export const post3 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdCancelResponse) export const cancel = { - post: post3, + post: post4, } -export const post4 = oc +export const post5 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -491,7 +542,7 @@ export const post4 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdBackgroundTasksByTaskKindByTaskIdRetryResponse) export const retry = { - post: post4, + post: post5, } export const byTaskId2 = { @@ -503,7 +554,7 @@ export const byTaskKind = { byTaskId: byTaskId2, } -export const get4 = oc +export const get5 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -520,11 +571,11 @@ export const get4 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdBackgroundTasksResponse) export const backgroundTasks = { - get: get4, + get: get5, byTaskKind, } -export const get5 = oc +export const get6 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -535,15 +586,15 @@ export const get5 = oc .input(z.object({ params: zGetKnowledgeFsSpacesByControlSpaceIdBulkJobsByJobIdPath })) .output(zGetKnowledgeFsSpacesByControlSpaceIdBulkJobsByJobIdResponse) -export const byJobId = { - get: get5, +export const byJobId2 = { + get: get6, } export const bulkJobs = { - byJobId, + byJobId: byJobId2, } -export const delete2 = oc +export const delete3 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -558,10 +609,10 @@ export const delete2 = oc .output(zDeleteKnowledgeFsSpacesByControlSpaceIdCredentialsByCredentialIdResponse) export const byCredentialId = { - delete: delete2, + delete: delete3, } -export const get6 = oc +export const get7 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -572,7 +623,7 @@ export const get6 = oc .input(z.object({ params: zGetKnowledgeFsSpacesByControlSpaceIdCredentialsPath })) .output(zGetKnowledgeFsSpacesByControlSpaceIdCredentialsResponse) -export const post5 = oc +export const post6 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -590,12 +641,12 @@ export const post5 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdCredentialsResponse) export const credentials = { - get: get6, - post: post5, + get: get7, + post: post6, byCredentialId, } -export const delete3 = oc +export const delete4 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -614,10 +665,10 @@ export const delete3 = oc .output(zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsBulkResponse) export const bulk = { - delete: delete3, + delete: delete4, } -export const post6 = oc +export const post7 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -634,10 +685,10 @@ export const post6 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdDocumentsReindexResponse) export const reindex = { - post: post6, + post: post7, } -export const get7 = oc +export const get8 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -651,10 +702,10 @@ export const get7 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdOutlineResponse) export const outline = { - get: get7, + get: get8, } -export const get8 = oc +export const get9 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -674,10 +725,10 @@ export const get8 = oc ) export const byChunkId = { - get: get8, + get: get9, } -export const get9 = oc +export const get10 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -699,7 +750,7 @@ export const get9 = oc ) export const chunks = { - get: get9, + get: get10, byChunkId, } @@ -707,7 +758,7 @@ export const byRevision = { chunks, } -export const get10 = oc +export const get11 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -724,11 +775,11 @@ export const get10 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdRevisionsResponse) export const revisions = { - get: get10, + get: get11, byRevision, } -export const delete4 = oc +export const delete5 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -746,7 +797,7 @@ export const delete4 = oc ) .output(zDeleteKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdResponse) -export const get11 = oc +export const get12 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -774,14 +825,14 @@ export const patch = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdDocumentsByDocumentIdResponse) export const byDocumentId = { - delete: delete4, - get: get11, + delete: delete5, + get: get12, patch, outline, revisions, } -export const get12 = oc +export const get13 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -800,7 +851,7 @@ export const get12 = oc /** * Claim a workspace-staged upload. Multipart file bodies remain accepted as a legacy compatibility path. */ -export const post7 = oc +export const post8 = oc .route({ description: 'Claim a workspace-staged upload. Multipart file bodies remain accepted as a legacy compatibility path.', @@ -820,14 +871,14 @@ export const post7 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdDocumentsResponse) export const documents = { - get: get12, - post: post7, + get: get13, + post: post8, bulk, reindex, byDocumentId, } -export const get13 = oc +export const get14 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -855,11 +906,11 @@ export const put2 = oc .output(zPutKnowledgeFsSpacesByControlSpaceIdExternalAccessResponse) export const externalAccess = { - get: get13, + get: get14, put: put2, } -export const post8 = oc +export const post9 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -877,10 +928,10 @@ export const post8 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsBulkImportResponse) export const bulkImport = { - post: post8, + post: post9, } -export const post9 = oc +export const post10 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -897,10 +948,10 @@ export const post9 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsEvidenceMatchesResponse) export const evidenceMatches = { - post: post9, + post: post10, } -export const delete5 = oc +export const delete6 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -931,11 +982,11 @@ export const patch2 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsByQuestionIdResponse) export const byQuestionId = { - delete: delete5, + delete: delete6, patch: patch2, } -export const get14 = oc +export const get15 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -951,7 +1002,7 @@ export const get14 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsResponse) -export const post10 = oc +export const post11 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -969,14 +1020,14 @@ export const post10 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdGoldenQuestionsResponse) export const goldenQuestions = { - get: get14, - post: post10, + get: get15, + post: post11, bulkImport, evidenceMatches, byQuestionId, } -export const post11 = oc +export const post12 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -988,10 +1039,10 @@ export const post11 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdJobsByJobIdRetryResponse) export const retry2 = { - post: post11, + post: post12, } -export const delete6 = oc +export const delete7 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -1002,7 +1053,7 @@ export const delete6 = oc .input(z.object({ params: zDeleteKnowledgeFsSpacesByControlSpaceIdJobsByJobIdPath })) .output(zDeleteKnowledgeFsSpacesByControlSpaceIdJobsByJobIdResponse) -export const get15 = oc +export const get16 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1013,17 +1064,17 @@ export const get15 = oc .input(z.object({ params: zGetKnowledgeFsSpacesByControlSpaceIdJobsByJobIdPath })) .output(zGetKnowledgeFsSpacesByControlSpaceIdJobsByJobIdResponse) -export const byJobId2 = { - delete: delete6, - get: get15, +export const byJobId3 = { + delete: delete7, + get: get16, retry: retry2, } -export const jobs = { - byJobId: byJobId2, +export const jobs2 = { + byJobId: byJobId3, } -export const delete7 = oc +export const delete8 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -1042,10 +1093,10 @@ export const delete7 = oc .output(zDeleteKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsBulkResponse) export const bulk2 = { - delete: delete7, + delete: delete8, } -export const post12 = oc +export const post13 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1062,10 +1113,10 @@ export const post12 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsDownloadZipResponse) export const downloadZip = { - post: post12, + post: post13, } -export const get16 = oc +export const get17 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1081,10 +1132,10 @@ export const get16 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdDownloadResponse) export const download = { - get: get16, + get: get17, } -export const delete8 = oc +export const delete9 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -1102,7 +1153,7 @@ export const delete8 = oc ) .output(zDeleteKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponse) -export const get17 = oc +export const get18 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1132,13 +1183,13 @@ export const patch3 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsByDocumentIdResponse) export const byDocumentId2 = { - delete: delete8, - get: get17, + delete: delete9, + get: get18, patch: patch3, download, } -export const get18 = oc +export const get19 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1171,7 +1222,7 @@ export const patch4 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdLogicalDocumentsResponse) export const logicalDocuments = { - get: get18, + get: get19, patch: patch4, bulk: bulk2, downloadZip, @@ -1198,7 +1249,7 @@ export const members = { put: put3, } -export const delete9 = oc +export const delete10 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -1231,11 +1282,11 @@ export const patch5 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdMetadataByFieldIdResponse) export const byFieldId = { - delete: delete9, + delete: delete10, patch: patch5, } -export const get19 = oc +export const get20 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1251,7 +1302,7 @@ export const get19 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdMetadataResponse) -export const post13 = oc +export const post14 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1269,12 +1320,12 @@ export const post13 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdMetadataResponse) export const metadata = { - get: get19, - post: post13, + get: get20, + post: post14, byFieldId, } -export const get20 = oc +export const get21 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1291,10 +1342,10 @@ export const get20 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdOverviewActivityResponse) export const activity = { - get: get20, + get: get21, } -export const get21 = oc +export const get22 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1311,10 +1362,10 @@ export const get21 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdOverviewAttentionResponse) export const attention = { - get: get21, + get: get22, } -export const get22 = oc +export const get23 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1326,10 +1377,10 @@ export const get22 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdOverviewHealthResponse) export const health = { - get: get22, + get: get23, } -export const get23 = oc +export const get24 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1341,10 +1392,10 @@ export const get23 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdOverviewInventoryResponse) export const inventory = { - get: get23, + get: get24, } -export const get24 = oc +export const get25 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1361,10 +1412,10 @@ export const get24 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdOverviewQueryOutcomesResponse) export const queryOutcomes = { - get: get24, + get: get25, } -export const get25 = oc +export const get26 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1381,7 +1432,7 @@ export const get25 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdOverviewStatsResponse) export const stats = { - get: get25, + get: get26, } export const overview = { @@ -1393,7 +1444,7 @@ export const overview = { stats, } -export const get26 = oc +export const get27 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1405,10 +1456,10 @@ export const get26 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdPermissionsResponse) export const permissions = { - get: get26, + get: get27, } -export const get27 = oc +export const get28 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1424,10 +1475,10 @@ export const get27 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdQualityBadCasesByBadCaseIdTraceReferenceResponse) export const traceReference = { - get: get27, + get: get28, } -export const get28 = oc +export const get29 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1455,12 +1506,12 @@ export const patch6 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdQualityBadCasesByBadCaseIdResponse) export const byBadCaseId = { - get: get28, + get: get29, patch: patch6, traceReference, } -export const get29 = oc +export const get30 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1476,7 +1527,7 @@ export const get29 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdQualityBadCasesResponse) -export const post14 = oc +export const post15 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1494,12 +1545,12 @@ export const post14 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQualityBadCasesResponse) export const badCases = { - get: get29, - post: post14, + get: get30, + post: post15, byBadCaseId, } -export const post15 = oc +export const post16 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1518,7 +1569,7 @@ export const post15 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQualityReplayRunsResponse) export const replayRuns = { - post: post15, + post: post16, } export const quality = { @@ -1526,7 +1577,7 @@ export const quality = { replayRuns, } -export const post16 = oc +export const post17 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1543,13 +1594,13 @@ export const post16 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQueriesAdmissionResponse) export const admission = { - post: post16, + post: post17, } /** * @deprecated */ -export const post17 = oc +export const post18 = oc .route({ deprecated: true, inputStructure: 'detailed', @@ -1568,14 +1619,14 @@ export const post17 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQueriesResponse) export const queries = { - post: post17, + post: post18, admission, } /** * @deprecated */ -export const post18 = oc +export const post19 = oc .route({ deprecated: true, inputStructure: 'detailed', @@ -1588,10 +1639,10 @@ export const post18 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdQueryStreamCapabilityResponse) export const queryStreamCapability = { - post: post18, + post: post19, } -export const post19 = oc +export const post20 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1608,10 +1659,10 @@ export const post19 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdResearchTasksPlanResponse) export const plan = { - post: post19, + post: post20, } -export const get30 = oc +export const get31 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1628,10 +1679,10 @@ export const get30 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdResearchTasksByTaskIdPartialsResponse) export const partials = { - get: get30, + get: get31, } -export const delete10 = oc +export const delete11 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -1642,7 +1693,7 @@ export const delete10 = oc .input(z.object({ params: zDeleteKnowledgeFsSpacesByControlSpaceIdResearchTasksByTaskIdPath })) .output(zDeleteKnowledgeFsSpacesByControlSpaceIdResearchTasksByTaskIdResponse) -export const get31 = oc +export const get32 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1654,12 +1705,12 @@ export const get31 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdResearchTasksByTaskIdResponse) export const byTaskId3 = { - delete: delete10, - get: get31, + delete: delete11, + get: get32, partials, } -export const get32 = oc +export const get33 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1675,7 +1726,7 @@ export const get32 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdResearchTasksResponse) -export const post20 = oc +export const post21 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1693,13 +1744,13 @@ export const post20 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdResearchTasksResponse) export const researchTasks2 = { - get: get32, - post: post20, + get: get33, + post: post21, plan, byTaskId: byTaskId3, } -export const get33 = oc +export const get34 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1713,14 +1764,14 @@ export const get33 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdSettingsMigrationsByMigrationIdResponse) export const byMigrationId = { - get: get33, + get: get34, } export const migrations = { byMigrationId, } -export const get34 = oc +export const get35 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1748,12 +1799,12 @@ export const patch7 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdSettingsResponse) export const settings = { - get: get34, + get: get35, patch: patch7, migrations, } -export const post21 = oc +export const post22 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1770,14 +1821,14 @@ export const post21 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsByConnectionIdRefreshResponse) export const refresh = { - post: post21, + post: post22, } export const byConnectionId = { refresh, } -export const get35 = oc +export const get36 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1793,7 +1844,7 @@ export const get35 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse) -export const post22 = oc +export const post23 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1811,12 +1862,12 @@ export const post22 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceConnectionsResponse) export const sourceConnections = { - get: get35, - post: post22, + get: get36, + post: post23, byConnectionId, } -export const get36 = oc +export const get37 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1828,10 +1879,10 @@ export const get36 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdSourceProvidersResponse) export const sourceProviders = { - get: get36, + get: get37, } -export const post23 = oc +export const post24 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1848,10 +1899,10 @@ export const post23 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdCancelResponse) export const cancel2 = { - post: post23, + post: post24, } -export const get37 = oc +export const get38 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1868,10 +1919,10 @@ export const get37 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdPagesResponse) export const pages = { - get: get37, + get: get38, } -export const post24 = oc +export const post25 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1885,10 +1936,10 @@ export const post24 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdRetryResponse) export const retry3 = { - post: post24, + post: post25, } -export const post25 = oc +export const post26 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1907,10 +1958,10 @@ export const post25 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdSelectionResponse) export const selection = { - post: post25, + post: post26, } -export const get38 = oc +export const get39 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1922,7 +1973,7 @@ export const get38 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdSourceWorkflowsByRunIdResponse) export const byRunId = { - get: get38, + get: get39, cancel: cancel2, pages, retry: retry3, @@ -1933,7 +1984,7 @@ export const sourceWorkflows = { byRunId, } -export const post26 = oc +export const post27 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1952,10 +2003,10 @@ export const post26 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlImportResponse) export const crawlImport = { - post: post26, + post: post27, } -export const post27 = oc +export const post28 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1973,10 +2024,10 @@ export const post27 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdCrawlPreviewResponse) export const crawlPreview = { - post: post27, + post: post28, } -export const get39 = oc +export const get40 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1993,10 +2044,10 @@ export const get39 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdFilesResponse) export const files = { - get: get39, + get: get40, } -export const post28 = oc +export const post29 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2013,10 +2064,10 @@ export const post28 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdImportResponse) export const import_ = { - post: post28, + post: post29, } -export const post29 = oc +export const post30 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2033,10 +2084,10 @@ export const post29 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdImportFilesResponse) export const importFiles = { - post: post29, + post: post30, } -export const get40 = oc +export const get41 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2053,10 +2104,10 @@ export const get40 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesResponse) export const pages2 = { - get: get40, + get: get41, } -export const post30 = oc +export const post31 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2074,10 +2125,10 @@ export const post30 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncResponse) export const sync = { - post: post30, + post: post31, } -export const get41 = oc +export const get42 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2105,11 +2156,11 @@ export const put4 = oc .output(zPutKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdSyncPolicyResponse) export const syncPolicy = { - get: get41, + get: get42, put: put4, } -export const post31 = oc +export const post32 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2121,10 +2172,10 @@ export const post31 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdTestResponse) export const test = { - post: post31, + post: post32, } -export const post32 = oc +export const post33 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2143,10 +2194,10 @@ export const post32 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdWorkflowImportsResponse) export const workflowImports = { - post: post32, + post: post33, } -export const delete11 = oc +export const delete12 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -2165,7 +2216,7 @@ export const delete11 = oc ) .output(zDeleteKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponse) -export const get42 = oc +export const get43 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2193,8 +2244,8 @@ export const patch8 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdResponse) export const bySourceId = { - delete: delete11, - get: get42, + delete: delete12, + get: get43, patch: patch8, crawlImport, crawlPreview, @@ -2208,7 +2259,7 @@ export const bySourceId = { workflowImports, } -export const get43 = oc +export const get44 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2224,7 +2275,7 @@ export const get43 = oc ) .output(zGetKnowledgeFsSpacesByControlSpaceIdSourcesResponse) -export const post33 = oc +export const post34 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2242,12 +2293,12 @@ export const post33 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdSourcesResponse) export const sources = { - get: get43, - post: post33, + get: get44, + post: post34, bySourceId, } -export const get44 = oc +export const get45 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2264,10 +2315,10 @@ export const get44 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdTracesByTraceIdConflictsResponse) export const conflicts = { - get: get44, + get: get45, } -export const get45 = oc +export const get46 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2284,10 +2335,10 @@ export const get45 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdTracesByTraceIdEvidenceResponse) export const evidence = { - get: get45, + get: get46, } -export const get46 = oc +export const get47 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2304,10 +2355,10 @@ export const get46 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdTracesByTraceIdMissingResponse) export const missing = { - get: get46, + get: get47, } -export const get47 = oc +export const get48 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2319,13 +2370,13 @@ export const get47 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdTracesByTraceIdResponse) export const byTraceId = { - get: get47, + get: get48, conflicts, evidence, missing, } -export const get48 = oc +export const get49 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2342,11 +2393,11 @@ export const get48 = oc .output(zGetKnowledgeFsSpacesByControlSpaceIdTracesResponse) export const traces = { - get: get48, + get: get49, byTraceId, } -export const post34 = oc +export const post35 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2363,10 +2414,10 @@ export const post34 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdAbortResponse) export const abort = { - post: post34, + post: post35, } -export const post35 = oc +export const post36 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2383,10 +2434,10 @@ export const post35 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdCompleteResponse) export const complete = { - post: post35, + post: post36, } -export const post36 = oc +export const post37 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2407,7 +2458,7 @@ export const post36 = oc ) export const presign = { - post: post36, + post: post37, } export const byPartNumber = { @@ -2418,7 +2469,7 @@ export const parts = { byPartNumber, } -export const post37 = oc +export const post38 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2435,7 +2486,7 @@ export const post37 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsByUploadSessionIdSmallFileResponse) export const smallFile = { - post: post37, + post: post38, } export const byUploadSessionId = { @@ -2445,7 +2496,7 @@ export const byUploadSessionId = { smallFile, } -export const post38 = oc +export const post39 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2464,11 +2515,11 @@ export const post38 = oc .output(zPostKnowledgeFsSpacesByControlSpaceIdUploadSessionsResponse) export const uploadSessions = { - post: post38, + post: post39, byUploadSessionId, } -export const delete12 = oc +export const delete13 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -2480,7 +2531,7 @@ export const delete12 = oc .input(z.object({ params: zDeleteKnowledgeFsSpacesByControlSpaceIdPath })) .output(zDeleteKnowledgeFsSpacesByControlSpaceIdResponse) -export const get49 = oc +export const get50 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2508,8 +2559,8 @@ export const patch9 = oc .output(zPatchKnowledgeFsSpacesByControlSpaceIdResponse) export const byControlSpaceId = { - delete: delete12, - get: get49, + delete: delete13, + get: get50, patch: patch9, appBindings, backgroundTasks, @@ -2518,7 +2569,7 @@ export const byControlSpaceId = { documents, externalAccess, goldenQuestions, - jobs, + jobs: jobs2, logicalDocuments, members, metadata, @@ -2537,7 +2588,7 @@ export const byControlSpaceId = { uploadSessions, } -export const get50 = oc +export const get51 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -2548,7 +2599,7 @@ export const get50 = oc .input(z.object({ query: zGetKnowledgeFsSpacesQuery.optional() })) .output(zGetKnowledgeFsSpacesResponse) -export const post39 = oc +export const post40 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2561,12 +2612,12 @@ export const post39 = oc .output(zPostKnowledgeFsSpacesResponse) export const spaces = { - get: get50, - post: post39, + get: get51, + post: post40, byControlSpaceId, } -export const post40 = oc +export const post41 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2583,7 +2634,7 @@ export const post40 = oc .output(zPostKnowledgeFsTasksByTaskIdStreamCapabilityResponse) export const streamCapability = { - post: post40, + post: post41, } export const byTaskId4 = { @@ -2594,7 +2645,7 @@ export const tasks = { byTaskId: byTaskId4, } -export const delete13 = oc +export const delete14 = oc .route({ inputStructure: 'detailed', method: 'DELETE', @@ -2607,10 +2658,10 @@ export const delete13 = oc .output(zDeleteKnowledgeFsUploadsByUploadIdResponse) export const byUploadId = { - delete: delete13, + delete: delete14, } -export const post41 = oc +export const post42 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -2623,7 +2674,7 @@ export const post41 = oc .output(zPostKnowledgeFsUploadsResponse) export const uploads = { - post: post41, + post: post42, byUploadId, } diff --git a/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts index 771f2877201..b19f6cd656e 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts @@ -33,10 +33,34 @@ export type KnowledgeFsInitialSourcePreviewPayload = { export type KnowledgeFsInitialSourcePreviewResponse = { documents?: Array files?: Array - kind: 'online_document' | 'online_drive' + kind: 'online_document' | 'online_drive' | 'website_crawl' next_page_parameters?: { [key: string]: JsonValue } | null + pages?: Array +} + +export type KnowledgeFsInitialWebsiteSourcePreviewPayload = { + credentialId: string + datasource: string + kind: 'website_crawl' + parameters?: { + [key: string]: JsonValue + } + pluginId: string + provider: string + providerDisplayName?: string | null +} + +export type KnowledgeFsInitialSourcePreviewJobCreateResponse = { + job_id: string + status?: 'pending' +} + +export type KnowledgeFsInitialSourcePreviewJobResponse = { + job_id: string + result?: KnowledgeFsInitialSourcePreviewResponse | null + status: 'canceled' | 'completed' | 'failed' | 'pending' | 'running' } export type KnowledgeFsSpaceListResponse = { @@ -1040,6 +1064,12 @@ export type KnowledgeFsInitialSourcePreviewFileResponse = { type: string } +export type KnowledgeFsInitialSourcePreviewPageResponse = { + description?: string | null + source_url: string + title?: string | null +} + export type KnowledgeFsSpaceListItemResponse = { control_space_id: string created_at: string @@ -1067,6 +1097,9 @@ export type KnowledgeFsInitialWebsiteSourcePayload = { datasource?: string kind: 'website_crawl' name: string + parameters?: { + [key: string]: JsonValue + } pluginId?: string | null provider: string providerDisplayName?: string | null @@ -1080,6 +1113,9 @@ export type KnowledgeFsInitialOnlineDocumentSourcePayload = { datasource: string kind: 'online_document' name: string + parameters?: { + [key: string]: JsonValue + } pluginId: string provider: string providerDisplayName?: string | null @@ -1092,6 +1128,9 @@ export type KnowledgeFsInitialOnlineDriveSourcePayload = { datasource: string kind: 'online_drive' name: string + parameters?: { + [key: string]: JsonValue + } pluginId: string provider: string providerDisplayName?: string | null @@ -1820,6 +1859,52 @@ export type PostKnowledgeFsSourceProviderPreviewResponses = { export type PostKnowledgeFsSourceProviderPreviewResponse = PostKnowledgeFsSourceProviderPreviewResponses[keyof PostKnowledgeFsSourceProviderPreviewResponses] +export type PostKnowledgeFsSourceProviderPreviewJobsData = { + body: KnowledgeFsInitialWebsiteSourcePreviewPayload + path?: never + query?: never + url: '/knowledge-fs/source-provider-preview/jobs' +} + +export type PostKnowledgeFsSourceProviderPreviewJobsResponses = { + 202: KnowledgeFsInitialSourcePreviewJobCreateResponse +} + +export type PostKnowledgeFsSourceProviderPreviewJobsResponse = + PostKnowledgeFsSourceProviderPreviewJobsResponses[keyof PostKnowledgeFsSourceProviderPreviewJobsResponses] + +export type DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdData = { + body?: never + path: { + job_id: string + } + query?: never + url: '/knowledge-fs/source-provider-preview/jobs/{job_id}' +} + +export type DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponses = { + 200: KnowledgeFsInitialSourcePreviewJobResponse +} + +export type DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponse = + DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponses[keyof DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponses] + +export type GetKnowledgeFsSourceProviderPreviewJobsByJobIdData = { + body?: never + path: { + job_id: string + } + query?: never + url: '/knowledge-fs/source-provider-preview/jobs/{job_id}' +} + +export type GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponses = { + 200: KnowledgeFsInitialSourcePreviewJobResponse +} + +export type GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponse = + GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponses[keyof GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponses] + export type GetKnowledgeFsSpacesData = { body?: never path?: never diff --git a/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts index 060dcd1bcd1..e2929ad5f8b 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts @@ -2,6 +2,14 @@ import * as z from 'zod' +/** + * KnowledgeFSInitialSourcePreviewJobCreateResponse + */ +export const zKnowledgeFsInitialSourcePreviewJobCreateResponse = z.object({ + job_id: z.string(), + status: z.literal('pending').optional().default('pending'), +}) + /** * KnowledgeFSBulkJobResponse */ @@ -755,6 +763,19 @@ export const zKnowledgeFsInitialSourcePreviewPayload = z.object({ providerDisplayName: z.string().min(1).max(255).nullish(), }) +/** + * KnowledgeFSInitialWebsiteSourcePreviewPayload + */ +export const zKnowledgeFsInitialWebsiteSourcePreviewPayload = z.object({ + credentialId: z.string().min(1).max(255), + datasource: z.string().min(1).max(255), + kind: z.literal('website_crawl'), + parameters: z.record(z.string(), zJsonValue).optional(), + pluginId: z.string().min(1).max(255), + provider: z.string().min(1).max(255), + providerDisplayName: z.string().min(1).max(255).nullish(), +}) + /** * KnowledgeFSInitialSourcePreviewDocumentResponse */ @@ -781,14 +802,33 @@ export const zKnowledgeFsInitialSourcePreviewFileResponse = z.object({ type: z.string(), }) +/** + * KnowledgeFSInitialSourcePreviewPageResponse + */ +export const zKnowledgeFsInitialSourcePreviewPageResponse = z.object({ + description: z.string().nullish(), + source_url: z.string(), + title: z.string().nullish(), +}) + /** * KnowledgeFSInitialSourcePreviewResponse */ export const zKnowledgeFsInitialSourcePreviewResponse = z.object({ documents: z.array(zKnowledgeFsInitialSourcePreviewDocumentResponse).optional(), files: z.array(zKnowledgeFsInitialSourcePreviewFileResponse).optional(), - kind: z.enum(['online_document', 'online_drive']), + kind: z.enum(['online_document', 'online_drive', 'website_crawl']), next_page_parameters: z.record(z.string(), zJsonValue).nullish(), + pages: z.array(zKnowledgeFsInitialSourcePreviewPageResponse).optional(), +}) + +/** + * KnowledgeFSInitialSourcePreviewJobResponse + */ +export const zKnowledgeFsInitialSourcePreviewJobResponse = z.object({ + job_id: z.string(), + result: zKnowledgeFsInitialSourcePreviewResponse.nullish(), + status: z.enum(['canceled', 'completed', 'failed', 'pending', 'running']), }) /** @@ -1742,6 +1782,7 @@ export const zKnowledgeFsInitialWebsiteSourcePayload = z.object({ datasource: z.string().min(1).max(255).optional().default('crawl'), kind: z.literal('website_crawl'), name: z.string().min(1).max(200), + parameters: z.record(z.string(), zJsonValue).optional(), pluginId: z.string().min(1).max(255).nullish(), provider: z.string().min(1).max(255), providerDisplayName: z.string().min(1).max(255).nullish(), @@ -1771,6 +1812,7 @@ export const zKnowledgeFsInitialOnlineDocumentSourcePayload = z.object({ datasource: z.string().min(1).max(255), kind: z.literal('online_document'), name: z.string().min(1).max(200), + parameters: z.record(z.string(), zJsonValue).optional(), pluginId: z.string().min(1).max(255), provider: z.string().min(1).max(255), providerDisplayName: z.string().min(1).max(255).nullish(), @@ -1806,6 +1848,7 @@ export const zKnowledgeFsInitialOnlineDriveSourcePayload = z.object({ datasource: z.string().min(1).max(255), kind: z.literal('online_drive'), name: z.string().min(1).max(200), + parameters: z.record(z.string(), zJsonValue).optional(), pluginId: z.string().min(1).max(255), provider: z.string().min(1).max(255), providerDisplayName: z.string().min(1).max(255).nullish(), @@ -2401,6 +2444,35 @@ export const zPostKnowledgeFsSourceProviderPreviewBody = zKnowledgeFsInitialSour export const zPostKnowledgeFsSourceProviderPreviewResponse = zKnowledgeFsInitialSourcePreviewResponse +export const zPostKnowledgeFsSourceProviderPreviewJobsBody = + zKnowledgeFsInitialWebsiteSourcePreviewPayload + +/** + * Website datasource preview queued + */ +export const zPostKnowledgeFsSourceProviderPreviewJobsResponse = + zKnowledgeFsInitialSourcePreviewJobCreateResponse + +export const zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdPath = z.object({ + job_id: z.string(), +}) + +/** + * Website datasource preview canceled + */ +export const zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponse = + zKnowledgeFsInitialSourcePreviewJobResponse + +export const zGetKnowledgeFsSourceProviderPreviewJobsByJobIdPath = z.object({ + job_id: z.string(), +}) + +/** + * Website datasource preview status + */ +export const zGetKnowledgeFsSourceProviderPreviewJobsByJobIdResponse = + zKnowledgeFsInitialSourcePreviewJobResponse + export const zGetKnowledgeFsSpacesQuery = z.object({ creator_ids: z.array(z.string().min(1).max(255)).max(100).optional(), limit: z.int().gte(1).lte(100).optional().default(20), diff --git a/web/features/new-rag/__tests__/add-source-page.spec.tsx b/web/features/new-rag/__tests__/add-source-page.spec.tsx index 7f678e5cce3..78c321b6fda 100644 --- a/web/features/new-rag/__tests__/add-source-page.spec.tsx +++ b/web/features/new-rag/__tests__/add-source-page.spec.tsx @@ -676,7 +676,6 @@ describe('AddSourcePage', () => { }) it('restores a website draft handed off by the creation flow', async () => { - const user = userEvent.setup() queryState.connections.data = { pages: [{ items: [connection('active')] }] } render( @@ -700,11 +699,12 @@ describe('AddSourcePage', () => { expect(screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ })).toHaveValue( 'Dify docs', ) - await user.click(screen.getByRole('button', { name: /dataset\.newKnowledge\.crawlOptions/ })) expect( - screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' }), + screen.getByRole('switch', { name: 'dataset.newKnowledge.includeSubpages' }), ).not.toBeChecked() - expect(screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })).toHaveValue('25') + expect(screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })).toHaveValue( + 25, + ) }) it('keeps the exact website provider selected while loading website dependencies', async () => { @@ -1061,6 +1061,57 @@ describe('AddSourcePage', () => { ).toBeEnabled() }) + it('clears website parameters when switching between configured providers', async () => { + const user = userEvent.setup() + queryState.providers.data = { items: [difyManagedFirecrawlProvider] } + queryState.datasourcePlugins.data = [firecrawlDatasourcePlugin, jinaDatasourcePlugin] + queryState.datasourceAuth.data = { result: [firecrawlDatasourceAuth, jinaDatasourceAuth] } + queryState.connections.data = { + pages: [ + { + items: [ + { + ...connection('active'), + authKind: 'endpoint', + configuration: { + credentialId: 'firecrawl-credential-1', + datasource: 'crawl', + pluginId: 'langgenius/firecrawl_datasource', + provider: 'firecrawl', + providerKind: 'website', + }, + }, + { + ...connection('active'), + authKind: 'endpoint', + configuration: { + credentialId: 'jina-credential-1', + datasource: 'jina_reader', + pluginId: 'langgenius/jina_datasource', + provider: 'jinareader', + providerKind: 'website', + }, + id: 'jina-connection-1', + name: 'Jina Reader', + }, + ], + }, + ], + } + + render() + const rootUrl = await screen.findByRole('textbox', { + name: /dataset\.newKnowledge\.rootUrl/, + }) + await user.type(rootUrl, 'https://firecrawl.example.com') + + await user.click(screen.getByRole('radio', { name: 'Jina Reader' })) + + expect( + await screen.findByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }), + ).toHaveValue('') + }) + it('opens Data Source settings when Dify has no Firecrawl credential', async () => { const user = userEvent.setup() queryState.providers.data = { items: [difyManagedFirecrawlProvider] } diff --git a/web/features/new-rag/__tests__/connected-source-setup.spec.tsx b/web/features/new-rag/__tests__/connected-source-setup.spec.tsx index 7191c8afe0d..3ada6dc9ec7 100644 --- a/web/features/new-rag/__tests__/connected-source-setup.spec.tsx +++ b/web/features/new-rag/__tests__/connected-source-setup.spec.tsx @@ -18,6 +18,7 @@ import type { DataSourceItem } from '@/app/components/workflow/block-selector/ty import { QueryClientProvider } from '@tanstack/react-query' import { act, screen, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' +import { useState } from 'react' import { render } from '@/test/console/render' import { createTestQueryClient } from '@/test/query-client' import { ConnectedSourceSetup } from '../connected-source-setup' @@ -263,6 +264,26 @@ const notionDatasourcePlugin: DataSourceItem = { provider: 'notion', } +const notionDatasourcePluginWithParameters: DataSourceItem = { + ...notionDatasourcePlugin, + declaration: { + ...notionDatasourcePlugin.declaration, + datasources: [ + { + ...notionDatasourcePlugin.declaration.datasources[0]!, + parameters: [ + { + label: { en_US: 'Workspace' }, + name: 'workspace', + required: true, + type: 'string', + }, + ], + }, + ], + }, +} + const s3Provider: KnowledgeFsSourceProviderResponse = { auth_kinds: ['endpoint'], available: true, @@ -489,6 +510,35 @@ function renderSetup(draft: ConnectedDraft = defaultDraft) { } } +function renderStatefulSetup(draft: ConnectedDraft = defaultDraft) { + const queryClient = createTestQueryClient() + const onCompleted = vi.fn() + const onDirtyChange = vi.fn() + const onDraftChange = vi.fn() + const onExit = vi.fn() + + function StatefulSetup() { + const [currentDraft, setCurrentDraft] = useState(draft) + return ( + + { + onDraftChange(nextDraft) + setCurrentDraft(nextDraft as ConnectedDraft) + }} + onExit={onExit} + /> + + ) + } + + return { ...render(), onDraftChange } +} + describe('ConnectedSourceSetup', () => { beforeEach(() => { for (const mock of Object.values(clientMock)) mock.mockReset() @@ -623,6 +673,8 @@ describe('ConnectedSourceSetup', () => { connectionId: 'connection-1', metadata: { clientRequestId: expect.any(String), + datasourceParameterMode: 'exact', + parameters: {}, preview: true, providerId: 'notion-provider', providerKind: 'online-document', @@ -657,6 +709,8 @@ describe('ConnectedSourceSetup', () => { expectedVersion: 3, metadata: { clientRequestId: expect.any(String), + datasourceParameterMode: 'exact', + parameters: {}, preview: false, providerId: 'notion-provider', providerKind: 'online-document', @@ -713,6 +767,35 @@ describe('ConnectedSourceSetup', () => { expect(clientMock.deleteSource).not.toHaveBeenCalled() }) + it('applies datasource parameters once instead of rebuilding the preview on every keypress', async () => { + const user = userEvent.setup() + clientMock.listDatasourcePlugins.mockResolvedValue([notionDatasourcePluginWithParameters]) + clientMock.listDatasourceAuth.mockResolvedValue({ + result: [notionDatasourceAuth([notionCredential])], + }) + clientMock.listConnections.mockResolvedValue({ + data: [connectionResponse()], + next_cursor: null, + } satisfies KnowledgeFsSourceConnectionListResponse) + + renderStatefulSetup({ ...defaultDraft, parameters: {} }) + + const workspace = await screen.findByRole('textbox', { name: 'Workspace' }) + await user.type(workspace, 'product-docs') + + expect(clientMock.createSource).not.toHaveBeenCalled() + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })) + + await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce()) + expect(clientMock.createSource).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + metadata: expect.objectContaining({ parameters: { workspace: 'product-docs' } }), + }), + }), + ) + }) + it('creates the managed endpoint connection automatically from the default credential', async () => { clientMock.listDatasourceAuth.mockResolvedValue({ result: [notionDatasourceAuth([notionCredential])], @@ -966,6 +1049,8 @@ describe('ConnectedSourceSetup', () => { connectionId: googleConnection.id, metadata: { clientRequestId: expect.any(String), + datasourceParameterMode: 'exact', + parameters: {}, preview: true, providerId: googleProvider.id, providerKind: 'online-drive', @@ -1207,6 +1292,8 @@ describe('ConnectedSourceSetup', () => { connectionId: 's3-connection', metadata: { clientRequestId: expect.any(String), + datasourceParameterMode: 'exact', + parameters: {}, preview: true, providerId: 's3-provider', providerKind: 'online-drive', diff --git a/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx b/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx index 527d36aeae8..881d395288c 100644 --- a/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx +++ b/web/features/new-rag/__tests__/create-knowledge-page.spec.tsx @@ -10,7 +10,10 @@ const serviceMock = vi.hoisted(() => ({ createKfsSource: vi.fn(), getKfsSource: vi.fn(), getCrawlStatus: vi.fn(), + cancelWebsitePreview: vi.fn(), + getWebsitePreview: vi.fn(), previewInitialSource: vi.fn(), + startWebsitePreview: vi.fn(), getDefaultModel: vi.fn(), getSpace: vi.fn(), getSyncPolicy: vi.fn(), @@ -102,6 +105,13 @@ vi.mock('@/service/client', () => ({ knowledgeFs: { sourceProviderPreview: { post: serviceMock.previewInitialSource, + jobs: { + post: serviceMock.startWebsitePreview, + byJobId: { + delete: serviceMock.cancelWebsitePreview, + get: serviceMock.getWebsitePreview, + }, + }, }, spaces: { byControlSpaceId: { @@ -272,6 +282,66 @@ const jinaDatasourceAuth = { provider: 'jinareader', } +const tavilyDatasourcePlugin = { + ...firecrawlDatasourcePlugin, + declaration: { + ...firecrawlDatasourcePlugin.declaration, + datasources: [ + { + description: { en_US: 'Search and extract', zh_Hans: '搜索与提取' }, + identity: { + author: 'langgenius', + label: { en_US: 'Tavily', zh_Hans: 'Tavily' }, + name: 'search_extract', + provider: 'tavily', + }, + parameters: [ + { + label: { en_US: 'Search query', zh_Hans: '搜索词' }, + name: 'query', + required: true, + type: 'string', + }, + { + default: 'basic', + label: { en_US: 'Search depth', zh_Hans: '搜索深度' }, + name: 'search_depth', + options: [ + { label: { en_US: 'Basic', zh_Hans: '基础' }, value: 'basic' }, + { label: { en_US: 'Advanced', zh_Hans: '高级' }, value: 'advanced' }, + ], + type: 'select', + }, + ], + }, + ], + identity: { + ...firecrawlDatasourcePlugin.declaration.identity, + label: { en_US: 'Tavily', zh_Hans: 'Tavily' }, + name: 'tavily', + }, + }, + plugin_id: 'langgenius/tavily_datasource', + plugin_unique_identifier: 'langgenius/tavily_datasource:1.0.0@local', + provider: 'tavily', +} + +const tavilyDatasourceAuth = { + ...firecrawlDatasourceAuth, + credentials_list: [ + { + ...firecrawlDatasourceAuth.credentials_list[0], + id: 'tavily-credential-1', + name: 'Default Tavily', + }, + ], + label: { en_US: 'Tavily' }, + name: 'tavily', + plugin_id: 'langgenius/tavily_datasource', + plugin_unique_identifier: 'langgenius/tavily_datasource:1.0.0@local', + provider: 'tavily', +} + const notionDatasourcePlugin = { ...firecrawlDatasourcePlugin, declaration: { @@ -300,6 +370,26 @@ const notionDatasourcePlugin = { provider: 'notion', } +const notionDatasourcePluginWithParameters = { + ...notionDatasourcePlugin, + declaration: { + ...notionDatasourcePlugin.declaration, + datasources: [ + { + ...notionDatasourcePlugin.declaration.datasources[0], + parameters: [ + { + label: { en_US: 'Workspace', zh_Hans: '工作区' }, + name: 'workspace', + required: true, + type: 'string', + }, + ], + }, + ], + }, +} + const notionDatasourceAuth = { ...firecrawlDatasourceAuth, credentials_list: [ @@ -316,6 +406,49 @@ const notionDatasourceAuth = { provider: 'notion', } +const outlineDatasourcePlugin = { + ...notionDatasourcePlugin, + declaration: { + ...notionDatasourcePlugin.declaration, + datasources: [ + { + ...notionDatasourcePlugin.declaration.datasources[0]!, + description: { en_US: 'Outline', zh_Hans: 'Outline' }, + identity: { + ...notionDatasourcePlugin.declaration.datasources[0]!.identity, + label: { en_US: 'Outline', zh_Hans: 'Outline' }, + name: 'outline', + provider: 'outline', + }, + }, + ], + identity: { + ...notionDatasourcePlugin.declaration.identity, + label: { en_US: 'Outline', zh_Hans: 'Outline' }, + name: 'outline', + }, + }, + plugin_id: 'langgenius/outline_datasource', + plugin_unique_identifier: 'langgenius/outline_datasource:1.0.0@local', + provider: 'outline', +} + +const outlineDatasourceAuth = { + ...notionDatasourceAuth, + credentials_list: [ + { + ...notionDatasourceAuth.credentials_list[0], + id: 'outline-credential-1', + name: 'Default Outline', + }, + ], + label: { en_US: 'Outline' }, + name: 'outline', + plugin_id: 'langgenius/outline_datasource', + plugin_unique_identifier: 'langgenius/outline_datasource:1.0.0@local', + provider: 'outline', +} + const googleDriveDatasourcePlugin = { ...firecrawlDatasourcePlugin, declaration: { @@ -465,6 +598,25 @@ describe('CreateKnowledgePage', () => { datasourceQueryMock.auth.error = null datasourceQueryMock.auth.isPending = false serviceMock.create.mockResolvedValue(createdKnowledge) + serviceMock.startWebsitePreview.mockResolvedValue({ job_id: 'website-preview-1' }) + serviceMock.getWebsitePreview.mockResolvedValue({ + job_id: 'website-preview-1', + status: 'completed', + result: { + kind: 'website_crawl', + pages: [ + { + description: 'Getting started', + source_url: 'https://docs.dify.ai/getting-started', + title: 'Getting started', + }, + ], + }, + }) + serviceMock.cancelWebsitePreview.mockResolvedValue({ + job_id: 'website-preview-1', + status: 'canceled', + }) serviceMock.createCrawl.mockResolvedValue({ job_id: 'crawl-job-1' }) serviceMock.createKfsSource.mockResolvedValue(kfsSourceResponse()) serviceMock.getKfsSource.mockResolvedValue(kfsSourceResponse({ status: 'active', version: 3 })) @@ -1025,12 +1177,10 @@ describe('CreateKnowledgePage', () => { await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' })) expect(screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' })).toBeEnabled() expect(screen.queryByRole('alert')).not.toBeInTheDocument() - expect(screen.getByText('dataset.newKnowledge.crawlOptions')).toBeInTheDocument() expect( screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }), ).toBeDisabled() expect(screen.getByText('dataset.newKnowledge.pagesAppearTitle')).toBeInTheDocument() - expect(screen.getByText('dataset.newKnowledge.usingDefaults')).toBeInTheDocument() const rootUrl = screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder') const sourceName = screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder') expect(rootUrl).toBeEnabled() @@ -1146,27 +1296,23 @@ describe('CreateKnowledgePage', () => { ) await user.keyboard('{Enter}') expect(serviceMock.create).not.toHaveBeenCalled() - await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlOptions' })) - await user.click(screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' })) - const maxPages = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' }) + await user.click(screen.getByRole('switch', { name: 'dataset.newKnowledge.includeSubpages' })) + const maxPages = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' }) await user.clear(maxPages) await user.type(maxPages, '25') - await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlOptions' })) - expect( - screen.getByText( - 'dataset.newKnowledge.includeSubpages: dataset.newKnowledge.booleanFalse · dataset.newKnowledge.maxPages: 25', - ), - ).toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })) expect(serviceMock.create).not.toHaveBeenCalled() expect(routerMock.replace).not.toHaveBeenCalled() - expect(serviceMock.createCrawl).toHaveBeenCalledWith({ - options: expect.objectContaining({ - crawl_sub_pages: false, - limit: 25, + expect(serviceMock.startWebsitePreview).toHaveBeenCalledWith({ + body: expect.objectContaining({ + kind: 'website_crawl', + parameters: expect.objectContaining({ + crawl_subpages: false, + limit: 25, + url: 'https://docs.dify.ai', + }), }), - url: 'https://docs.dify.ai', }) expect(await screen.findByText('Getting started')).toBeInTheDocument() expect( @@ -1187,17 +1333,46 @@ describe('CreateKnowledgePage', () => { expect(screen.getByText('Getting started')).toBeInTheDocument() }) - it('creates an initial source from a synchronous Jina Reader preview', async () => { + it('lets users edit website parameters after a successful preview', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + renderPage() + await fillRequiredFields(user) + const rootUrl = screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder') + await user.type(rootUrl, 'https://docs.dify.ai') + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Dify docs', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })) + await screen.findByText('Getting started') + + expect(rootUrl).toBeEnabled() + await user.clear(rootUrl) + + expect(screen.queryByText('Getting started')).not.toBeInTheDocument() + expect( + screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }), + ).toBeDisabled() + }) + + it('creates an initial source from a Jina Reader preview job', async () => { const user = userEvent.setup() navigationMock.startMode = 'source' datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, jinaDatasourcePlugin] datasourceQueryMock.auth.data = { result: [firecrawlDatasourceAuth, jinaDatasourceAuth] } - serviceMock.createCrawl.mockResolvedValueOnce({ - data: { - content: '# Dify introduction', - description: 'Introduction', - title: 'Dify introduction', - url: 'https://docs.dify.ai/introduction', + serviceMock.getWebsitePreview.mockResolvedValueOnce({ + job_id: 'website-preview-1', + status: 'completed', + result: { + kind: 'website_crawl', + pages: [ + { + description: 'Introduction', + source_url: 'https://docs.dify.ai/introduction', + title: 'Dify introduction', + }, + ], }, }) renderPage() @@ -1235,10 +1410,71 @@ describe('CreateKnowledgePage', () => { }) }) - it('shows and can stop an ongoing website crawl', async () => { + it('previews and persists a declaration-driven website datasource without a root URL', async () => { const user = userEvent.setup() navigationMock.startMode = 'source' - serviceMock.getCrawlStatus.mockImplementation(() => new Promise(() => {})) + datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, tavilyDatasourcePlugin] + datasourceQueryMock.auth.data = { result: [firecrawlDatasourceAuth, tavilyDatasourceAuth] } + serviceMock.getWebsitePreview.mockResolvedValueOnce({ + job_id: 'website-preview-1', + status: 'completed', + result: { + kind: 'website_crawl', + pages: [ + { + description: 'Tavily result', + source_url: 'https://example.com/result', + title: 'Tavily result', + }, + ], + }, + }) + renderPage() + await fillRequiredFields(user) + await user.click(screen.getByRole('radio', { name: 'Tavily' })) + await user.type(screen.getByRole('textbox', { name: /Search query/ }), 'agentic RAG') + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Tavily research', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })) + + expect(serviceMock.startWebsitePreview).toHaveBeenCalledWith({ + body: { + credentialId: 'tavily-credential-1', + datasource: 'search_extract', + kind: 'website_crawl', + parameters: { query: 'agentic RAG', search_depth: 'basic' }, + pluginId: 'langgenius/tavily_datasource', + provider: 'tavily', + providerDisplayName: 'Tavily', + }, + }) + await user.click(await screen.findByRole('checkbox', { name: 'Tavily result' })) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' })) + + await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce()) + expect(serviceMock.create.mock.calls[0]?.[0].body.initial_source).toEqual( + expect.objectContaining({ + parameters: { query: 'agentic RAG', search_depth: 'basic' }, + root_url: expect.stringMatching(/^datasource:\/\//), + }), + ) + }) + + it('shows and can stop an ongoing website crawl', async () => { + const user = userEvent.setup() + let resolveCancellation: ((value: { job_id: string; status: 'canceled' }) => void) | undefined + serviceMock.cancelWebsitePreview.mockReturnValue( + new Promise((resolve) => { + resolveCancellation = resolve + }), + ) + navigationMock.startMode = 'source' + serviceMock.getWebsitePreview.mockResolvedValue({ + job_id: 'website-preview-1', + status: 'running', + }) renderPage() await fillRequiredFields(user) await user.type( @@ -1259,6 +1495,54 @@ describe('CreateKnowledgePage', () => { expect(stopButton).toBeEnabled() await user.click(stopButton) + expect(serviceMock.cancelWebsitePreview).toHaveBeenCalledWith({ + params: { job_id: 'website-preview-1' }, + }) + expect(screen.getByRole('status')).toHaveTextContent('dataset.newKnowledge.crawlingPages') + + resolveCancellation?.({ job_id: 'website-preview-1', status: 'canceled' }) + await waitFor(() => + expect(screen.getByRole('status')).toHaveTextContent('dataset.newKnowledge.crawlStopped'), + ) + }) + + it('keeps the preview job available when stopping fails so cancellation can be retried', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + serviceMock.getWebsitePreview.mockResolvedValue({ + job_id: 'website-preview-1', + status: 'running', + }) + serviceMock.cancelWebsitePreview + .mockRejectedValueOnce(new Error('cancel response lost')) + .mockResolvedValueOnce({ + job_id: 'website-preview-1', + status: 'canceled', + }) + renderPage() + await fillRequiredFields(user) + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder'), + 'https://docs.dify.ai', + ) + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Dify docs', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })) + + const stopButton = await screen.findByRole('button', { + name: 'dataset.newKnowledge.stopCrawl', + }) + await user.click(stopButton) + await waitFor(() => expect(stopButton).toBeEnabled()) + + await user.click(stopButton) + + await waitFor(() => expect(serviceMock.cancelWebsitePreview).toHaveBeenCalledTimes(2)) + expect(serviceMock.cancelWebsitePreview).toHaveBeenNthCalledWith(2, { + params: { job_id: 'website-preview-1' }, + }) expect(screen.getByRole('status')).toHaveTextContent('dataset.newKnowledge.crawlStopped') }) @@ -1300,10 +1584,15 @@ describe('CreateKnowledgePage', () => { datasource: 'crawl', kind: 'website_crawl', name: 'Dify docs', + parameters: { + crawl_subpages: true, + limit: 100, + url: 'https://docs.dify.ai', + }, pluginId: 'langgenius/firecrawl_datasource', provider: 'firecrawl', providerDisplayName: 'Firecrawl', - root_url: 'https://docs.dify.ai', + root_url: 'https://docs.dify.ai/', selection: [ { source_url: 'https://docs.dify.ai/getting-started', @@ -1385,6 +1674,7 @@ describe('CreateKnowledgePage', () => { datasource: 'notion', kind: 'online_document', name: 'Notion handbook', + parameters: {}, pluginId: 'langgenius/notion_datasource', provider: 'notion', providerDisplayName: 'Notion', @@ -1404,6 +1694,88 @@ describe('CreateKnowledgePage', () => { }) }) + it('does not inject website defaults when switching connected document providers', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + datasourceQueryMock.plugins.data = [ + firecrawlDatasourcePlugin, + notionDatasourcePlugin, + outlineDatasourcePlugin, + ] + datasourceQueryMock.auth.data = { + result: [firecrawlDatasourceAuth, notionDatasourceAuth, outlineDatasourceAuth], + } + serviceMock.previewInitialSource.mockResolvedValue({ + documents: [], + files: [], + kind: 'online_document', + next_page_parameters: null, + }) + renderPage() + await fillRequiredFields(user) + await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' })) + await user.click(screen.getByRole('radio', { name: 'Outline' })) + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Outline handbook', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })) + + await waitFor(() => expect(serviceMock.previewInitialSource).toHaveBeenCalledOnce()) + expect(serviceMock.previewInitialSource).toHaveBeenCalledWith({ + body: expect.objectContaining({ + datasource: 'outline', + parameters: {}, + pluginId: 'langgenius/outline_datasource', + provider: 'outline', + }), + }) + }) + + it('resets a connected-source preview when datasource parameters change', async () => { + const user = userEvent.setup() + navigationMock.startMode = 'source' + datasourceQueryMock.plugins.data = [ + firecrawlDatasourcePlugin, + notionDatasourcePluginWithParameters, + ] + datasourceQueryMock.auth.data = { + result: [firecrawlDatasourceAuth, notionDatasourceAuth], + } + serviceMock.previewInitialSource.mockResolvedValue({ + documents: [ + { + last_edited_time: '2026-08-10T08:00:00Z', + name: 'Product handbook', + page_id: 'page-1', + provider_item_id: '["workspace-1","page-1"]', + type: 'page', + workspace_id: 'workspace-1', + workspace_name: 'Dify', + }, + ], + kind: 'online_document', + next_page_parameters: null, + }) + renderPage() + await fillRequiredFields(user) + await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' })) + await user.type(screen.getByRole('textbox', { name: 'Workspace' }), 'Product') + await user.type( + screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'), + 'Notion handbook', + ) + await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })) + await screen.findByRole('checkbox', { name: 'Product handbook' }) + + const workspace = screen.getByRole('textbox', { name: 'Workspace' }) + expect(workspace).toBeEnabled() + await user.clear(workspace) + + expect(screen.queryByRole('checkbox', { name: 'Product handbook' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })).toBeDisabled() + }) + it('creates a knowledge space atomically with a selected drive file', async () => { const user = userEvent.setup() navigationMock.startMode = 'source' @@ -1451,6 +1823,7 @@ describe('CreateKnowledgePage', () => { datasource: 'google_drive', kind: 'online_drive', name: 'Drive runbook', + parameters: {}, pluginId: 'langgenius/google_drive', provider: 'google_drive', providerDisplayName: 'Google Drive', diff --git a/web/features/new-rag/__tests__/datasource-parameter-form.spec.tsx b/web/features/new-rag/__tests__/datasource-parameter-form.spec.tsx new file mode 100644 index 00000000000..8c823f784a4 --- /dev/null +++ b/web/features/new-rag/__tests__/datasource-parameter-form.spec.tsx @@ -0,0 +1,136 @@ +import type { DatasourceParameters, DatasourceParameterSchema } from '../datasource-parameter-model' +import { screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { useState } from 'react' +import { render } from '@/test/console/render' +import { DatasourceParameterForm } from '../datasource-parameter-form' +import { + datasourceParameterDefaults, + websiteDatasourceParameterSchemas, +} from '../datasource-parameter-model' + +const limitSchema: DatasourceParameterSchema = { + label: { en_US: 'Limit' }, + max: 200, + min: 100, + name: 'limit', + options: [], + required: true, + type: 'number', +} + +const urlSchema: DatasourceParameterSchema = { + label: { en_US: 'Root URL' }, + name: 'url', + options: [], + required: true, + type: 'string', +} + +const decimalSchema: DatasourceParameterSchema = { + label: { en_US: 'Threshold' }, + name: 'threshold', + options: [], + precision: 2, + required: false, + type: 'number', +} + +function NumberParameterForm() { + const [parameters, setParameters] = useState({}) + return ( + + ) +} + +function UrlParameterForm() { + const [parameters, setParameters] = useState({}) + return ( + + ) +} + +function DecimalParameterForm() { + const [parameters, setParameters] = useState({}) + return ( + + ) +} + +function LegacyWebsiteParameterForm() { + const schemas = websiteDatasourceParameterSchemas({ parameters: [] } as never) + const [parameters, setParameters] = useState(() => + datasourceParameterDefaults(schemas), + ) + return ( + + ) +} + +describe('DatasourceParameterForm', () => { + it('keeps an out-of-range numeric value visible and reports it as invalid', async () => { + const user = userEvent.setup() + render() + + const input = screen.getByRole('spinbutton', { name: 'Limit' }) + await user.type(input, '15') + expect(input).toHaveValue(15) + + await user.tab() + expect(input).toHaveValue(15) + expect(input).toHaveAttribute('aria-invalid', 'true') + }) + + it('associates an actionable error with an invalid URL field', async () => { + const user = userEvent.setup() + render() + + const input = screen.getByRole('textbox', { name: 'Root URL' }) + await user.type(input, 'ftp://example.com') + await user.tab() + + expect(input).toHaveAttribute('aria-invalid', 'true') + expect(input).toHaveAccessibleDescription('dataset.newKnowledge.invalidRootUrl') + expect(screen.getByText('dataset.newKnowledge.invalidRootUrl')).toBeInTheDocument() + }) + + it('allows decimal input using the declaration precision', async () => { + const user = userEvent.setup() + render() + + const input = screen.getByRole('spinbutton', { name: 'Threshold' }) + expect(input).toHaveAttribute('inputmode', 'decimal') + expect(input).toHaveAttribute('step', '0.01') + + await user.type(input, '0.25') + expect(input).toHaveValue(0.25) + expect(input).not.toHaveAttribute('aria-invalid', 'true') + }) + + it('resolves translation keys for legacy website parameter labels', () => { + render() + + expect(screen.getByRole('textbox', { name: 'dataset.newKnowledge.rootUrl' })).toHaveAttribute( + 'placeholder', + 'dataset.newKnowledge.rootUrlPlaceholder', + ) + expect( + screen.getByRole('switch', { name: 'dataset.newKnowledge.includeSubpages' }), + ).toBeChecked() + expect(screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })).toHaveValue( + 100, + ) + }) +}) diff --git a/web/features/new-rag/__tests__/datasource-parameter-model.spec.ts b/web/features/new-rag/__tests__/datasource-parameter-model.spec.ts new file mode 100644 index 00000000000..89c83c0d6b0 --- /dev/null +++ b/web/features/new-rag/__tests__/datasource-parameter-model.spec.ts @@ -0,0 +1,163 @@ +import { + datasourceParameterDefaults, + datasourceParameterRecord, + datasourceParameterSchemas, + invalidDatasourceParameters, + missingRequiredDatasourceParameters, + websiteDatasourceParameterSchemas, + withDatasourceParameterDefaults, +} from '../datasource-parameter-model' + +describe('datasource parameter model', () => { + it('parses plugin-declared fields and applies defaults without replacing saved values', () => { + const schemas = datasourceParameterSchemas({ + parameters: [ + { + default: 'basic', + label: { en_US: 'Search depth' }, + name: 'search_depth', + options: [ + { label: { en_US: 'Basic' }, value: 'basic' }, + { label: { en_US: 'Advanced' }, value: 'advanced' }, + ], + required: true, + type: 'select', + }, + { + label: { en_US: 'Query' }, + name: 'query', + required: true, + type: 'string', + }, + ], + } as never) + + expect(datasourceParameterDefaults(schemas)).toEqual({ search_depth: 'basic' }) + expect(withDatasourceParameterDefaults(schemas, { search_depth: 'advanced' })).toEqual({ + search_depth: 'advanced', + }) + expect(missingRequiredDatasourceParameters(schemas, { search_depth: 'advanced' })).toEqual([ + expect.objectContaining({ name: 'query' }), + ]) + }) + + it('validates URL, numeric, and select constraints from a declaration', () => { + const schemas = datasourceParameterSchemas({ + parameters: [ + { label: { en_US: 'URL' }, name: 'url', required: true, type: 'string' }, + { label: { en_US: 'Limit' }, max: 200, min: 1, name: 'limit', type: 'integer' }, + { + label: { en_US: 'Mode' }, + name: 'mode', + options: [{ label: { en_US: 'One' }, value: 'one' }], + type: 'select', + }, + ], + } as never) + + expect( + invalidDatasourceParameters(schemas, { + limit: 201, + mode: 'unknown', + url: 'ftp://example.com', + }).map((schema) => schema.name), + ).toEqual(['url', 'limit', 'mode']) + expect( + invalidDatasourceParameters(schemas, { + limit: 1.5, + mode: 'one', + url: 'https://example.com', + }).map((schema) => schema.name), + ).toEqual(['limit']) + expect( + invalidDatasourceParameters(schemas, { + limit: 25, + mode: 'one', + url: 'https://example.com', + }), + ).toEqual([]) + }) + + it('preserves and validates decimal precision for numeric parameters', () => { + const schemas = datasourceParameterSchemas({ + parameters: [ + { label: { en_US: 'Threshold' }, name: 'threshold', precision: 2, type: 'number' }, + ], + } as never) + + expect(schemas[0]).toEqual(expect.objectContaining({ precision: 2, type: 'number' })) + expect(invalidDatasourceParameters(schemas, { threshold: 0.25 })).toEqual([]) + expect(invalidDatasourceParameters(schemas, { threshold: 0.251 })).toEqual([ + expect.objectContaining({ name: 'threshold' }), + ]) + }) + + it('keeps legacy website providers functional when their declaration has no fields', () => { + const schemas = websiteDatasourceParameterSchemas({ parameters: [] } as never) + + expect(schemas).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'url', required: true }), + expect.objectContaining({ defaultValue: true, name: 'crawl_subpages' }), + expect.objectContaining({ defaultValue: 100, name: 'limit' }), + ]), + ) + expect(schemas[0]).toEqual( + expect.objectContaining({ + label: {}, + labelTranslationKey: 'newKnowledge.rootUrl', + placeholder: {}, + placeholderTranslationKey: 'newKnowledge.rootUrlPlaceholder', + }), + ) + }) + + it('uses the provider-specific legacy subpage parameter name', () => { + expect( + websiteDatasourceParameterSchemas({ + identity: { name: 'jina_reader', provider: 'jinareader' }, + parameters: [], + } as never), + ).toEqual(expect.arrayContaining([expect.objectContaining({ name: 'crawl_sub_pages' })])) + }) + + it('retains required unsupported parameters so the datasource cannot run without them', () => { + const schemas = datasourceParameterSchemas({ + parameters: [ + { + label: { en_US: 'Private token' }, + name: 'private_token', + required: true, + type: 'secret-input', + }, + { + label: { en_US: 'Attachment' }, + name: 'attachment', + required: false, + type: 'file', + }, + ], + } as never) + + expect(schemas).toEqual([ + expect.objectContaining({ name: 'private_token', type: 'unsupported' }), + expect.objectContaining({ name: 'attachment', type: 'unsupported' }), + ]) + expect(missingRequiredDatasourceParameters(schemas, {})).toEqual([ + expect.objectContaining({ name: 'private_token' }), + ]) + expect(invalidDatasourceParameters(schemas, { attachment: 'file-1' })).toEqual([ + expect.objectContaining({ name: 'attachment' }), + ]) + }) + + it('accepts only bounded scalar draft parameters', () => { + expect(datasourceParameterRecord({ enabled: true, limit: 5, query: 'dify' })).toEqual({ + enabled: true, + limit: 5, + query: 'dify', + }) + expect(datasourceParameterRecord({ nested: { token: 'no' } })).toBeUndefined() + expect(datasourceParameterRecord({ value: Number.NaN })).toBeUndefined() + }) +}) diff --git a/web/features/new-rag/__tests__/routes.spec.ts b/web/features/new-rag/__tests__/routes.spec.ts index e4856fd909f..63d72063074 100644 --- a/web/features/new-rag/__tests__/routes.spec.ts +++ b/web/features/new-rag/__tests__/routes.spec.ts @@ -86,6 +86,7 @@ describe('New RAG routes', () => { ).toEqual({ includeSubpages: false, maxPages: 25, + parameters: {}, provider: 'Firecrawl', rootUrl: 'https://docs.dify.ai', sourceName: 'Dify docs', diff --git a/web/features/new-rag/__tests__/website-crawl-preview.spec.tsx b/web/features/new-rag/__tests__/website-crawl-preview.spec.tsx index 14cfa66e073..77a9e268589 100644 --- a/web/features/new-rag/__tests__/website-crawl-preview.spec.tsx +++ b/web/features/new-rag/__tests__/website-crawl-preview.spec.tsx @@ -278,6 +278,12 @@ describe('WebsiteCrawlPreview', () => { metadata: { clientRequestId: expect.any(String), crawlOptions: { includeSubpages: true, limit: 100 }, + datasourceParameterMode: 'exact', + parameters: { + crawl_subpages: true, + limit: 100, + url: 'https://docs.dify.ai', + }, preview: true, providerId: 'plugin-daemon-website', providerName: 'Firecrawl', @@ -523,12 +529,11 @@ describe('WebsiteCrawlPreview', () => { async (invalidLimit) => { render() const user = await fillValidForm() - await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ })) - const pageLimit = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' }) + const pageLimit = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' }) await user.clear(pageLimit) await user.type(pageLimit, invalidLimit) - expect(pageLimit).toHaveValue(invalidLimit) + expect(pageLimit).toHaveValue(Number(invalidLimit)) expect(pageLimit).toHaveAttribute('aria-invalid', 'true') expect(pageLimit).toHaveAccessibleDescription('dataset.newKnowledge.maxPages: 1–200') expect( @@ -542,11 +547,10 @@ describe('WebsiteCrawlPreview', () => { it.each([1, 200])('submits the exact valid page limit %s', async (validLimit) => { render() const user = await fillValidForm() - await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ })) - const pageLimit = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' }) + const pageLimit = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' }) await user.clear(pageLimit) await user.type(pageLimit, String(validLimit)) - expect(pageLimit).toHaveValue(String(validLimit)) + expect(pageLimit).toHaveValue(validLimit) await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })) await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce()) @@ -558,17 +562,10 @@ describe('WebsiteCrawlPreview', () => { it('preserves a replacement crawl page limit after clearing the input', async () => { render() const user = await fillValidForm() - await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ })) - const pageLimit = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' }) + const pageLimit = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' }) await user.clear(pageLimit) await user.type(pageLimit, '50') - expect(pageLimit).toHaveValue('50') - await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ })) - expect( - screen.getByText( - 'dataset.newKnowledge.includeSubpages: dataset.newKnowledge.booleanTrue · dataset.newKnowledge.maxPages: 50', - ), - ).toBeInTheDocument() + expect(pageLimit).toHaveValue(50) await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' })) await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce()) diff --git a/web/features/new-rag/add-source-page.tsx b/web/features/new-rag/add-source-page.tsx index 88f34c443ce..c94fc5609c5 100644 --- a/web/features/new-rag/add-source-page.tsx +++ b/web/features/new-rag/add-source-page.tsx @@ -28,6 +28,10 @@ import { consoleClient, consoleQuery } from '@/service/client' import { useDataSourceList } from '@/service/use-pipeline' import { AddSourceExitDialog } from './components/add-source-exit-dialog' import { ConnectedSourceSetup } from './connected-source-setup' +import { + datasourceParameterDefaults, + websiteDatasourceParameterSchemas, +} from './datasource-parameter-model' import { createNewKnowledgeSourceDraft, newKnowledgeDetailPath, @@ -1276,8 +1280,14 @@ export function AddSourcePage({ if (!nextProvider) return updateSourceDraft({ ...sourceDraft, + parameters: nextProvider.installed + ? datasourceParameterDefaults( + websiteDatasourceParameterSchemas(nextProvider.datasource), + ) + : {}, provider: nextProvider.label, providerKey: nextProvider.key, + rootUrl: '', }) }} /> @@ -1325,13 +1335,14 @@ export function AddSourcePage({ ) : activeConnection && websitePreviewReady ? ( ) : activeConnection ? (
diff --git a/web/features/new-rag/connected-source-setup.tsx b/web/features/new-rag/connected-source-setup.tsx index b8fd07821ca..176bbcc0cad 100644 --- a/web/features/new-rag/connected-source-setup.tsx +++ b/web/features/new-rag/connected-source-setup.tsx @@ -4,6 +4,7 @@ import type { KnowledgeFsSourceFileResponse, KnowledgeFsSourcePageResponse, } from '@dify/contracts/api/console/knowledge-fs/types.gen' +import type { DatasourceParameters, DatasourceParameterSchema } from './datasource-parameter-model' import type { NewKnowledgeOnlineDocumentsSourceDraft, NewKnowledgeOnlineDriveSourceDraft, @@ -27,6 +28,14 @@ import { consoleClient, consoleQuery } from '@/service/client' import { useGetDataSourceListAuth } from '@/service/use-datasource' import { useDataSourceList } from '@/service/use-pipeline' import { formatFileSize } from '@/utils/format' +import { DatasourceParameterForm } from './datasource-parameter-form' +import { + datasourceParameterDefaults, + datasourceParameterSchemas, + invalidDatasourceParameters, + missingRequiredDatasourceParameters, + withDatasourceParameterDefaults, +} from './datasource-parameter-model' import { createRequestId } from './request-id' import { NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH } from './routes' import { @@ -252,13 +261,16 @@ function sourceType() { function sourceMetadata( draft: ConnectedSourceDraft, + parameters: DatasourceParameters, provider: SourceProvider, clientRequestId: string, preview: boolean, ) { return { clientRequestId, + datasourceParameterMode: 'exact', preview, + parameters, providerId: provider.id, providerKind: usesDriveTransport(draft) ? 'online-drive' : 'online-document', providerName: draft.provider, @@ -616,6 +628,7 @@ function ResourceConfiguration({ onDraftChange, onDirtyChange, onExit, + parameters, provider, providerRegion, }: { @@ -626,6 +639,7 @@ function ResourceConfiguration({ onDraftChange: (draft: NewKnowledgeSourceDraft) => void onDirtyChange: (dirty: boolean) => void onExit: () => void + parameters: DatasourceParameters provider: SourceProvider providerRegion?: string }) { @@ -690,6 +704,8 @@ function ResourceConfiguration({ connectionId: connection.id, metadata: { clientRequestId, + datasourceParameterMode: 'exact', + parameters, preview: true, providerId: provider.id, providerKind: driveTransport ? 'online-drive' : 'online-document', @@ -732,6 +748,7 @@ function ResourceConfiguration({ draft.sourceType, driveTransport, knowledgeSpaceId, + parameters, previewUri, provider.id, ]) @@ -1271,7 +1288,13 @@ function ResourceConfiguration({ await consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.patch({ body: { expectedVersion: previewSource.version, - metadata: sourceMetadata(draft, provider, previewRequestIdRef.current, false), + metadata: sourceMetadata( + draft, + parameters, + provider, + previewRequestIdRef.current, + false, + ), name: draft.sourceName.trim(), status: 'active', }, @@ -1510,6 +1533,81 @@ function ResourceConfiguration({ ) } +function AppliedResourceConfiguration({ + connection, + draft, + knowledgeSpaceId, + onCompleted, + onDraftChange, + onDirtyChange, + onExit, + parameters, + parametersValid, + parameterSchemas, + provider, + providerRegion, +}: { + connection: SourceConnection + draft: ConnectedSourceDraft + knowledgeSpaceId: string + onCompleted: () => void + onDraftChange: (draft: NewKnowledgeSourceDraft) => void + onDirtyChange: (dirty: boolean) => void + onExit: () => void + parameters: DatasourceParameters + parametersValid: boolean + parameterSchemas: DatasourceParameterSchema[] + provider: SourceProvider + providerRegion?: string +}) { + const { t } = useTranslation('dataset') + const [appliedParameters, setAppliedParameters] = useState( + () => (parametersValid ? parameters : undefined), + ) + const parametersApplied = + appliedParameters !== undefined && + JSON.stringify(appliedParameters) === JSON.stringify(parameters) + + return ( + <> + + onDraftChange({ + ...draft, + parameters: nextParameters, + }) + } + /> + {parametersValid && !parametersApplied && ( + + )} + {parametersApplied && appliedParameters && ( + + )} + + ) +} + export function ConnectedSourceSetup({ draft, knowledgeSpaceId, @@ -1558,6 +1656,18 @@ export function ConnectedSourceSetup({ ) const providerOption = sourceProviderOptionForDraft(providerOptions, draft) const installedProviderOption = providerOption?.installed ? providerOption : undefined + const parameterSchemas = useMemo( + () => + installedProviderOption ? datasourceParameterSchemas(installedProviderOption.datasource) : [], + [installedProviderOption], + ) + const parameters = useMemo( + () => withDatasourceParameterDefaults(parameterSchemas, draft.parameters), + [draft.parameters, parameterSchemas], + ) + const parametersValid = + !missingRequiredDatasourceParameters(parameterSchemas, parameters).length && + !invalidDatasourceParameters(parameterSchemas, parameters).length const provider = providerForDraft(providersQuery.data ?? [], draft, providerOption) const driveTransport = usesDriveTransport(draft) const datasourceProvider = datasourceProviderForOption(installedProviderOption) @@ -1725,6 +1835,9 @@ export function ConnectedSourceSetup({ if (draft.sourceType === 'onlineDocuments') { onDraftChange({ ...draft, + parameters: nextProvider.installed + ? datasourceParameterDefaults(datasourceParameterSchemas(nextProvider.datasource)) + : {}, provider: nextProvider.label, providerKey: nextProvider.key, sourceName: '', @@ -1733,6 +1846,9 @@ export function ConnectedSourceSetup({ } onDraftChange({ ...draft, + parameters: nextProvider.installed + ? datasourceParameterDefaults(datasourceParameterSchemas(nextProvider.datasource)) + : {}, provider: nextProvider.label, providerKey: nextProvider.key, sourceName: '', @@ -1806,7 +1922,7 @@ export function ConnectedSourceSetup({

) : connection?.status === 'active' ? ( - diff --git a/web/features/new-rag/crawl-selection-form.tsx b/web/features/new-rag/crawl-selection-form.tsx index 7d924c1f23b..92b0fccd323 100644 --- a/web/features/new-rag/crawl-selection-form.tsx +++ b/web/features/new-rag/crawl-selection-form.tsx @@ -108,9 +108,8 @@ async function waitForImportTerminal( throw new Error('Source import did not reach a terminal state') } -function pageSkipReason(page: PreviewPage, rootUrl: string): PageSkipReason | undefined { +function pageSkipReason(page: PreviewPage, rootUrl?: string): PageSkipReason | undefined { try { - const root = new URL(rootUrl) const candidate = new URL(page.sourceUrl) if ( !['http:', 'https:'].includes(candidate.protocol) || @@ -118,6 +117,8 @@ function pageSkipReason(page: PreviewPage, rootUrl: string): PageSkipReason | un candidate.password ) return 'failed' + if (!rootUrl) return undefined + const root = new URL(rootUrl) if (candidate.hostname.toLocaleLowerCase() !== root.hostname.toLocaleLowerCase()) return 'off-domain' return undefined @@ -186,6 +187,7 @@ export function CrawlPreviewPageSelection({ progressFailed = 0, recrawlDisabled, rootUrl, + sourceLabel, selectedPageIds, }: { busy?: boolean @@ -195,7 +197,8 @@ export function CrawlPreviewPageSelection({ pages: PreviewPage[] progressFailed?: number recrawlDisabled?: boolean - rootUrl: string + rootUrl?: string + sourceLabel?: string selectedPageIds: Set }) { const { t } = useTranslation('dataset') @@ -251,7 +254,7 @@ export function CrawlPreviewPageSelection({ > {t(($) => $['newKnowledge.pagesCrawled'], { count: pages.length, - host: new URL(rootUrl).host, + host: sourceLabel ?? (rootUrl ? new URL(rootUrl).host : ''), })} @@ -372,7 +375,7 @@ function ReadyCrawlSelectionForm({ onWorkflowRun: (run: SourceWorkflowRun) => void pages: PreviewPage[] policy: SyncPolicy - rootUrl: string + rootUrl?: string run: SourceWorkflowRun source: Source workflowUncertain: boolean @@ -765,7 +768,7 @@ export function CrawlSelectionForm({ onWorkflowPending: (request: Promise) => void onWorkflowRun: (run: SourceWorkflowRun) => void pages: PreviewPage[] - rootUrl: string + rootUrl?: string run: SourceWorkflowRun source: Source workflowUncertain?: boolean diff --git a/web/features/new-rag/create-connected-source-setup.tsx b/web/features/new-rag/create-connected-source-setup.tsx index e88d0159e98..3c24b35cd74 100644 --- a/web/features/new-rag/create-connected-source-setup.tsx +++ b/web/features/new-rag/create-connected-source-setup.tsx @@ -18,6 +18,13 @@ import { useCallback, useEffect, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import Loading from '@/app/components/base/loading' import { consoleClient } from '@/service/client' +import { DatasourceParameterForm } from './datasource-parameter-form' +import { + datasourceParameterSchemas, + invalidDatasourceParameters, + missingRequiredDatasourceParameters, + withDatasourceParameterDefaults, +} from './datasource-parameter-model' import { SourceNameField, SourceSyncPolicyField } from './source-setup-fields' type ConnectedDraft = NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft @@ -88,6 +95,17 @@ export function CreateConnectedSourceSetup({ () => new Map(), ) const driveTransport = providerOption.providerType === 'online_drive' + const parameterSchemas = useMemo( + () => datasourceParameterSchemas(providerOption.datasource), + [providerOption.datasource], + ) + const parameters = useMemo( + () => withDatasourceParameterDefaults(parameterSchemas, draft.parameters), + [draft.parameters, parameterSchemas], + ) + const parametersValid = + !missingRequiredDatasourceParameters(parameterSchemas, parameters).length && + !invalidDatasourceParameters(parameterSchemas, parameters).length const selectionAtLimit = selected.size >= MAX_SELECTION const selectableResources = useMemo( () => @@ -121,6 +139,7 @@ export function CreateConnectedSourceSetup({ pluginId: providerOption.plugin.plugin_id, provider: providerOption.plugin.provider, providerDisplayName: providerOption.label, + parameters, } if (!driveTransport) { onInitialSourceChange({ @@ -173,6 +192,7 @@ export function CreateConnectedSourceSetup({ providerOption.label, providerOption.plugin.plugin_id, providerOption.plugin.provider, + parameters, selectableResources, selected, ]) @@ -193,6 +213,7 @@ export function CreateConnectedSourceSetup({ parentKey?: string prefix?: string } = {}) => { + if (!parametersValid) return append ? setLoadingMore(true) : setLoading(true) setError(false) try { @@ -202,6 +223,7 @@ export function CreateConnectedSourceSetup({ datasource: providerOption.datasource.identity.name, kind: driveTransport ? 'online_drive' : 'online_document', parameters: { + ...parameters, ...(bucket ? { bucket } : {}), ...(prefix ? { prefix } : {}), ...(nextPage ? { next_page_parameters: nextPage } : {}), @@ -255,7 +277,7 @@ export function CreateConnectedSourceSetup({ setLoadingMore(false) } }, - [credential.id, driveTransport, providerOption], + [credential.id, driveTransport, parameters, parametersValid, providerOption], ) const toggle = (key: string) => { @@ -321,13 +343,26 @@ export function CreateConnectedSourceSetup({ onDraftChange={onDraftChange} /> + { + setResources([]) + setSelected(new Set()) + setExpanded(new Set()) + setNextPageRequests(new Map()) + setPreviewed(false) + onDraftChange({ ...draft, parameters: nextParameters }) + }} + /> {!previewed && ( @@ -664,8 +614,9 @@ export function CreateSourceSetup({ onRecrawl={() => void startPreview()} onSelectionChange={updateSelectedPageIds} pages={selectionPages} - rootUrl={previewRootUrl} + rootUrl={selectionRootUrl} selectedPageIds={selectedPageIds} + sourceLabel={installedProviderOption.label} /> )} diff --git a/web/features/new-rag/datasource-parameter-form.tsx b/web/features/new-rag/datasource-parameter-form.tsx new file mode 100644 index 00000000000..365fd72198c --- /dev/null +++ b/web/features/new-rag/datasource-parameter-form.tsx @@ -0,0 +1,263 @@ +'use client' + +import type { + DatasourceParameters, + DatasourceParameterSchema, + DatasourceParameterValue, +} from './datasource-parameter-model' +import { + Field, + FieldControl, + FieldDescription, + FieldError, + FieldLabel, +} from '@langgenius/dify-ui/field' +import { + Select, + SelectContent, + SelectItem, + SelectItemIndicator, + SelectItemText, + SelectLabel, + SelectTrigger, +} from '@langgenius/dify-ui/select' +import { Switch } from '@langgenius/dify-ui/switch' +import { useId, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { invalidDatasourceParameters, localizedDatasourceText } from './datasource-parameter-model' + +function ParameterLabel({ label, required }: { label: string; required: boolean }) { + return ( + <> + {label} + {required && ( + + * + + )} + + ) +} + +function DatasourceParameterField({ + disabled, + parameter, + value, + onChange, +}: { + disabled: boolean + parameter: DatasourceParameterSchema + value: DatasourceParameterValue | undefined + onChange: (value: DatasourceParameterValue | undefined) => void +}) { + const { i18n, t } = useTranslation('dataset') + const generatedId = useId() + const [numberDraft, setNumberDraft] = useState() + const description = localizedDatasourceText(parameter.description, i18n.language, '') + const descriptionId = description ? `${generatedId}-description` : undefined + const errorId = `${generatedId}-error` + const label = + parameter.labelTranslationKey === 'newKnowledge.rootUrl' + ? t(($) => $['newKnowledge.rootUrl']) + : parameter.labelTranslationKey === 'newKnowledge.includeSubpages' + ? t(($) => $['newKnowledge.includeSubpages']) + : parameter.labelTranslationKey === 'newKnowledge.maxPages' + ? t(($) => $['newKnowledge.maxPages']) + : localizedDatasourceText(parameter.label, i18n.language, parameter.name) + const placeholder = parameter.placeholderTranslationKey + ? t(($) => $['newKnowledge.rootUrlPlaceholder']) + : localizedDatasourceText(parameter.placeholder, i18n.language, '') + const invalid = Boolean( + (parameter.required && (value === undefined || (typeof value === 'string' && !value.trim()))) || + invalidDatasourceParameters([parameter], { + ...(value === undefined ? {} : { [parameter.name]: value }), + }).length, + ) + const error = invalid + ? parameter.name === 'url' && typeof value === 'string' && value + ? t(($) => $['newKnowledge.invalidRootUrl']) + : parameter.labelTranslationKey === 'newKnowledge.maxPages' && + parameter.min !== undefined && + parameter.max !== undefined + ? `${label}: ${parameter.min}–${parameter.max}` + : t(($) => $['newKnowledge.invalidDatasourceParameter'], { parameter: label }) + : undefined + const describedBy = + [descriptionId, error ? errorId : undefined].filter(Boolean).join(' ') || undefined + const numberStep = parameter.integer + ? 1 + : parameter.precision === undefined + ? 'any' + : 10 ** -parameter.precision + + if (parameter.type === 'unsupported') + return ( + + + + + {t(($) => $['newKnowledge.providerUnavailable'])} + + ) + + if (parameter.type === 'boolean') + return ( + +
+ + + + onChange(checked)} + /> +
+ {description && {description}} + {error && ( + + {error} + + )} + + ) + + if (parameter.type === 'select') { + const selectedValue = typeof value === 'string' ? value : null + return ( + + + name={parameter.name} + disabled={disabled} + required={parameter.required} + value={selectedValue} + onValueChange={(nextValue) => onChange(nextValue ?? undefined)} + > + + + + + {parameter.options.find((option) => option.value === selectedValue) + ? localizedDatasourceText( + parameter.options.find((option) => option.value === selectedValue)?.label, + i18n.language, + selectedValue ?? '—', + ) + : '—'} + + + {!parameter.required && ( + + — + + + )} + {parameter.options.map((option) => ( + + + {localizedDatasourceText(option.label, i18n.language, option.value)} + + + + ))} + + + {description && {description}} + {error && ( + + {error} + + )} + + ) + } + + return ( + + + + + { + if (parameter.type === 'number') { + setNumberDraft(nextValue) + const number = Number(nextValue) + onChange(nextValue && Number.isFinite(number) ? number : undefined) + return + } + onChange(nextValue || undefined) + }} + onBlur={() => { + if (parameter.type !== 'number' || numberDraft === undefined) return + setNumberDraft(undefined) + }} + /> + {description && {description}} + {error && ( + + {error} + + )} + + ) +} + +export function DatasourceParameterForm({ + disabled = false, + parameters, + schemas, + onChange, +}: { + disabled?: boolean + parameters: DatasourceParameters + schemas: DatasourceParameterSchema[] + onChange: (parameters: DatasourceParameters) => void +}) { + if (!schemas.length) return null + + return ( +
+ {schemas.map((parameter) => ( + { + const next = { ...parameters } + if (value === undefined) delete next[parameter.name] + else next[parameter.name] = value + onChange(next) + }} + /> + ))} +
+ ) +} diff --git a/web/features/new-rag/datasource-parameter-model.ts b/web/features/new-rag/datasource-parameter-model.ts new file mode 100644 index 00000000000..5727d368cb5 --- /dev/null +++ b/web/features/new-rag/datasource-parameter-model.ts @@ -0,0 +1,269 @@ +import type { DataSourceItem } from '@/app/components/workflow/block-selector/types' + +type Datasource = DataSourceItem['declaration']['datasources'][number] + +const LEGACY_WEBSITE_PARAMETER_SCHEMAS: DatasourceParameterSchema[] = [ + { + label: {}, + labelTranslationKey: 'newKnowledge.rootUrl', + name: 'url', + options: [], + placeholder: {}, + placeholderTranslationKey: 'newKnowledge.rootUrlPlaceholder', + required: true, + type: 'string', + }, + { + defaultValue: true, + label: {}, + labelTranslationKey: 'newKnowledge.includeSubpages', + name: 'crawl_subpages', + options: [], + required: false, + type: 'boolean', + }, + { + defaultValue: 100, + integer: true, + label: {}, + labelTranslationKey: 'newKnowledge.maxPages', + max: 200, + min: 1, + name: 'limit', + options: [], + required: false, + type: 'number', + }, +] + +export type DatasourceParameterValue = boolean | number | string +export type DatasourceParameters = Record + +export type DatasourceParameterOption = { + label: Record + value: string +} + +export type DatasourceParameterSchema = { + defaultValue?: DatasourceParameterValue + description?: Record + integer?: boolean + label: Record + labelTranslationKey?: + | 'newKnowledge.includeSubpages' + | 'newKnowledge.maxPages' + | 'newKnowledge.rootUrl' + max?: number + min?: number + name: string + options: DatasourceParameterOption[] + placeholder?: Record + placeholderTranslationKey?: 'newKnowledge.rootUrlPlaceholder' + precision?: number + required: boolean + type: 'boolean' | 'number' | 'select' | 'string' | 'unsupported' +} + +function record(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +function localizedText(value: unknown): Record { + const candidate = record(value) + if (!candidate) return {} + return Object.fromEntries( + Object.entries(candidate).flatMap(([key, text]) => + typeof text === 'string' ? [[key, text] as const] : [], + ), + ) +} + +function finiteNumber(value: unknown) { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +function boundedPrecision(value: unknown) { + return Number.isSafeInteger(value) && Number(value) >= 0 && Number(value) <= 20 + ? Number(value) + : undefined +} + +function parameterType(value: unknown): DatasourceParameterSchema['type'] | undefined { + if (value === 'boolean') return 'boolean' + if (value === 'number' || value === 'integer') return 'number' + if (value === 'select') return 'select' + if (value === 'string' || value === 'text-input') return 'string' + if (['file', 'files', 'secret-input', 'system-files'].includes(String(value))) + return 'unsupported' +} + +function defaultValue( + value: unknown, + type: DatasourceParameterSchema['type'], +): DatasourceParameterValue | undefined { + if (type === 'boolean' && typeof value === 'boolean') return value + if (type === 'number' && typeof value === 'number' && Number.isFinite(value)) return value + if ((type === 'select' || type === 'string') && typeof value === 'string') return value +} + +export function datasourceParameterSchemas(datasource: Datasource): DatasourceParameterSchema[] { + return datasource.parameters.flatMap((rawParameter) => { + const parameter = record(rawParameter) + if (!parameter || typeof parameter.name !== 'string' || !parameter.name.trim()) return [] + const type = parameterType(parameter.type) + if (!type) return [] + const options = Array.isArray(parameter.options) + ? parameter.options.flatMap((rawOption) => { + const option = record(rawOption) + if (!option || typeof option.value !== 'string') return [] + return [ + { + label: localizedText(option.label), + value: option.value, + }, + ] + }) + : [] + return [ + { + ...(defaultValue(parameter.default, type) === undefined + ? {} + : { defaultValue: defaultValue(parameter.default, type) }), + description: localizedText(parameter.description), + ...(parameter.type === 'integer' ? { integer: true } : {}), + label: localizedText(parameter.label), + ...(finiteNumber(parameter.max) === undefined ? {} : { max: finiteNumber(parameter.max) }), + ...(finiteNumber(parameter.min) === undefined ? {} : { min: finiteNumber(parameter.min) }), + name: parameter.name.trim(), + options, + placeholder: localizedText(parameter.placeholder), + ...(boundedPrecision(parameter.precision) === undefined + ? {} + : { precision: boundedPrecision(parameter.precision) }), + required: parameter.required === true, + type, + }, + ] + }) +} + +export function websiteDatasourceParameterSchemas(datasource?: Datasource) { + const schemas = datasource ? datasourceParameterSchemas(datasource) : [] + if (schemas.length) return schemas + const providerIdentity = [datasource?.identity?.name, datasource?.identity?.provider] + .filter(Boolean) + .join(' ') + .toLocaleLowerCase() + const crawlSubpagesName = /jina|watercrawl/.test(providerIdentity) + ? 'crawl_sub_pages' + : 'crawl_subpages' + return LEGACY_WEBSITE_PARAMETER_SCHEMAS.map((schema) => + schema.name === 'crawl_subpages' ? { ...schema, name: crawlSubpagesName } : schema, + ) +} + +export function datasourceIncludeSubpages(parameters: DatasourceParameters, fallback = true) { + const value = parameters.crawl_subpages ?? parameters.crawl_sub_pages + return typeof value === 'boolean' ? value : fallback +} + +export function datasourceParameterDefaults(schemas: DatasourceParameterSchema[]) { + return Object.fromEntries( + schemas.flatMap((schema) => + schema.defaultValue === undefined ? [] : ([[schema.name, schema.defaultValue]] as const), + ), + ) satisfies DatasourceParameters +} + +export function withDatasourceParameterDefaults( + schemas: DatasourceParameterSchema[], + parameters: DatasourceParameters | undefined, +) { + return { + ...datasourceParameterDefaults(schemas), + ...parameters, + } +} + +export function missingRequiredDatasourceParameters( + schemas: DatasourceParameterSchema[], + parameters: DatasourceParameters, +) { + return schemas.filter((schema) => { + if (!schema.required) return false + const value = parameters[schema.name] + if (schema.type === 'unsupported') return true + return value === undefined || (typeof value === 'string' && !value.trim()) + }) +} + +export function invalidDatasourceParameters( + schemas: DatasourceParameterSchema[], + parameters: DatasourceParameters, +) { + return schemas.filter((schema) => { + const value = parameters[schema.name] + if (value === undefined || value === '') return false + if (schema.type === 'unsupported') return true + if (schema.type === 'boolean') return typeof value !== 'boolean' + if (schema.type === 'number') + return ( + typeof value !== 'number' || + !Number.isFinite(value) || + (schema.integer === true && !Number.isInteger(value)) || + (schema.precision !== undefined && + Math.abs(value - Number(value.toFixed(schema.precision))) > + Number.EPSILON * Math.max(1, Math.abs(value))) || + (schema.min !== undefined && value < schema.min) || + (schema.max !== undefined && value > schema.max) + ) + if (typeof value !== 'string') return true + if (schema.type === 'select') + return ( + Boolean(schema.options.length) && !schema.options.some((option) => option.value === value) + ) + if (schema.name !== 'url') return false + try { + const parsed = new URL(value) + return ( + !['http:', 'https:'].includes(parsed.protocol) || + Boolean(parsed.username || parsed.password) + ) + } catch { + return true + } + }) +} + +export function datasourceParameterRecord(value: unknown): DatasourceParameters | undefined { + const candidate = record(value) + if (!candidate || Object.keys(candidate).length > 50) return undefined + const entries: Array<[string, DatasourceParameterValue]> = [] + for (const [key, parameter] of Object.entries(candidate)) { + if (!key || key.length > 255) return undefined + if ( + typeof parameter !== 'string' && + typeof parameter !== 'boolean' && + (typeof parameter !== 'number' || !Number.isFinite(parameter)) + ) + return undefined + entries.push([key, parameter]) + } + return Object.fromEntries(entries) +} + +export function localizedDatasourceText( + text: Record | undefined, + language: string, + fallback: string, +) { + return ( + text?.[language] ?? + text?.[language.replaceAll('-', '_')] ?? + text?.en_US ?? + text?.zh_Hans ?? + fallback + ) +} diff --git a/web/features/new-rag/routes.ts b/web/features/new-rag/routes.ts index 78db7eacb24..142e21ee695 100644 --- a/web/features/new-rag/routes.ts +++ b/web/features/new-rag/routes.ts @@ -1,3 +1,6 @@ +import type { DatasourceParameters } from './datasource-parameter-model' +import { datasourceParameterRecord } from './datasource-parameter-model' + export type NewKnowledgeStartMode = 'empty' | 'source' | 'upload' export type NewKnowledgeSourceType = 'onlineDocuments' | 'onlineDrive' | 'websiteCrawl' type NewKnowledgeSyncPolicy = 'daily' | 'manual' | 'provider' @@ -10,6 +13,7 @@ export type NewKnowledgeSourceProvider = | NewKnowledgeWebsiteProvider type NewKnowledgeSourceDraftBase = { + parameters?: DatasourceParameters sourceName: string syncPolicy: NewKnowledgeSyncPolicy providerKey?: string @@ -51,6 +55,7 @@ export function createNewKnowledgeSourceDraft( if (sourceType === 'onlineDocuments') return { provider: initialProvider?.trim() || 'Notion', + parameters: {}, sourceName: '', sourceType, syncPolicy: 'provider', @@ -58,6 +63,7 @@ export function createNewKnowledgeSourceDraft( if (sourceType === 'onlineDrive') return { provider: initialProvider?.trim() || 'Google Drive', + parameters: {}, sourceName: '', sourceType, syncPolicy: 'provider', @@ -65,6 +71,7 @@ export function createNewKnowledgeSourceDraft( return { includeSubpages: true, maxPages: 100, + parameters: {}, provider: initialProvider?.trim() || 'Firecrawl', rootUrl: '', sourceName: '', @@ -127,6 +134,8 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD : candidate.syncPolicy === undefined ? 'provider' : undefined + const parameters = + candidate.parameters === undefined ? {} : datasourceParameterRecord(candidate.parameters) if ( typeof candidate.sourceName !== 'string' || candidate.sourceName.length > NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH || @@ -137,12 +146,14 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD (typeof candidate.providerKey !== 'string' || !candidate.providerKey || candidate.providerKey.length > NEW_KNOWLEDGE_PROVIDER_KEY_MAX_LENGTH)) || - !syncPolicy + !syncPolicy || + !parameters ) return undefined if (candidate.sourceType === 'onlineDocuments') { return { provider: candidate.provider, + parameters, ...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}), sourceName: candidate.sourceName, sourceType: candidate.sourceType, @@ -152,6 +163,7 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD if (candidate.sourceType === 'onlineDrive') { return { provider: candidate.provider, + parameters, ...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}), sourceName: candidate.sourceName, sourceType: candidate.sourceType, @@ -172,6 +184,7 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD return { includeSubpages: candidate.includeSubpages, maxPages: candidate.maxPages, + parameters, provider: candidate.provider, ...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}), rootUrl: candidate.rootUrl, diff --git a/web/features/new-rag/website-crawl-preview.tsx b/web/features/new-rag/website-crawl-preview.tsx index 40fee13715d..eef7566f9a0 100644 --- a/web/features/new-rag/website-crawl-preview.tsx +++ b/web/features/new-rag/website-crawl-preview.tsx @@ -1,7 +1,9 @@ 'use client' +import type { DatasourceParameters } from './datasource-parameter-model' import type { NewKnowledgeWebsiteSourceDraft } from './routes' import type { CrawlPreviewPage as PreviewPage, Source, SourceWorkflowRun } from './source-models' +import type { InstalledSourceProviderOption } from './source-provider-options' import { AlertDialog, AlertDialogActions, @@ -14,27 +16,26 @@ import { import { Button } from '@langgenius/dify-ui/button' import { Checkbox } from '@langgenius/dify-ui/checkbox' import { cn } from '@langgenius/dify-ui/cn' -import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from '@langgenius/dify-ui/collapsible' -import { Field, FieldControl, FieldError, FieldLabel } from '@langgenius/dify-ui/field' +import { Field, FieldControl, FieldLabel } from '@langgenius/dify-ui/field' import { Fieldset } from '@langgenius/dify-ui/fieldset' import { Form } from '@langgenius/dify-ui/form' -import { - NumberField, - NumberFieldControls, - NumberFieldDecrement, - NumberFieldGroup, - NumberFieldIncrement, - NumberFieldInput, -} from '@langgenius/dify-ui/number-field' -import { memo, useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useRouter } from '@/next/navigation' import { consoleClient } from '@/service/client' import { CrawlSelectionForm } from './crawl-selection-form' +import { DatasourceParameterForm } from './datasource-parameter-form' +import { + datasourceIncludeSubpages, + datasourceParameterDefaults, + invalidDatasourceParameters, + missingRequiredDatasourceParameters, + websiteDatasourceParameterSchemas, + withDatasourceParameterDefaults, +} from './datasource-parameter-model' import { createRequestId } from './request-id' import { NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH, - NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH, newKnowledgeDetailPath, normalizeWebsiteSourceUrl, } from './routes' @@ -46,10 +47,10 @@ type ConnectionReference = { } type CrawlConfiguration = { - includeSubpages: boolean - limit: number name: string - url: string + parameters: DatasourceParameters + rootUrl?: string + uri: string } type PreviewDraft = { @@ -79,8 +80,6 @@ function previewPagesEqual(left: PreviewPage, right: PreviewPage) { const PAGE_SIZE = 200 const MAX_CURSOR_PAGES = 100 const POLL_INTERVAL_MS = 1500 -const DEFAULT_PAGE_LIMIT = 100 -const MAX_PAGE_LIMIT = 200 const MAX_PREVIEW_SELECTION = 200 const SUCCESS_STATES = new Set([ 'complete', @@ -191,6 +190,15 @@ function configurationKey(configuration: CrawlConfiguration) { return JSON.stringify(configuration) } +function datasourceSourceUri(parameters: DatasourceParameters, fallback: string) { + const url = parameters.url + if (typeof url === 'string') { + const normalized = normalizeWebsiteSourceUrl(url) + if (normalized) return normalized.toString() + } + return `datasource://${encodeURIComponent(fallback)}` +} + function workflowAttemptKey(run: SourceWorkflowRun) { return `${run.id}:${run.executionAttempts}` } @@ -418,6 +426,7 @@ export function WebsiteCrawlPreview({ knowledgeSpaceId, onDraftFinished, onInteractionLockChange, + providerOption, providerName = 'Firecrawl', }: { connection: ConnectionReference @@ -425,25 +434,44 @@ export function WebsiteCrawlPreview({ knowledgeSpaceId: string onDraftFinished?: () => void onInteractionLockChange?: (locked: boolean) => void + providerOption?: InstalledSourceProviderOption providerName?: string }) { const { t } = useTranslation('dataset') const router = useRouter() - const rootUrlErrorId = useId() - const pageLimitErrorId = useId() - const [rootUrl, setRootUrl] = useState(initialDraft?.rootUrl ?? '') - const [sourceName, setSourceName] = useState(initialDraft?.sourceName ?? '') - const [urlTouched, setUrlTouched] = useState(false) - const [optionsExpanded, setOptionsExpanded] = useState(false) - const [includeSubpages, setIncludeSubpages] = useState(initialDraft?.includeSubpages ?? true) - const [pageLimit, setPageLimit] = useState(() => { - const initialLimit = initialDraft?.maxPages - return initialLimit && initialLimit > 0 && initialLimit <= MAX_PAGE_LIMIT - ? initialLimit - : DEFAULT_PAGE_LIMIT + const parameterSchemas = useMemo( + () => + providerOption + ? websiteDatasourceParameterSchemas(providerOption.datasource) + : websiteDatasourceParameterSchemas(), + [providerOption], + ) + const defaultParameters = useMemo( + () => datasourceParameterDefaults(parameterSchemas), + [parameterSchemas], + ) + const [parameters, setParameters] = useState(() => { + const initialParameters = withDatasourceParameterDefaults( + parameterSchemas, + initialDraft?.parameters, + ) + if (!providerOption || providerOption.datasource.parameters.length === 0) { + const crawlSubpagesParameter = parameterSchemas.find((parameter) => + ['crawl_sub_pages', 'crawl_subpages'].includes(parameter.name), + ) + if (crawlSubpagesParameter) + initialParameters[crawlSubpagesParameter.name] = initialDraft?.includeSubpages ?? true + initialParameters.limit = initialDraft?.maxPages ?? 100 + } + if ( + initialDraft?.rootUrl && + parameterSchemas.some((parameter) => parameter.name === 'url') && + initialParameters.url === undefined + ) + initialParameters.url = initialDraft.rootUrl + return initialParameters }) - const crawlOptionsAreDefault = - includeSubpages && (pageLimit === '' || pageLimit === DEFAULT_PAGE_LIMIT) + const [sourceName, setSourceName] = useState(initialDraft?.sourceName ?? '') const [run, setRun] = useState() const [pages, setPages] = useState([]) const [selectedPageIds, setSelectedPageIds] = useState>(() => new Set()) @@ -464,7 +492,6 @@ export function WebsiteCrawlPreview({ const actionPendingRef = useRef(false) const retryFingerprintRef = useRef(undefined) const cancelFingerprintRef = useRef(undefined) - const rootUrlInputRef = useRef(null) const sourceNameInputRef = useRef(null) const pageMapRef = useRef(new Map()) const pageCursorRef = useRef(undefined) @@ -568,27 +595,29 @@ export function WebsiteCrawlPreview({ setSelectionUncertain(uncertain) }, []) - const normalizedURL = useMemo(() => normalizeWebsiteSourceUrl(rootUrl), [rootUrl]) - const pageLimitValid = - typeof pageLimit === 'number' && - Number.isInteger(pageLimit) && - pageLimit >= 1 && - pageLimit <= MAX_PAGE_LIMIT + const normalizedURL = useMemo( + () => + typeof parameters.url === 'string' ? normalizeWebsiteSourceUrl(parameters.url) : undefined, + [parameters.url], + ) + const hasUrlParameter = parameterSchemas.some((parameter) => parameter.name === 'url') + const parametersValid = + !missingRequiredDatasourceParameters(parameterSchemas, parameters).length && + !invalidDatasourceParameters(parameterSchemas, parameters).length && + !(hasUrlParameter && parameters.url && !normalizedURL) const configuration = useMemo( () => - normalizedURL && + parametersValid && sourceName.trim() && - sourceName.trim().length <= NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH && - typeof pageLimit === 'number' && - pageLimitValid + sourceName.trim().length <= NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH ? { - includeSubpages, - limit: pageLimit, name: sourceName.trim(), - url: normalizedURL.toString(), + parameters, + ...(normalizedURL ? { rootUrl: normalizedURL.toString() } : {}), + uri: datasourceSourceUri(parameters, providerOption?.key ?? providerName), } : undefined, - [includeSubpages, pageLimit, pageLimitValid, normalizedURL, sourceName], + [normalizedURL, parameters, parametersValid, providerName, providerOption?.key, sourceName], ) const currentConfigurationKey = configuration ? configurationKey(configuration) : undefined const previewConfigurationMatches = Boolean( @@ -612,9 +641,9 @@ export function WebsiteCrawlPreview({ const runId = run?.id const locked = starting || stopping || active || uncertainOperation || selectionInteractionLocked const dirty = Boolean( - rootUrl || sourceName || run || !includeSubpages || pageLimit !== DEFAULT_PAGE_LIMIT, + sourceName || run || JSON.stringify(parameters) !== JSON.stringify(defaultParameters), ) - const host = normalizedURL?.host ?? '' + const host = normalizedURL?.host ?? providerName const completedCount = Math.max(run?.progressCompleted ?? 0, pages.length) const crawlingStatusText = t(($) => $['newKnowledge.crawlingPages'], { count: completedCount, @@ -795,9 +824,14 @@ export function WebsiteCrawlPreview({ metadata: { clientRequestId: draft.clientRequestId, crawlOptions: { - includeSubpages: nextConfiguration.includeSubpages, - limit: nextConfiguration.limit, + includeSubpages: datasourceIncludeSubpages(nextConfiguration.parameters), + limit: + typeof nextConfiguration.parameters.limit === 'number' + ? nextConfiguration.parameters.limit + : 200, }, + datasourceParameterMode: 'exact', + parameters: nextConfiguration.parameters, preview: true, providerId: connection.providerId, providerName, @@ -805,7 +839,7 @@ export function WebsiteCrawlPreview({ name: nextConfiguration.name, status: 'disabled', type: 'web', - uri: nextConfiguration.url, + uri: nextConfiguration.uri, }, params: { control_space_id: knowledgeSpaceId }, }), @@ -1136,9 +1170,7 @@ export function WebsiteCrawlPreview({ const handlePrimaryAction = () => { if (!configuration) { - setUrlTouched(true) - if (!normalizedURL) rootUrlInputRef.current?.focus() - else sourceNameInputRef.current?.focus() + sourceNameInputRef.current?.focus() return } if (requestError === 'POLL_FAILED' && run) { @@ -1294,31 +1326,13 @@ export function WebsiteCrawlPreview({

+
- - - {t(($) => $['newKnowledge.rootUrl'])} - * - - $['newKnowledge.rootUrlPlaceholder'])} - aria-describedby={urlTouched && !normalizedURL ? rootUrlErrorId : undefined} - onBlur={() => setUrlTouched(true)} - onValueChange={setRootUrl} - /> - {urlTouched && !normalizedURL && ( - - {t(($) => $['newKnowledge.invalidRootUrl'])} - - )} - {t(($) => $['newKnowledge.sourceName'])} @@ -1336,81 +1350,6 @@ export function WebsiteCrawlPreview({ />
- - - - - {t(($) => $['newKnowledge.crawlOptions'])} - - {!optionsExpanded && ( - - {crawlOptionsAreDefault - ? t(($) => $['newKnowledge.usingDefaults']) - : `${t(($) => $['newKnowledge.includeSubpages'])}: ${t(($) => - includeSubpages - ? $['newKnowledge.booleanTrue'] - : $['newKnowledge.booleanFalse'], - )} · ${t(($) => $['newKnowledge.maxPages'])}: ${ - pageLimit === '' ? DEFAULT_PAGE_LIMIT : pageLimit - }`} - - )} - - -
- -
-
- - {t(($) => $['newKnowledge.maxPages'])} - - setPageLimit(value === null ? '' : value)} - > - - $['newKnowledge.maxPages'])} - aria-describedby={!pageLimitValid ? pageLimitErrorId : undefined} - aria-invalid={!pageLimitValid} - onBlur={() => { - if (pageLimit === '') setPageLimit(DEFAULT_PAGE_LIMIT) - }} - /> - - - - - - -
- {!pageLimitValid && ( - - )} -
-
- -
{!showSuccess && ( @@ -1506,7 +1445,7 @@ export function WebsiteCrawlPreview({ pendingCancelRunRef.current = nextRun }} pages={pages} - rootUrl={configuration.url} + rootUrl={configuration.rootUrl} run={run} source={draftRef.current.source} workflowUncertain={workflowUncertain} diff --git a/web/i18n/ar-TN/dataset.json b/web/i18n/ar-TN/dataset.json index 305098879b5..3bf6db6fa35 100644 --- a/web/i18n/ar-TN/dataset.json +++ b/web/i18n/ar-TN/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "تضمين الصفحات الفرعية", "newKnowledge.indexInformation": "الفهرسة", "newKnowledge.interruptTask": "إيقاف", + "newKnowledge.invalidDatasourceParameter": "أدخل قيمة صالحة لـ {{parameter}}.", "newKnowledge.invalidRootUrl": "أدخل عنوان URL صالحًا يبدأ بـ http(s).", "newKnowledge.keepEditing": "متابعة التحرير", "newKnowledge.lastIndexed": "آخر فهرسة", diff --git a/web/i18n/de-DE/dataset.json b/web/i18n/de-DE/dataset.json index ee398b25a8f..1879c49f033 100644 --- a/web/i18n/de-DE/dataset.json +++ b/web/i18n/de-DE/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Unterseiten einbeziehen", "newKnowledge.indexInformation": "Index", "newKnowledge.interruptTask": "Unterbrechen", + "newKnowledge.invalidDatasourceParameter": "Gib einen gültigen Wert für {{parameter}} ein.", "newKnowledge.invalidRootUrl": "Gib eine gültige http(s)-URL ein.", "newKnowledge.keepEditing": "Weiter bearbeiten", "newKnowledge.lastIndexed": "Zuletzt indexiert", diff --git a/web/i18n/en-US/dataset.json b/web/i18n/en-US/dataset.json index ceec0dc3df0..bc88624a820 100644 --- a/web/i18n/en-US/dataset.json +++ b/web/i18n/en-US/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Include sub-pages", "newKnowledge.indexInformation": "Index", "newKnowledge.interruptTask": "Interrupt", + "newKnowledge.invalidDatasourceParameter": "Enter a valid value for {{parameter}}.", "newKnowledge.invalidRootUrl": "Enter a valid http(s) URL.", "newKnowledge.keepEditing": "Keep editing", "newKnowledge.lastIndexed": "Last indexed", diff --git a/web/i18n/es-ES/dataset.json b/web/i18n/es-ES/dataset.json index d1d563d2764..0a476e232cf 100644 --- a/web/i18n/es-ES/dataset.json +++ b/web/i18n/es-ES/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Incluir subpáginas", "newKnowledge.indexInformation": "Índice", "newKnowledge.interruptTask": "Interrumpir", + "newKnowledge.invalidDatasourceParameter": "Introduce un valor válido para {{parameter}}.", "newKnowledge.invalidRootUrl": "Introduce una URL http(s) válida.", "newKnowledge.keepEditing": "Seguir editando", "newKnowledge.lastIndexed": "Última indexación", diff --git a/web/i18n/fa-IR/dataset.json b/web/i18n/fa-IR/dataset.json index dec3d5e0efc..a02a15e296e 100644 --- a/web/i18n/fa-IR/dataset.json +++ b/web/i18n/fa-IR/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "شامل صفحه‌های فرعی", "newKnowledge.indexInformation": "نمایه", "newKnowledge.interruptTask": "توقف", + "newKnowledge.invalidDatasourceParameter": "یک مقدار معتبر برای {{parameter}} وارد کنید.", "newKnowledge.invalidRootUrl": "یک نشانی معتبر http(s) وارد کنید.", "newKnowledge.keepEditing": "ادامه ویرایش", "newKnowledge.lastIndexed": "آخرین نمایه‌سازی", diff --git a/web/i18n/fr-FR/dataset.json b/web/i18n/fr-FR/dataset.json index c2409804d08..d16fa3a9b43 100644 --- a/web/i18n/fr-FR/dataset.json +++ b/web/i18n/fr-FR/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Inclure les sous-pages", "newKnowledge.indexInformation": "Index", "newKnowledge.interruptTask": "Interrompre", + "newKnowledge.invalidDatasourceParameter": "Saisissez une valeur valide pour {{parameter}}.", "newKnowledge.invalidRootUrl": "Saisissez une URL http(s) valide.", "newKnowledge.keepEditing": "Continuer la modification", "newKnowledge.lastIndexed": "Dernière indexation", diff --git a/web/i18n/hi-IN/dataset.json b/web/i18n/hi-IN/dataset.json index fc3d5c7b13c..9ff96672b34 100644 --- a/web/i18n/hi-IN/dataset.json +++ b/web/i18n/hi-IN/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "उप-पेज शामिल करें", "newKnowledge.indexInformation": "अनुक्रमणिका", "newKnowledge.interruptTask": "रोकें", + "newKnowledge.invalidDatasourceParameter": "{{parameter}} के लिए मान्य मान दर्ज करें।", "newKnowledge.invalidRootUrl": "मान्य http(s) URL दर्ज करें।", "newKnowledge.keepEditing": "संपादन जारी रखें", "newKnowledge.lastIndexed": "पिछली बार अनुक्रमित", diff --git a/web/i18n/id-ID/dataset.json b/web/i18n/id-ID/dataset.json index 8e40fb4da49..0fa1a4c660f 100644 --- a/web/i18n/id-ID/dataset.json +++ b/web/i18n/id-ID/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Sertakan subhalaman", "newKnowledge.indexInformation": "Indeks", "newKnowledge.interruptTask": "Hentikan", + "newKnowledge.invalidDatasourceParameter": "Masukkan nilai yang valid untuk {{parameter}}.", "newKnowledge.invalidRootUrl": "Masukkan URL http(s) yang valid.", "newKnowledge.keepEditing": "Lanjutkan mengedit", "newKnowledge.lastIndexed": "Terakhir diindeks", diff --git a/web/i18n/it-IT/dataset.json b/web/i18n/it-IT/dataset.json index f336281a38c..67a0667e39f 100644 --- a/web/i18n/it-IT/dataset.json +++ b/web/i18n/it-IT/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Includi sottopagine", "newKnowledge.indexInformation": "Indice", "newKnowledge.interruptTask": "Interrompere", + "newKnowledge.invalidDatasourceParameter": "Inserisci un valore valido per {{parameter}}.", "newKnowledge.invalidRootUrl": "Inserisci un URL http(s) valido.", "newKnowledge.keepEditing": "Continua a modificare", "newKnowledge.lastIndexed": "Ultima indicizzazione", diff --git a/web/i18n/ja-JP/dataset.json b/web/i18n/ja-JP/dataset.json index 490e3f16834..e0a7f729de6 100644 --- a/web/i18n/ja-JP/dataset.json +++ b/web/i18n/ja-JP/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "サブページを含める", "newKnowledge.indexInformation": "インデックス", "newKnowledge.interruptTask": "中断", + "newKnowledge.invalidDatasourceParameter": "{{parameter}} に有効な値を入力してください。", "newKnowledge.invalidRootUrl": "有効なhttp(s) URLを入力してください。", "newKnowledge.keepEditing": "編集を続ける", "newKnowledge.lastIndexed": "最終インデックス日時", diff --git a/web/i18n/ko-KR/dataset.json b/web/i18n/ko-KR/dataset.json index fccd9787fca..f8cb1360214 100644 --- a/web/i18n/ko-KR/dataset.json +++ b/web/i18n/ko-KR/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "하위 페이지 포함", "newKnowledge.indexInformation": "인덱스", "newKnowledge.interruptTask": "중단", + "newKnowledge.invalidDatasourceParameter": "{{parameter}}에 유효한 값을 입력하세요.", "newKnowledge.invalidRootUrl": "유효한 http(s) URL을 입력하세요.", "newKnowledge.keepEditing": "계속 편집", "newKnowledge.lastIndexed": "마지막 인덱싱", diff --git a/web/i18n/lo-LA/dataset.json b/web/i18n/lo-LA/dataset.json index 91e4216d05e..51ed8d83ef3 100644 --- a/web/i18n/lo-LA/dataset.json +++ b/web/i18n/lo-LA/dataset.json @@ -286,6 +286,7 @@ "newKnowledge.includeSubpages": "ລວມເອົາໜ້າຍ່ອຍ", "newKnowledge.indexInformation": "ຂໍ້ມູນດັດຊະນີ", "newKnowledge.interruptTask": "ຂັດຂວາງ", + "newKnowledge.invalidDatasourceParameter": "ປ້ອນຄ່າທີ່ຖືກຕ້ອງສຳລັບ {{parameter}}.", "newKnowledge.invalidRootUrl": "ລະບຸ URL http(s) ທີ່ຖືກຕ້ອງ.", "newKnowledge.keepEditing": "ສືບຕໍ່ແກ້ໄຂ", "newKnowledge.lastIndexed": "ດັດສະນີຫຼ້າສຸດ", diff --git a/web/i18n/nl-NL/dataset.json b/web/i18n/nl-NL/dataset.json index 344e54f2b12..a38ad85b502 100644 --- a/web/i18n/nl-NL/dataset.json +++ b/web/i18n/nl-NL/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Subpagina’s opnemen", "newKnowledge.indexInformation": "Index", "newKnowledge.interruptTask": "Onderbreken", + "newKnowledge.invalidDatasourceParameter": "Voer een geldige waarde in voor {{parameter}}.", "newKnowledge.invalidRootUrl": "Voer een geldige http(s)-URL in.", "newKnowledge.keepEditing": "Doorgaan met bewerken", "newKnowledge.lastIndexed": "Laatst geïndexeerd", diff --git a/web/i18n/pl-PL/dataset.json b/web/i18n/pl-PL/dataset.json index 16a497d5e4c..ed9d2cf8a74 100644 --- a/web/i18n/pl-PL/dataset.json +++ b/web/i18n/pl-PL/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Uwzględnij podstrony", "newKnowledge.indexInformation": "Indeks", "newKnowledge.interruptTask": "Przerwij", + "newKnowledge.invalidDatasourceParameter": "Wprowadź prawidłową wartość dla {{parameter}}.", "newKnowledge.invalidRootUrl": "Wprowadź prawidłowy adres URL http(s).", "newKnowledge.keepEditing": "Kontynuuj edycję", "newKnowledge.lastIndexed": "Ostatnio zindeksowano", diff --git a/web/i18n/pt-BR/dataset.json b/web/i18n/pt-BR/dataset.json index ef630d1018c..a6d39e601df 100644 --- a/web/i18n/pt-BR/dataset.json +++ b/web/i18n/pt-BR/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Incluir subpáginas", "newKnowledge.indexInformation": "Índice", "newKnowledge.interruptTask": "Interromper", + "newKnowledge.invalidDatasourceParameter": "Insira um valor válido para {{parameter}}.", "newKnowledge.invalidRootUrl": "Insira uma URL http(s) válida.", "newKnowledge.keepEditing": "Continuar editando", "newKnowledge.lastIndexed": "Última indexação", diff --git a/web/i18n/ro-RO/dataset.json b/web/i18n/ro-RO/dataset.json index 53a15b72faa..ce99e8eba1d 100644 --- a/web/i18n/ro-RO/dataset.json +++ b/web/i18n/ro-RO/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Include subpaginile", "newKnowledge.indexInformation": "Index", "newKnowledge.interruptTask": "Întrerupe", + "newKnowledge.invalidDatasourceParameter": "Introdu o valoare validă pentru {{parameter}}.", "newKnowledge.invalidRootUrl": "Introdu o adresă URL http(s) validă.", "newKnowledge.keepEditing": "Continuă editarea", "newKnowledge.lastIndexed": "Ultima indexare", diff --git a/web/i18n/ru-RU/dataset.json b/web/i18n/ru-RU/dataset.json index 78bd2676390..1bc525e9c45 100644 --- a/web/i18n/ru-RU/dataset.json +++ b/web/i18n/ru-RU/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Включать вложенные страницы", "newKnowledge.indexInformation": "Индекс", "newKnowledge.interruptTask": "Прерывание", + "newKnowledge.invalidDatasourceParameter": "Введите допустимое значение для {{parameter}}.", "newKnowledge.invalidRootUrl": "Введите корректный URL с http(s).", "newKnowledge.keepEditing": "Продолжить редактирование", "newKnowledge.lastIndexed": "Последняя индексация", diff --git a/web/i18n/sl-SI/dataset.json b/web/i18n/sl-SI/dataset.json index 53968123893..86cf7b80802 100644 --- a/web/i18n/sl-SI/dataset.json +++ b/web/i18n/sl-SI/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Vključi podstrani", "newKnowledge.indexInformation": "Indeks", "newKnowledge.interruptTask": "Prekini", + "newKnowledge.invalidDatasourceParameter": "Vnesite veljavno vrednost za {{parameter}}.", "newKnowledge.invalidRootUrl": "Vnesite veljaven URL http(s).", "newKnowledge.keepEditing": "Nadaljuj urejanje", "newKnowledge.lastIndexed": "Nazadnje indeksirano", diff --git a/web/i18n/th-TH/dataset.json b/web/i18n/th-TH/dataset.json index d62643b56fd..17abd8ceb84 100644 --- a/web/i18n/th-TH/dataset.json +++ b/web/i18n/th-TH/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "รวมหน้าย่อย", "newKnowledge.indexInformation": "ดัชนี", "newKnowledge.interruptTask": "ขัดจังหวะ", + "newKnowledge.invalidDatasourceParameter": "ป้อนค่าที่ถูกต้องสำหรับ {{parameter}}", "newKnowledge.invalidRootUrl": "ป้อน URL http(s) ที่ถูกต้อง", "newKnowledge.keepEditing": "แก้ไขต่อ", "newKnowledge.lastIndexed": "จัดทำดัชนีล่าสุด", diff --git a/web/i18n/tr-TR/dataset.json b/web/i18n/tr-TR/dataset.json index 9146e51780c..14e8f673a59 100644 --- a/web/i18n/tr-TR/dataset.json +++ b/web/i18n/tr-TR/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Alt sayfaları dahil et", "newKnowledge.indexInformation": "Dizin", "newKnowledge.interruptTask": "Kesinti", + "newKnowledge.invalidDatasourceParameter": "{{parameter}} için geçerli bir değer girin.", "newKnowledge.invalidRootUrl": "Geçerli bir http(s) URL’si girin.", "newKnowledge.keepEditing": "Düzenlemeye devam et", "newKnowledge.lastIndexed": "Son dizinleme", diff --git a/web/i18n/uk-UA/dataset.json b/web/i18n/uk-UA/dataset.json index 5d015367e5d..422a71d665d 100644 --- a/web/i18n/uk-UA/dataset.json +++ b/web/i18n/uk-UA/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Включати підсторінки", "newKnowledge.indexInformation": "Індекс", "newKnowledge.interruptTask": "Переривати", + "newKnowledge.invalidDatasourceParameter": "Введіть припустиме значення для {{parameter}}.", "newKnowledge.invalidRootUrl": "Введіть дійсну URL-адресу http(s).", "newKnowledge.keepEditing": "Продовжити редагування", "newKnowledge.lastIndexed": "Останнє індексування", diff --git a/web/i18n/vi-VN/dataset.json b/web/i18n/vi-VN/dataset.json index 5a4bdcca79e..f864ac8c154 100644 --- a/web/i18n/vi-VN/dataset.json +++ b/web/i18n/vi-VN/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "Bao gồm các trang con", "newKnowledge.indexInformation": "Chỉ mục", "newKnowledge.interruptTask": "Ngắt", + "newKnowledge.invalidDatasourceParameter": "Nhập giá trị hợp lệ cho {{parameter}}.", "newKnowledge.invalidRootUrl": "Nhập URL http(s) hợp lệ.", "newKnowledge.keepEditing": "Tiếp tục chỉnh sửa", "newKnowledge.lastIndexed": "Lập chỉ mục gần nhất", diff --git a/web/i18n/zh-Hans/dataset.json b/web/i18n/zh-Hans/dataset.json index c80cd3d4387..a83d480f775 100644 --- a/web/i18n/zh-Hans/dataset.json +++ b/web/i18n/zh-Hans/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "包含子页面", "newKnowledge.indexInformation": "索引", "newKnowledge.interruptTask": "中断", + "newKnowledge.invalidDatasourceParameter": "请输入有效的 {{parameter}}。", "newKnowledge.invalidRootUrl": "请输入有效的 http(s) URL。", "newKnowledge.keepEditing": "继续编辑", "newKnowledge.lastIndexed": "最近索引时间", diff --git a/web/i18n/zh-Hant/dataset.json b/web/i18n/zh-Hant/dataset.json index 086c4679cab..67b1dfd0704 100644 --- a/web/i18n/zh-Hant/dataset.json +++ b/web/i18n/zh-Hant/dataset.json @@ -294,6 +294,7 @@ "newKnowledge.includeSubpages": "包含子頁面", "newKnowledge.indexInformation": "索引", "newKnowledge.interruptTask": "中斷", + "newKnowledge.invalidDatasourceParameter": "請輸入有效的 {{parameter}}。", "newKnowledge.invalidRootUrl": "請輸入有效的 http(s) URL。", "newKnowledge.keepEditing": "繼續編輯", "newKnowledge.lastIndexed": "最近建立索引時間",