feat(knowledge-fs): support datasource-driven source previews

api/controllers/console/knowledge_fs/resources.py

@@ -60,6 +60,11 @@ from services.knowledge_fs.download_service import (
     KnowledgeFSDownloadTooLargeError,
 )
 from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService
+from services.knowledge_fs.initial_source_preview_job import (
+    KnowledgeFSInitialSourcePreviewJobAlreadyRunningError,
+    KnowledgeFSInitialSourcePreviewJobNotFoundError,
+    KnowledgeFSInitialSourcePreviewJobService,
+)
 from services.knowledge_fs.product_authorization import (
     KnowledgeFSProductNotFoundError,
 )
@@ -119,8 +124,11 @@ from services.knowledge_fs.product_dto import (
     KnowledgeFSGoldenQuestionPayload,
     KnowledgeFSGoldenQuestionResponse,
     KnowledgeFSIdempotencyHeader,
+    KnowledgeFSInitialSourcePreviewJobCreateResponse,
+    KnowledgeFSInitialSourcePreviewJobResponse,
     KnowledgeFSInitialSourcePreviewPayload,
     KnowledgeFSInitialSourcePreviewResponse,
+    KnowledgeFSInitialWebsiteSourcePreviewPayload,
     KnowledgeFSJWKSResponse,
     KnowledgeFSLogicalDocumentDeletePayload,
     KnowledgeFSLogicalDocumentListResponse,
@@ -281,6 +289,7 @@ register_schema_models(
     KnowledgeFSCrawlImportPayload,
     KnowledgeFSCrawlPreviewSelectionPayload,
     KnowledgeFSInitialSourcePreviewPayload,
+    KnowledgeFSInitialWebsiteSourcePreviewPayload,
     KnowledgeFSSourceDeletePayload,
     KnowledgeFSSourceDeleteQuery,
     KnowledgeFSSourceFilesQuery,
@@ -380,6 +389,8 @@ register_response_schema_models(
     KnowledgeFSOverviewStatsResponse,
     KnowledgeFSPresignedUploadResponse,
     KnowledgeFSInitialSourcePreviewResponse,
+    KnowledgeFSInitialSourcePreviewJobCreateResponse,
+    KnowledgeFSInitialSourcePreviewJobResponse,
     KnowledgeFSUploadSessionCreateResponse,
     KnowledgeFSUploadSessionMutationResponse,
 )
@@ -401,6 +412,8 @@ def _knowledge_fs_errors[**P, R](view: Callable[P, R]) -> Callable[P, R]:
             raise KnowledgeFSSpaceNotFoundHTTPError() from exc
         except KnowledgeFSOperationUnavailableError as exc:
             raise KnowledgeFSOperationUnavailableHTTPError() from exc
+        except KnowledgeFSInitialSourcePreviewJobAlreadyRunningError as exc:
+            raise Conflict() from exc
         except KnowledgeFSProductResourceNotFoundError as exc:
             raise NotFound() from exc
         except KnowledgeFSStagedUploadNotFoundError as exc:
@@ -666,6 +679,73 @@ class KnowledgeFSInitialSourcePreviewApi(Resource):
         return dump_response(KnowledgeFSInitialSourcePreviewResponse, result)
 
 
+@console_ns.route("/knowledge-fs/source-provider-preview/jobs")
+class KnowledgeFSInitialSourcePreviewJobsApi(Resource):
+    @console_ns.expect(console_ns.models[KnowledgeFSInitialWebsiteSourcePreviewPayload.__name__])
+    @console_ns.response(
+        HTTPStatus.ACCEPTED,
+        "Website datasource preview queued",
+        console_ns.models[KnowledgeFSInitialSourcePreviewJobCreateResponse.__name__],
+    )
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @_knowledge_fs_errors
+    def post(self):
+        account, tenant_id = current_account_with_tenant()
+        result = KnowledgeFSInitialSourcePreviewJobService(session_factory.get_session_maker()).start(
+            tenant_id=tenant_id,
+            account=account,
+            payload=_payload(KnowledgeFSInitialWebsiteSourcePreviewPayload),
+        )
+        return dump_response(KnowledgeFSInitialSourcePreviewJobCreateResponse, result), HTTPStatus.ACCEPTED
+
+
+@console_ns.route("/knowledge-fs/source-provider-preview/jobs/<string:job_id>")
+class KnowledgeFSInitialSourcePreviewJobApi(Resource):
+    @console_ns.response(
+        HTTPStatus.OK,
+        "Website datasource preview status",
+        console_ns.models[KnowledgeFSInitialSourcePreviewJobResponse.__name__],
+    )
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @_knowledge_fs_errors
+    def get(self, job_id: str):
+        account, tenant_id = current_account_with_tenant()
+        try:
+            result = KnowledgeFSInitialSourcePreviewJobService.get(
+                tenant_id=tenant_id,
+                account_id=account.id,
+                job_id=job_id,
+            )
+        except KnowledgeFSInitialSourcePreviewJobNotFoundError as exc:
+            raise NotFound() from exc
+        return dump_response(KnowledgeFSInitialSourcePreviewJobResponse, result)
+
+    @console_ns.response(
+        HTTPStatus.OK,
+        "Website datasource preview canceled",
+        console_ns.models[KnowledgeFSInitialSourcePreviewJobResponse.__name__],
+    )
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @_knowledge_fs_errors
+    def delete(self, job_id: str):
+        account, tenant_id = current_account_with_tenant()
+        try:
+            result = KnowledgeFSInitialSourcePreviewJobService.cancel(
+                tenant_id=tenant_id,
+                account_id=account.id,
+                job_id=job_id,
+            )
+        except KnowledgeFSInitialSourcePreviewJobNotFoundError as exc:
+            raise NotFound() from exc
+        return dump_response(KnowledgeFSInitialSourcePreviewJobResponse, result)
+
+
 @console_ns.route("/knowledge-fs/uploads")
 class KnowledgeFSStagedUploadsApi(Resource):
     @console_ns.doc(consumes=["multipart/form-data"], params=_STAGED_UPLOAD_PARAMS)

api/extensions/ext_celery.py

@@ -175,6 +175,7 @@ def init_app(app: DifyApp) -> Celery:
         "tasks.refresh_billing_vector_space_task",  # billing vector-space cache refresh
         "tasks.app_generate.resume_agent_app_task",  # ENG-635: Agent v2 chat ask_human resume
         "tasks.workflow_run_archive_download_tasks",  # workflow-run archive download preparation
+        "tasks.knowledge_fs_initial_source_preview_tasks",  # datasource previews use the standard dataset queue
     ]
     day = dify_config.CELERY_BEAT_SCHEDULER_TIME
 

api/services/datasource_provider_service.py

@@ -197,6 +197,7 @@ class DatasourceProviderService:
         provider: str,
         plugin_id: str,
         credential_id: str | None = None,
+        current_user: Any | None = None,
     ) -> dict[str, Any]:
         """
         Return decrypted datasource credentials.
@@ -230,7 +231,8 @@ class DatasourceProviderService:
             if not datasource_provider:
                 return {}
             if self._should_refresh_credentials(datasource_provider):
-                current_user = get_current_user()
+                if current_user is None:
+                    current_user = get_current_user()
                 encrypted_credentials, expires_at = self._refresh_datasource_credentials(
                     tenant_id=tenant_id,
                     provider=provider,

api/services/knowledge_fs/initial_source_preview.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+from collections.abc import Callable
 from typing import Any, cast
 
 from sqlalchemy import select
@@ -14,6 +15,7 @@ from core.datasource.entities.datasource_entities import (
 )
 from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
 from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
+from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
 from models.account import Account
 from models.credential_permission import CredentialType
 from models.oauth import DatasourceProvider
@@ -22,23 +24,34 @@ from services.datasource_provider_service import DatasourceProviderService
 from services.knowledge_fs.product_dto import (
     KnowledgeFSInitialSourcePreviewDocumentResponse,
     KnowledgeFSInitialSourcePreviewFileResponse,
+    KnowledgeFSInitialSourcePreviewPageResponse,
     KnowledgeFSInitialSourcePreviewPayload,
     KnowledgeFSInitialSourcePreviewResponse,
+    KnowledgeFSInitialWebsiteSourcePreviewPayload,
 )
 
 _MAX_PREVIEW_ITEMS = 200
 
 
+class KnowledgeFSInitialSourcePreviewCanceledError(RuntimeError):
+    pass
+
+
+def _raise_if_canceled(is_canceled: Callable[[], bool] | None) -> None:
+    if is_canceled is not None and is_canceled():
+        raise KnowledgeFSInitialSourcePreviewCanceledError("Datasource preview was canceled")
+
+
 class KnowledgeFSInitialSourcePreviewService:
     def __init__(self, session_maker) -> None:
         self._session_maker = session_maker
 
-    def _require_visible_credential(
+    def require_visible_credential(
         self,
         *,
         tenant_id: str,
         account: Account,
-        payload: KnowledgeFSInitialSourcePreviewPayload,
+        payload: KnowledgeFSInitialSourcePreviewPayload | KnowledgeFSInitialWebsiteSourcePreviewPayload,
     ) -> None:
         query = select(DatasourceProvider).where(
             DatasourceProvider.tenant_id == tenant_id,
@@ -63,14 +76,17 @@ class KnowledgeFSInitialSourcePreviewService:
         *,
         tenant_id: str,
         account: Account,
-        payload: KnowledgeFSInitialSourcePreviewPayload,
+        payload: KnowledgeFSInitialSourcePreviewPayload | KnowledgeFSInitialWebsiteSourcePreviewPayload,
+        is_canceled: Callable[[], bool] | None = None,
     ) -> KnowledgeFSInitialSourcePreviewResponse:
-        self._require_visible_credential(tenant_id=tenant_id, account=account, payload=payload)
+        _raise_if_canceled(is_canceled)
+        self.require_visible_credential(tenant_id=tenant_id, account=account, payload=payload)
         credentials = DatasourceProviderService().get_datasource_credentials(
             tenant_id=tenant_id,
             provider=payload.provider,
             plugin_id=payload.plugin_id,
             credential_id=payload.credential_id,
+            current_user=account,
         )
         if not credentials:
             raise PermissionError("Datasource credential is unavailable")
@@ -83,6 +99,30 @@ class KnowledgeFSInitialSourcePreviewService:
         )
         runtime.runtime.credentials = credentials
         parameters = dict(payload.parameters)
+        if payload.kind == "website_crawl":
+            website_runtime = cast(WebsiteCrawlDatasourcePlugin, runtime)
+            pages_by_url: dict[str, KnowledgeFSInitialSourcePreviewPageResponse] = {}
+            for website_message in website_runtime.get_website_crawl(
+                user_id=account.id,
+                datasource_parameters=parameters,
+                provider_type=website_runtime.datasource_provider_type(),
+            ):
+                _raise_if_canceled(is_canceled)
+                for page in website_message.result.web_info_list or []:
+                    pages_by_url[page.source_url] = KnowledgeFSInitialSourcePreviewPageResponse(
+                        description=page.description or None,
+                        source_url=page.source_url,
+                        title=page.title or None,
+                    )
+                    if len(pages_by_url) >= _MAX_PREVIEW_ITEMS:
+                        return KnowledgeFSInitialSourcePreviewResponse(
+                            kind=payload.kind,
+                            pages=list(pages_by_url.values()),
+                        )
+            return KnowledgeFSInitialSourcePreviewResponse(
+                kind=payload.kind,
+                pages=list(pages_by_url.values()),
+            )
         if payload.kind == "online_document":
             document_runtime = cast(OnlineDocumentDatasourcePlugin, runtime)
             documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = []
@@ -174,4 +214,4 @@ class KnowledgeFSInitialSourcePreviewService:
         )
 
 
-__all__ = ["KnowledgeFSInitialSourcePreviewService"]
+__all__ = ["KnowledgeFSInitialSourcePreviewCanceledError", "KnowledgeFSInitialSourcePreviewService"]

api/services/knowledge_fs/initial_source_preview_job.py

@@ -0,0 +1,190 @@
+"""Asynchronous website datasource previews that do not hold Console HTTP workers."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from typing import Literal
+
+from celery import current_app as celery_app
+
+from extensions.ext_redis import redis_client
+from models.account import Account
+from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService
+from services.knowledge_fs.product_dto import (
+    KnowledgeFSInitialSourcePreviewJobCreateResponse,
+    KnowledgeFSInitialSourcePreviewJobResponse,
+    KnowledgeFSInitialSourcePreviewResponse,
+    KnowledgeFSInitialWebsiteSourcePreviewPayload,
+)
+
+_JOB_TTL_SECONDS = 60 * 60
+_RELEASE_ACTIVE_JOB_SCRIPT = """
+local current = redis.call('GET', KEYS[1])
+if current == ARGV[1] then
+  return redis.call('DEL', KEYS[1])
+end
+return 0
+"""
+_TRANSITION_STATUS_SCRIPT = """
+local current = redis.call('GET', KEYS[1])
+if not current then
+  return 0
+end
+local decoded = cjson.decode(current)
+for index = 3, #ARGV do
+  if decoded.status == ARGV[index] then
+    redis.call('SETEX', KEYS[1], ARGV[2], ARGV[1])
+    return 1
+  end
+end
+return 0
+"""
+
+
+class KnowledgeFSInitialSourcePreviewJobNotFoundError(LookupError):
+    pass
+
+
+class KnowledgeFSInitialSourcePreviewJobAlreadyRunningError(RuntimeError):
+    pass
+
+
+def _job_key(*, tenant_id: str, account_id: str, job_id: str) -> str:
+    return f"knowledge_fs:initial_source_preview:{tenant_id}:{account_id}:{job_id}"
+
+
+def _active_job_key(*, tenant_id: str, account_id: str) -> str:
+    return f"knowledge_fs:initial_source_preview:{tenant_id}:{account_id}:active"
+
+
+class KnowledgeFSInitialSourcePreviewJobService:
+    def __init__(self, session_maker) -> None:
+        self._preview_service = KnowledgeFSInitialSourcePreviewService(session_maker)
+
+    def start(
+        self,
+        *,
+        tenant_id: str,
+        account: Account,
+        payload: KnowledgeFSInitialWebsiteSourcePreviewPayload,
+    ) -> KnowledgeFSInitialSourcePreviewJobCreateResponse:
+        self._preview_service.require_visible_credential(tenant_id=tenant_id, account=account, payload=payload)
+        job_id = str(uuid.uuid4())
+        active_job_key = _active_job_key(tenant_id=tenant_id, account_id=account.id)
+        if not redis_client.set(active_job_key, job_id, ex=_JOB_TTL_SECONDS, nx=True):
+            raise KnowledgeFSInitialSourcePreviewJobAlreadyRunningError(
+                "A datasource preview is already running for this account"
+            )
+        try:
+            self.set_status(
+                tenant_id=tenant_id,
+                account_id=account.id,
+                job_id=job_id,
+                status="pending",
+            )
+            from tasks.knowledge_fs_initial_source_preview_tasks import run_knowledge_fs_initial_source_preview
+
+            run_knowledge_fs_initial_source_preview.apply_async(
+                kwargs={
+                    "account_id": account.id,
+                    "job_id": job_id,
+                    "payload": payload.model_dump(mode="json", by_alias=True),
+                    "tenant_id": tenant_id,
+                },
+                task_id=job_id,
+            )
+        except Exception:
+            redis_client.delete(_job_key(tenant_id=tenant_id, account_id=account.id, job_id=job_id))
+            self.release_active_job(tenant_id=tenant_id, account_id=account.id, job_id=job_id)
+            raise
+        return KnowledgeFSInitialSourcePreviewJobCreateResponse(job_id=job_id)
+
+    @staticmethod
+    def get(*, tenant_id: str, account_id: str, job_id: str) -> KnowledgeFSInitialSourcePreviewJobResponse:
+        raw = redis_client.get(_job_key(tenant_id=tenant_id, account_id=account_id, job_id=job_id))
+        if raw is None:
+            raise KnowledgeFSInitialSourcePreviewJobNotFoundError(job_id)
+        return KnowledgeFSInitialSourcePreviewJobResponse.model_validate_json(raw)
+
+    @classmethod
+    def cancel(cls, *, tenant_id: str, account_id: str, job_id: str) -> KnowledgeFSInitialSourcePreviewJobResponse:
+        current = cls.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id)
+        if current.status in {"completed", "failed"}:
+            cls.release_active_job(tenant_id=tenant_id, account_id=account_id, job_id=job_id)
+            return current
+
+        if current.status != "canceled":
+            transitioned = cls.transition_status(
+                tenant_id=tenant_id,
+                account_id=account_id,
+                job_id=job_id,
+                status="canceled",
+                allowed_from=("pending", "running"),
+            )
+            if not transitioned:
+                current = cls.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id)
+                if current.status != "canceled":
+                    return current
+
+        # Reissue termination for an already-canceled job so a client can safely retry
+        # after a lost response or broker publication failure.
+        celery_app.control.revoke(job_id, terminate=True, signal="SIGTERM")
+        response = cls.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id)
+        cls.release_active_job(tenant_id=tenant_id, account_id=account_id, job_id=job_id)
+        return response
+
+    @staticmethod
+    def release_active_job(*, tenant_id: str, account_id: str, job_id: str) -> bool:
+        released = redis_client.eval(
+            _RELEASE_ACTIVE_JOB_SCRIPT,
+            1,
+            _active_job_key(tenant_id=tenant_id, account_id=account_id),
+            job_id,
+        )
+        return bool(released)
+
+    @staticmethod
+    def set_status(
+        *,
+        tenant_id: str,
+        account_id: str,
+        job_id: str,
+        status: Literal["pending", "running", "completed", "failed", "canceled"],
+        result: KnowledgeFSInitialSourcePreviewResponse | None = None,
+    ) -> None:
+        response = KnowledgeFSInitialSourcePreviewJobResponse(job_id=job_id, result=result, status=status)
+        redis_client.setex(
+            _job_key(tenant_id=tenant_id, account_id=account_id, job_id=job_id),
+            _JOB_TTL_SECONDS,
+            json.dumps(response.model_dump(mode="json", by_alias=True), separators=(",", ":")),
+        )
+
+    @staticmethod
+    def transition_status(
+        *,
+        tenant_id: str,
+        account_id: str,
+        job_id: str,
+        status: Literal["pending", "running", "completed", "failed", "canceled"],
+        allowed_from: tuple[Literal["pending", "running", "completed", "failed", "canceled"], ...],
+        result: KnowledgeFSInitialSourcePreviewResponse | None = None,
+    ) -> bool:
+        response = KnowledgeFSInitialSourcePreviewJobResponse(job_id=job_id, result=result, status=status)
+        serialized = json.dumps(response.model_dump(mode="json", by_alias=True), separators=(",", ":"))
+        transitioned = redis_client.eval(
+            _TRANSITION_STATUS_SCRIPT,
+            1,
+            _job_key(tenant_id=tenant_id, account_id=account_id, job_id=job_id),
+            serialized,
+            _JOB_TTL_SECONDS,
+            *allowed_from,
+        )
+        return bool(transitioned)
+
+
+__all__ = [
+    "KnowledgeFSInitialSourcePreviewJobAlreadyRunningError",
+    "KnowledgeFSInitialSourcePreviewJobNotFoundError",
+    "KnowledgeFSInitialSourcePreviewJobService",
+]

api/services/knowledge_fs/product_dto.py

@@ -142,6 +142,7 @@ class KnowledgeFSInitialDatasourceBindingPayload(BaseModel):
     plugin_id: str = Field(min_length=1, max_length=255, alias="pluginId")
     provider: str = Field(min_length=1, max_length=255)
     provider_display_name: str | None = Field(default=None, min_length=1, max_length=255, alias="providerDisplayName")
+    parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50)
 
     model_config = ConfigDict(extra="forbid", validate_by_alias=True, validate_by_name=True)
 
@@ -154,6 +155,7 @@ class KnowledgeFSInitialWebsiteSourcePayload(BaseModel):
     datasource: str = Field(default="crawl", min_length=1, max_length=255)
     credential_id: str | None = Field(default=None, min_length=1, max_length=255, alias="credentialId")
     provider_display_name: str | None = Field(default=None, min_length=1, max_length=255, alias="providerDisplayName")
+    parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50)
     root_url: str = Field(min_length=1, max_length=4_096)
     crawl_options: KnowledgeFSInitialWebsiteCrawlOptionsPayload
     selection: list[KnowledgeFSInitialWebsiteSelectionPayload] = Field(min_length=1, max_length=200)
@@ -185,7 +187,16 @@ class KnowledgeFSInitialOnlineDriveSourcePayload(KnowledgeFSInitialDatasourceBin
 
 class KnowledgeFSInitialSourcePreviewPayload(KnowledgeFSInitialDatasourceBindingPayload):
     kind: Literal["online_document", "online_drive"]
-    parameters: dict[str, JsonValue] = Field(default_factory=dict, max_length=50)
+
+
+class KnowledgeFSInitialWebsiteSourcePreviewPayload(KnowledgeFSInitialDatasourceBindingPayload):
+    kind: Literal["website_crawl"]
+
+
+class KnowledgeFSInitialSourcePreviewPageResponse(ResponseModel):
+    description: str | None = None
+    source_url: str = Field(validation_alias=AliasChoices("source_url", "sourceUrl"))
+    title: str | None = None
 
 
 class KnowledgeFSInitialSourcePreviewDocumentResponse(ResponseModel):
@@ -213,11 +224,23 @@ class KnowledgeFSInitialSourcePreviewFileResponse(ResponseModel):
 class KnowledgeFSInitialSourcePreviewResponse(ResponseModel):
     documents: list[KnowledgeFSInitialSourcePreviewDocumentResponse] = Field(default_factory=list)
     files: list[KnowledgeFSInitialSourcePreviewFileResponse] = Field(default_factory=list)
-    kind: Literal["online_document", "online_drive"]
+    kind: Literal["online_document", "online_drive", "website_crawl"]
     next_page_parameters: dict[str, JsonValue] | None = Field(
         default=None,
         validation_alias=AliasChoices("next_page_parameters", "nextPageParameters"),
     )
+    pages: list[KnowledgeFSInitialSourcePreviewPageResponse] = Field(default_factory=list)
+
+
+class KnowledgeFSInitialSourcePreviewJobCreateResponse(ResponseModel):
+    job_id: str = Field(validation_alias=AliasChoices("job_id", "jobId"))
+    status: Literal["pending"] = "pending"
+
+
+class KnowledgeFSInitialSourcePreviewJobResponse(ResponseModel):
+    job_id: str = Field(validation_alias=AliasChoices("job_id", "jobId"))
+    result: KnowledgeFSInitialSourcePreviewResponse | None = None
+    status: Literal["pending", "running", "completed", "failed", "canceled"]
 
 
 KnowledgeFSInitialSourcePayload = Annotated[
@@ -2993,11 +3016,15 @@ __all__ = [
     "KnowledgeFSInitialSourcePayload",
     "KnowledgeFSInitialSourcePreviewDocumentResponse",
     "KnowledgeFSInitialSourcePreviewFileResponse",
+    "KnowledgeFSInitialSourcePreviewJobCreateResponse",
+    "KnowledgeFSInitialSourcePreviewJobResponse",
+    "KnowledgeFSInitialSourcePreviewPageResponse",
     "KnowledgeFSInitialSourcePreviewPayload",
     "KnowledgeFSInitialSourcePreviewResponse",
     "KnowledgeFSInitialWebsiteCrawlOptionsPayload",
     "KnowledgeFSInitialWebsiteSelectionPayload",
     "KnowledgeFSInitialWebsiteSourcePayload",
+    "KnowledgeFSInitialWebsiteSourcePreviewPayload",
     "KnowledgeFSJWKResponse",
     "KnowledgeFSJWKSResponse",
     "KnowledgeFSLogicalDocumentDeletePayload",

api/tasks/knowledge_fs_initial_source_preview_tasks.py

@@ -0,0 +1,119 @@
+"""Runs website datasource preview jobs outside Console request workers."""
+
+from __future__ import annotations
+
+import logging
+
+from celery import shared_task
+from sqlalchemy import select
+
+from core.db.session_factory import session_factory
+from models.account import Account, AccountStatus, TenantAccountJoin
+from services.knowledge_fs.initial_source_preview import (
+    KnowledgeFSInitialSourcePreviewCanceledError,
+    KnowledgeFSInitialSourcePreviewService,
+)
+from services.knowledge_fs.initial_source_preview_job import (
+    KnowledgeFSInitialSourcePreviewJobNotFoundError,
+    KnowledgeFSInitialSourcePreviewJobService,
+)
+from services.knowledge_fs.product_dto import KnowledgeFSInitialWebsiteSourcePreviewPayload
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(queue="dataset")
+def run_knowledge_fs_initial_source_preview(
+    *,
+    tenant_id: str,
+    account_id: str,
+    job_id: str,
+    payload: dict[str, object],
+) -> None:
+    job_service = KnowledgeFSInitialSourcePreviewJobService(session_factory.get_session_maker())
+    try:
+        if not job_service.transition_status(
+            tenant_id=tenant_id,
+            account_id=account_id,
+            job_id=job_id,
+            status="running",
+            allowed_from=("pending",),
+        ):
+            return
+        with session_factory.create_session() as session:
+            account = session.get(Account, account_id)
+            if account is None or account.status != AccountStatus.ACTIVE:
+                raise LookupError("Datasource preview account is unavailable")
+            membership = session.scalar(
+                select(TenantAccountJoin.id).where(
+                    TenantAccountJoin.tenant_id == tenant_id,
+                    TenantAccountJoin.account_id == account_id,
+                )
+            )
+            if membership is None:
+                raise PermissionError("Datasource preview account is not a tenant member")
+            account.set_tenant_id_with_session(tenant_id, session=session)
+            if account.current_tenant_id != tenant_id:
+                raise PermissionError("Datasource preview account is not a tenant member")
+            session.expunge(account)
+        result = KnowledgeFSInitialSourcePreviewService(session_factory.get_session_maker()).preview(
+            tenant_id=tenant_id,
+            account=account,
+            payload=KnowledgeFSInitialWebsiteSourcePreviewPayload.model_validate(payload),
+            is_canceled=lambda: _preview_was_canceled(
+                job_service=job_service,
+                tenant_id=tenant_id,
+                account_id=account_id,
+                job_id=job_id,
+            ),
+        )
+        job_service.transition_status(
+            tenant_id=tenant_id,
+            account_id=account_id,
+            job_id=job_id,
+            status="completed",
+            allowed_from=("running",),
+            result=result,
+        )
+    except KnowledgeFSInitialSourcePreviewCanceledError:
+        logger.info(
+            "KnowledgeFS initial source preview canceled",
+            extra={"account_id": account_id, "job_id": job_id, "tenant_id": tenant_id},
+        )
+    except Exception:
+        logger.exception(
+            "KnowledgeFS initial source preview failed",
+            extra={"account_id": account_id, "job_id": job_id, "tenant_id": tenant_id},
+        )
+        job_service.transition_status(
+            tenant_id=tenant_id,
+            account_id=account_id,
+            job_id=job_id,
+            status="failed",
+            allowed_from=("pending", "running"),
+        )
+    finally:
+        job_service.release_active_job(
+            tenant_id=tenant_id,
+            account_id=account_id,
+            job_id=job_id,
+        )
+
+
+def _preview_was_canceled(
+    *,
+    job_service: KnowledgeFSInitialSourcePreviewJobService,
+    tenant_id: str,
+    account_id: str,
+    job_id: str,
+) -> bool:
+    try:
+        return (
+            job_service.get(tenant_id=tenant_id, account_id=account_id, job_id=job_id).status
+            == "canceled"
+        )
+    except KnowledgeFSInitialSourcePreviewJobNotFoundError:
+        return True
+
+
+__all__ = ["run_knowledge_fs_initial_source_preview"]

api/tasks/knowledge_fs_initial_source_tasks.py

@@ -230,6 +230,9 @@ def _source_payload(
         "providerKind": binding.provider_kind,
         "providerName": payload.provider_display_name or payload.provider,
     }
+    if "parameters" in payload.model_fields_set:
+        metadata["datasourceParameterMode"] = "exact"
+        metadata["parameters"] = dict(payload.parameters)
     if isinstance(payload, KnowledgeFSInitialWebsiteSourcePayload):
         metadata["crawlOptions"] = {
             "includeSubpages": payload.crawl_options.include_subpages,

api/tests/unit_tests/extensions/test_knowledge_fs_celery.py

@@ -52,8 +52,22 @@ def test_celery_registers_initial_source_task_when_knowledge_fs_lifecycle_is_rea
         celery_app = init_app(DifyApp(__name__))
 
     assert "tasks.knowledge_fs_initial_source_tasks" in celery_app.conf["imports"]
+    assert "tasks.knowledge_fs_initial_source_preview_tasks" in celery_app.conf["imports"]
     assert "tasks.knowledge_fs_lifecycle_tasks" in celery_app.conf["imports"]
     assert celery_app.conf["beat_schedule"]["knowledge_fs_staged_upload_cleanup"] == {
         "task": "tasks.knowledge_fs_lifecycle_tasks.cleanup_knowledge_fs_staged_uploads",
         "schedule": timedelta(seconds=2),
     }
+
+    with (
+        patch("extensions.ext_celery.dify_config", config),
+        patch(
+            "services.knowledge_fs.lifecycle_readiness.get_configured_knowledge_fs_lifecycle_worker_readiness",
+            return_value=SimpleNamespace(ready=False),
+        ),
+    ):
+        preview_only_app = init_app(DifyApp(f"{__name__}.preview_only"))
+
+    assert "tasks.knowledge_fs_initial_source_preview_tasks" in preview_only_app.conf["imports"]
+    assert "tasks.knowledge_fs_initial_source_tasks" not in preview_only_app.conf["imports"]
+    assert "tasks.knowledge_fs_lifecycle_tasks" not in preview_only_app.conf["imports"]

api/tests/unit_tests/services/test_datasource_provider_service.py

@@ -345,7 +345,7 @@ class TestDatasourceProviderService:
 
     def test_should_fetch_by_credential_id_when_provided(self, service, sqlite_session, mock_user):
         """When credential_id is passed, the credential_id filter path (line 113) is taken."""
-        p = make_provider(credential_id="cred-id", provider="other-provider", plugin_id="other-plugin")
+        p = make_provider(credential_id="cred-id")
         persist(sqlite_session, p)
         with (
             patch("services.datasource_provider_service.get_current_user", return_value=mock_user),
@@ -354,13 +354,6 @@ class TestDatasourceProviderService:
             result = service.get_datasource_credentials("t1", "prov", "org/plug", credential_id="cred-id")
         assert result == {"k": "v"}
 
-        statement = mock_db_session.scalar.call_args.args[0]
-        sql = str(statement.compile(compile_kwargs={"literal_binds": True}))
-        assert "datasource_providers.tenant_id = 't1'" in sql
-        assert "datasource_providers.id = 'cred-id'" in sql
-        assert "datasource_providers.provider = 'prov'" in sql
-        assert "datasource_providers.plugin_id = 'org/plug'" in sql
-
     # -----------------------------------------------------------------------
     # get_all_datasource_credentials_by_provider (lines 176-228)
     # -----------------------------------------------------------------------

api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview.py

@@ -11,10 +11,19 @@ from core.datasource.entities.datasource_entities import (
     OnlineDriveBrowseFilesResponse,
     OnlineDriveFile,
     OnlineDriveFileBucket,
+    WebsiteCrawlMessage,
+    WebSiteInfo,
+    WebSiteInfoDetail,
 )
 from models.account import Account
-from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewService
-from services.knowledge_fs.product_dto import KnowledgeFSInitialSourcePreviewPayload
+from services.knowledge_fs.initial_source_preview import (
+    KnowledgeFSInitialSourcePreviewCanceledError,
+    KnowledgeFSInitialSourcePreviewService,
+)
+from services.knowledge_fs.product_dto import (
+    KnowledgeFSInitialSourcePreviewPayload,
+    KnowledgeFSInitialWebsiteSourcePreviewPayload,
+)
 
 _CREDENTIAL = object()
 
@@ -27,8 +36,13 @@ def _service(credential=_CREDENTIAL) -> tuple[KnowledgeFSInitialSourcePreviewSer
     return KnowledgeFSInitialSourcePreviewService(MagicMock(return_value=context)), session
 
 
-def _payload(kind: str) -> KnowledgeFSInitialSourcePreviewPayload:
-    return KnowledgeFSInitialSourcePreviewPayload.model_validate(
+def _payload(kind: str) -> KnowledgeFSInitialSourcePreviewPayload | KnowledgeFSInitialWebsiteSourcePreviewPayload:
+    payload_type = (
+        KnowledgeFSInitialWebsiteSourcePreviewPayload
+        if kind == "website_crawl"
+        else KnowledgeFSInitialSourcePreviewPayload
+    )
+    return payload_type.model_validate(
         {
             "credentialId": "credential-1",
             "datasource": "pages" if kind == "online_document" else "drive",
@@ -136,14 +150,95 @@ def test_preview_browses_online_drive_and_preserves_pagination() -> None:
     assert request.prefix == "folder-1"
 
 
+def test_preview_crawls_a_website_with_exact_declared_parameters() -> None:
+    service, _session = _service()
+    runtime = MagicMock()
+    runtime.datasource_provider_type.return_value = "website_crawl"
+    runtime.get_website_crawl.return_value = [
+        WebsiteCrawlMessage(
+            result=WebSiteInfo(
+                status="completed",
+                web_info_list=[
+                    WebSiteInfoDetail(
+                        content="# Result",
+                        description="Search result",
+                        source_url="https://example.com/result",
+                        title="Result",
+                    )
+                ],
+            )
+        )
+    ]
+    payload = _payload("website_crawl")
+    payload.parameters = {"query": "agentic RAG", "search_depth": "advanced"}
+    account = cast(Account, SimpleNamespace(id="account-1"))
+    with (
+        patch(
+            "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials",
+            return_value={"api_key": "secret"},
+        ) as get_credentials,
+        patch(
+            "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime",
+            return_value=runtime,
+        ),
+    ):
+        response = service.preview(
+            tenant_id="tenant-1",
+            account=account,
+            payload=payload,
+        )
+
+    assert response.kind == "website_crawl"
+    assert response.pages[0].source_url == "https://example.com/result"
+    get_credentials.assert_called_once_with(
+        tenant_id="tenant-1",
+        provider="provider",
+        plugin_id="langgenius/provider",
+        credential_id="credential-1",
+        current_user=account,
+    )
+    runtime.get_website_crawl.assert_called_once_with(
+        user_id="account-1",
+        datasource_parameters={"query": "agentic RAG", "search_depth": "advanced"},
+        provider_type="website_crawl",
+    )
+
+
+def test_preview_stops_consuming_website_results_after_cancellation() -> None:
+    service, _session = _service()
+    runtime = MagicMock()
+    runtime.datasource_provider_type.return_value = "website_crawl"
+    runtime.get_website_crawl.return_value = [
+        WebsiteCrawlMessage(result=WebSiteInfo(status="running", web_info_list=[]))
+    ]
+    is_canceled = MagicMock(side_effect=[False, True])
+    with (
+        patch(
+            "services.knowledge_fs.initial_source_preview.DatasourceProviderService.get_datasource_credentials",
+            return_value={"api_key": "secret"},
+        ),
+        patch(
+            "services.knowledge_fs.initial_source_preview.DatasourceManager.get_datasource_runtime",
+            return_value=runtime,
+        ),
+        pytest.raises(KnowledgeFSInitialSourcePreviewCanceledError),
+    ):
+        service.preview(
+            tenant_id="tenant-1",
+            account=cast(Account, SimpleNamespace(id="account-1")),
+            payload=_payload("website_crawl"),
+            is_canceled=is_canceled,
+        )
+
+    assert is_canceled.call_count == 2
+
+
 def test_preview_exposes_an_empty_drive_bucket_as_a_browsable_container() -> None:
     service, _session = _service()
     runtime = MagicMock()
     runtime.datasource_provider_type.return_value = "online_drive"
     runtime.online_drive_browse_files.return_value = [
-        OnlineDriveBrowseFilesResponse(
-            result=[OnlineDriveFileBucket(bucket="manuals", files=[], is_truncated=False)]
-        )
+        OnlineDriveBrowseFilesResponse(result=[OnlineDriveFileBucket(bucket="manuals", files=[], is_truncated=False)])
     ]
     with (
         patch(

api/tests/unit_tests/services/test_knowledge_fs_initial_source_preview_job.py

@@ -0,0 +1,203 @@
+import json
+from types import SimpleNamespace
+from typing import cast
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from models.account import Account
+from services.knowledge_fs.initial_source_preview_job import (
+    KnowledgeFSInitialSourcePreviewJobAlreadyRunningError,
+    KnowledgeFSInitialSourcePreviewJobNotFoundError,
+    KnowledgeFSInitialSourcePreviewJobService,
+)
+from services.knowledge_fs.product_dto import KnowledgeFSInitialWebsiteSourcePreviewPayload
+
+
+def _payload() -> KnowledgeFSInitialWebsiteSourcePreviewPayload:
+    return KnowledgeFSInitialWebsiteSourcePreviewPayload.model_validate(
+        {
+            "credentialId": "credential-1",
+            "datasource": "crawl",
+            "kind": "website_crawl",
+            "parameters": {"url": "https://docs.dify.ai"},
+            "pluginId": "langgenius/firecrawl_datasource",
+            "provider": "firecrawl",
+        }
+    )
+
+
+def test_start_validates_binding_and_enqueues_preview_without_running_it_inline() -> None:
+    service = KnowledgeFSInitialSourcePreviewJobService(MagicMock())
+    account = cast(Account, SimpleNamespace(id="account-1"))
+
+    with (
+        patch(
+            "services.knowledge_fs.initial_source_preview_job.KnowledgeFSInitialSourcePreviewService.require_visible_credential"
+        ) as require_credential,
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.set", return_value=True) as acquire,
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex") as setex,
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.run_knowledge_fs_initial_source_preview.apply_async"
+        ) as apply_async,
+    ):
+        response = service.start(tenant_id="tenant-1", account=account, payload=_payload())
+
+    require_credential.assert_called_once()
+    acquire.assert_called_once_with(
+        "knowledge_fs:initial_source_preview:tenant-1:account-1:active",
+        response.job_id,
+        ex=3600,
+        nx=True,
+    )
+    setex.assert_called_once()
+    assert '"status":"pending"' in setex.call_args.args[2]
+    apply_async.assert_called_once_with(
+        kwargs={
+            "account_id": "account-1",
+            "job_id": response.job_id,
+            "payload": {
+                "credentialId": "credential-1",
+                "datasource": "crawl",
+                "kind": "website_crawl",
+                "parameters": {"url": "https://docs.dify.ai"},
+                "pluginId": "langgenius/firecrawl_datasource",
+                "provider": "firecrawl",
+                "providerDisplayName": None,
+            },
+            "tenant_id": "tenant-1",
+        },
+        task_id=response.job_id,
+    )
+
+
+def test_start_rejects_a_second_active_preview_for_the_same_account() -> None:
+    service = KnowledgeFSInitialSourcePreviewJobService(MagicMock())
+    account = cast(Account, SimpleNamespace(id="account-1"))
+
+    with (
+        patch(
+            "services.knowledge_fs.initial_source_preview_job.KnowledgeFSInitialSourcePreviewService.require_visible_credential"
+        ),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.set", return_value=False),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex") as setex,
+        pytest.raises(KnowledgeFSInitialSourcePreviewJobAlreadyRunningError),
+    ):
+        service.start(tenant_id="tenant-1", account=account, payload=_payload())
+
+    setex.assert_not_called()
+
+
+def test_start_releases_the_active_preview_slot_when_enqueue_fails() -> None:
+    service = KnowledgeFSInitialSourcePreviewJobService(MagicMock())
+    account = cast(Account, SimpleNamespace(id="account-1"))
+
+    with (
+        patch(
+            "services.knowledge_fs.initial_source_preview_job.KnowledgeFSInitialSourcePreviewService.require_visible_credential"
+        ),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.set", return_value=True),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex"),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.delete") as delete,
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval", return_value=1) as redis_eval,
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.run_knowledge_fs_initial_source_preview.apply_async",
+            side_effect=RuntimeError("broker unavailable"),
+        ),
+        pytest.raises(RuntimeError, match="broker unavailable"),
+    ):
+        service.start(tenant_id="tenant-1", account=account, payload=_payload())
+
+    job_id = redis_eval.call_args.args[3]
+    delete.assert_called_once_with(
+        f"knowledge_fs:initial_source_preview:tenant-1:account-1:{job_id}"
+    )
+    assert redis_eval.call_args.args[2:] == (
+        "knowledge_fs:initial_source_preview:tenant-1:account-1:active",
+        job_id,
+    )
+
+
+def test_cancel_is_scoped_to_the_requesting_tenant_and_account() -> None:
+    cache: dict[str, str] = {}
+
+    def setex(key: str, _ttl: int, value: str) -> None:
+        cache[key] = value
+
+    def transition(
+        _script: str,
+        _key_count: int,
+        key: str,
+        value: str,
+        _ttl: int | None = None,
+        *allowed_from: str,
+    ) -> int:
+        if _ttl is None:
+            return 0
+        current = cache.get(key)
+        if current is None or json.loads(current)["status"] not in allowed_from:
+            return 0
+        cache[key] = value
+        return 1
+
+    with (
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.get", side_effect=cache.get),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.setex", side_effect=setex),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval", side_effect=transition),
+        patch("services.knowledge_fs.initial_source_preview_job.celery_app.control.revoke") as revoke,
+    ):
+        KnowledgeFSInitialSourcePreviewJobService.set_status(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            status="running",
+        )
+        response = KnowledgeFSInitialSourcePreviewJobService.cancel(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+        )
+        with pytest.raises(KnowledgeFSInitialSourcePreviewJobNotFoundError):
+            KnowledgeFSInitialSourcePreviewJobService.get(
+                tenant_id="tenant-1",
+                account_id="account-2",
+                job_id="job-1",
+            )
+
+    assert response.status == "canceled"
+    revoke.assert_called_once_with("job-1", terminate=True, signal="SIGTERM")
+
+
+def test_cancel_reissues_termination_for_an_already_canceled_job() -> None:
+    canceled = json.dumps({"jobId": "job-1", "result": None, "status": "canceled"})
+
+    with (
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.get", return_value=canceled),
+        patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval") as redis_eval,
+        patch("services.knowledge_fs.initial_source_preview_job.celery_app.control.revoke") as revoke,
+    ):
+        response = KnowledgeFSInitialSourcePreviewJobService.cancel(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+        )
+
+    assert response.status == "canceled"
+    redis_eval.assert_called_once()
+    revoke.assert_called_once_with("job-1", terminate=True, signal="SIGTERM")
+
+
+def test_terminal_canceled_status_cannot_be_overwritten() -> None:
+    with patch("services.knowledge_fs.initial_source_preview_job.redis_client.eval", return_value=0) as eval_status:
+        transitioned = KnowledgeFSInitialSourcePreviewJobService.transition_status(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            status="completed",
+            allowed_from=("running",),
+            result=None,
+        )
+
+    assert transitioned is False
+    assert '"status":"completed"' in eval_status.call_args.args[3]
+    assert eval_status.call_args.args[5:] == ("running",)

api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_preview_tasks.py

@@ -0,0 +1,224 @@
+from types import SimpleNamespace
+from unittest.mock import MagicMock, call, patch
+
+from models.account import AccountStatus
+from services.knowledge_fs.initial_source_preview import KnowledgeFSInitialSourcePreviewCanceledError
+from services.knowledge_fs.product_dto import KnowledgeFSInitialSourcePreviewResponse
+from tasks.knowledge_fs_initial_source_preview_tasks import run_knowledge_fs_initial_source_preview
+
+
+def _payload() -> dict[str, object]:
+    return {
+        "credentialId": "credential-1",
+        "datasource": "crawl",
+        "kind": "website_crawl",
+        "parameters": {"url": "https://docs.dify.ai"},
+        "pluginId": "langgenius/firecrawl_datasource",
+        "provider": "firecrawl",
+    }
+
+
+def test_preview_task_uses_the_dataset_queue() -> None:
+    assert run_knowledge_fs_initial_source_preview.queue == "dataset"
+
+
+def test_preview_task_persists_running_and_completed_states() -> None:
+    account = SimpleNamespace(
+        current_tenant_id="tenant-1",
+        id="account-1",
+        set_tenant_id_with_session=MagicMock(),
+        status=AccountStatus.ACTIVE,
+    )
+    session = MagicMock()
+    session.get.return_value = account
+    session.scalar.return_value = "membership-1"
+    session_context = MagicMock()
+    session_context.__enter__.return_value = session
+    job_service = MagicMock()
+    job_service.transition_status.side_effect = [True, True]
+    preview_service = MagicMock()
+    result = KnowledgeFSInitialSourcePreviewResponse(kind="website_crawl")
+    preview_service.preview.return_value = result
+
+    with (
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService",
+            return_value=job_service,
+        ),
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService",
+            return_value=preview_service,
+        ),
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.session_factory.create_session",
+            return_value=session_context,
+        ),
+        patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"),
+    ):
+        run_knowledge_fs_initial_source_preview.run(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            payload=_payload(),
+        )
+
+    assert job_service.transition_status.call_args_list == [
+        call(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            status="running",
+            allowed_from=("pending",),
+        ),
+        call(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            status="completed",
+            allowed_from=("running",),
+            result=result,
+        ),
+    ]
+    job_service.release_active_job.assert_called_once_with(
+        tenant_id="tenant-1",
+        account_id="account-1",
+        job_id="job-1",
+    )
+    account.set_tenant_id_with_session.assert_called_once_with("tenant-1", session=session)
+    session.expunge.assert_called_once_with(account)
+    preview_service.preview.assert_called_once()
+
+
+def test_preview_task_does_not_start_a_canceled_job() -> None:
+    job_service = MagicMock()
+    job_service.transition_status.return_value = False
+
+    with (
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService",
+            return_value=job_service,
+        ),
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService"
+        ) as preview_service,
+        patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"),
+    ):
+        run_knowledge_fs_initial_source_preview.run(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            payload=_payload(),
+        )
+
+    job_service.transition_status.assert_called_once_with(
+        tenant_id="tenant-1",
+        account_id="account-1",
+        job_id="job-1",
+        status="running",
+        allowed_from=("pending",),
+    )
+    preview_service.assert_not_called()
+    job_service.release_active_job.assert_called_once_with(
+        tenant_id="tenant-1",
+        account_id="account-1",
+        job_id="job-1",
+    )
+
+
+def test_preview_task_keeps_a_cooperatively_canceled_job_terminal() -> None:
+    account = SimpleNamespace(
+        current_tenant_id="tenant-1",
+        id="account-1",
+        set_tenant_id_with_session=MagicMock(),
+        status=AccountStatus.ACTIVE,
+    )
+    session = MagicMock()
+    session.get.return_value = account
+    session.scalar.return_value = "membership-1"
+    session_context = MagicMock()
+    session_context.__enter__.return_value = session
+    job_service = MagicMock()
+    job_service.transition_status.return_value = True
+    job_service.get.return_value = SimpleNamespace(status="canceled")
+    preview_service = MagicMock()
+
+    def preview(**kwargs):
+        assert kwargs["is_canceled"]() is True
+        raise KnowledgeFSInitialSourcePreviewCanceledError("canceled")
+
+    preview_service.preview.side_effect = preview
+    with (
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService",
+            return_value=job_service,
+        ),
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService",
+            return_value=preview_service,
+        ),
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.session_factory.create_session",
+            return_value=session_context,
+        ),
+        patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"),
+    ):
+        run_knowledge_fs_initial_source_preview.run(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            payload=_payload(),
+        )
+
+    job_service.transition_status.assert_called_once_with(
+        tenant_id="tenant-1",
+        account_id="account-1",
+        job_id="job-1",
+        status="running",
+        allowed_from=("pending",),
+    )
+
+
+def test_preview_task_rejects_an_account_removed_from_the_tenant() -> None:
+    account = SimpleNamespace(
+        current_tenant_id=None,
+        id="account-1",
+        set_tenant_id_with_session=MagicMock(),
+        status=AccountStatus.ACTIVE,
+    )
+    session = MagicMock()
+    session.get.return_value = account
+    session.scalar.return_value = None
+    session_context = MagicMock()
+    session_context.__enter__.return_value = session
+    job_service = MagicMock()
+    job_service.transition_status.side_effect = [True, True]
+
+    with (
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewJobService",
+            return_value=job_service,
+        ),
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.KnowledgeFSInitialSourcePreviewService"
+        ) as preview_service,
+        patch(
+            "tasks.knowledge_fs_initial_source_preview_tasks.session_factory.create_session",
+            return_value=session_context,
+        ),
+        patch("tasks.knowledge_fs_initial_source_preview_tasks.session_factory.get_session_maker"),
+    ):
+        run_knowledge_fs_initial_source_preview.run(
+            tenant_id="tenant-1",
+            account_id="account-1",
+            job_id="job-1",
+            payload=_payload(),
+        )
+
+    preview_service.assert_not_called()
+    assert job_service.transition_status.call_args_list[-1] == call(
+        tenant_id="tenant-1",
+        account_id="account-1",
+        job_id="job-1",
+        status="failed",
+        allowed_from=("pending", "running"),
+    )

api/tests/unit_tests/tasks/test_knowledge_fs_initial_source_tasks.py

@@ -29,6 +29,7 @@ def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePaylo
             "name": "Dify docs",
             "provider": "firecrawl",
             "providerDisplayName": "Firecrawl",
+            "parameters": {"url": "https://docs.dify.ai", "limit": 25},
             "root_url": "https://docs.dify.ai",
             "crawl_options": {"include_subpages": True, "limit": 25},
             "selection": [
@@ -40,6 +41,21 @@ def _payload(sync_policy: str = "daily") -> KnowledgeFSInitialWebsiteSourcePaylo
     )
 
 
+def _legacy_payload() -> KnowledgeFSInitialWebsiteSourcePayload:
+    return KnowledgeFSInitialWebsiteSourcePayload.model_validate(
+        {
+            "kind": "website_crawl",
+            "name": "Dify docs",
+            "provider": "firecrawl",
+            "providerDisplayName": "Firecrawl",
+            "root_url": "https://docs.dify.ai",
+            "crawl_options": {"include_subpages": False, "limit": 25},
+            "selection": [{"source_url": "https://docs.dify.ai/a", "title": "A"}],
+            "sync_policy": "manual",
+        }
+    )
+
+
 def _document_payload() -> KnowledgeFSInitialOnlineDocumentSourcePayload:
     return KnowledgeFSInitialOnlineDocumentSourcePayload.model_validate(
         {
@@ -205,7 +221,12 @@ def test_initial_website_source_import_recrawls_exact_selection_and_configures_d
     source_payload = facade.create_source.call_args.kwargs["payload"]
     assert source_payload.status == "disabled"
     assert source_payload.connection_id == "connection-1"
+    assert source_payload.metadata["datasourceParameterMode"] == "exact"
     assert source_payload.metadata["preview"] is True
+    assert source_payload.metadata["parameters"] == {
+        "limit": 25,
+        "url": "https://docs.dify.ai",
+    }
     import_payload = facade.import_selected_source_crawl.call_args.kwargs["payload"]
     assert import_payload.source_urls == [
         "https://docs.dify.ai/a",
@@ -225,6 +246,21 @@ def test_initial_website_source_import_recrawls_exact_selection_and_configures_d
     assert sync_payload.expected_source_version == 4
 
 
+def test_initial_website_source_import_preserves_legacy_crawl_option_projection() -> None:
+    facade = _facade()
+
+    assert _start(facade, _legacy_payload()) == "workflow-1"
+
+    source_payload = facade.create_source.call_args.kwargs["payload"]
+    assert "datasourceParameterMode" not in source_payload.metadata
+    assert "parameters" not in source_payload.metadata
+    assert source_payload.metadata["crawlOptions"] == {
+        "includeSubpages": False,
+        "limit": 25,
+    }
+    assert source_payload.uri == "https://docs.dify.ai"
+
+
 @pytest.mark.parametrize(
     (
         "payload",

knowledge-fs/apps/api/src/dify-datasource-invocation-client.test.ts

@@ -3,6 +3,7 @@ import type { DifyDatasourceRuntimeClient } from "@knowledge/dify-datasource-run
 import { describe, expect, it, vi } from "vitest";
 
 import { createDifyDatasourceInvocationClient } from "./dify-datasource-invocation-client";
+import { createApiOnlineDriveConnector } from "./online-drive-options";
 
 const SOURCE: Source = {
   createdAt: "2026-07-03T00:00:00.000Z",
@@ -61,7 +62,7 @@ describe("createDifyDatasourceInvocationClient", () => {
     expect(JSON.stringify(getOnlineDocumentPages.mock.calls)).not.toContain("credentials");
   });
 
-  it("maps product crawl options to the Firecrawl datasource parameters", async () => {
+  it("maps legacy product crawl options to the Firecrawl datasource parameters", async () => {
     const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } }));
     const adapter = createDifyDatasourceInvocationClient({
       client: difyClient({ getWebsiteCrawl }),
@@ -103,6 +104,153 @@ describe("createDifyDatasourceInvocationClient", () => {
     });
   });
 
+  it("preserves exact datasource parameters for declaration-driven website sources", async () => {
+    const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } }));
+    const adapter = createDifyDatasourceInvocationClient({
+      client: difyClient({ getWebsiteCrawl }),
+    });
+    const source: Source = {
+      ...SOURCE,
+      metadata: {
+        credentialId: "dify-credential-1",
+        crawlOptions: { includeSubpages: false, limit: 1 },
+        datasource: "search_extract",
+        datasourceParameterMode: "exact",
+        parameters: { query: "dify knowledge", search_depth: "advanced" },
+        pluginId: "langgenius/tavily_datasource",
+        provider: "tavily",
+      },
+      type: "web",
+      uri: "datasource://tavily",
+    };
+
+    await collect(
+      adapter.dispatch({
+        operation: "get_website_crawl",
+        source,
+        tenantId: "tenant-1",
+      }),
+    );
+
+    expect(getWebsiteCrawl).toHaveBeenCalledWith({
+      credentialId: "dify-credential-1",
+      datasource: "search_extract",
+      datasourceParameters: { query: "dify knowledge", search_depth: "advanced" },
+      pluginId: "langgenius/tavily_datasource",
+      provider: "tavily",
+      tenantId: "tenant-1",
+    });
+  });
+
+  it("preserves an explicitly empty declaration-driven parameter set", async () => {
+    const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } }));
+    const adapter = createDifyDatasourceInvocationClient({
+      client: difyClient({ getWebsiteCrawl }),
+    });
+    const source: Source = {
+      ...SOURCE,
+      metadata: {
+        credentialId: "dify-credential-1",
+        crawlOptions: { includeSubpages: true, limit: 200 },
+        datasource: "optional_search",
+        datasourceParameterMode: "exact",
+        parameters: {},
+        pluginId: "langgenius/optional_search_datasource",
+        provider: "optional_search",
+      },
+      type: "web",
+      uri: "datasource://optional-search",
+    };
+
+    await collect(
+      adapter.dispatch({
+        operation: "get_website_crawl",
+        source,
+        tenantId: "tenant-1",
+      }),
+    );
+
+    expect(getWebsiteCrawl).toHaveBeenCalledWith(
+      expect.objectContaining({ datasourceParameters: {} }),
+    );
+  });
+
+  it("maps legacy Jina crawl options to crawl_sub_pages", async () => {
+    const getWebsiteCrawl = vi.fn(() => chunks({ result: { web_info_list: [] } }));
+    const adapter = createDifyDatasourceInvocationClient({
+      client: difyClient({ getWebsiteCrawl }),
+    });
+    const source: Source = {
+      ...SOURCE,
+      metadata: {
+        credentialId: "jina-credential-1",
+        crawlOptions: { includeSubpages: true, limit: 10 },
+        datasource: "jina_reader",
+        parameters: {},
+        pluginId: "langgenius/jina_datasource",
+        provider: "jinareader",
+      },
+      type: "web",
+      uri: "https://example.com",
+    };
+
+    await collect(
+      adapter.dispatch({
+        operation: "get_website_crawl",
+        source,
+        tenantId: "tenant-1",
+      }),
+    );
+
+    expect(getWebsiteCrawl).toHaveBeenCalledWith(
+      expect.objectContaining({
+        datasourceParameters: {
+          crawl_sub_pages: true,
+          limit: 10,
+          url: "https://example.com",
+        },
+      }),
+    );
+  });
+
+  it("uses declaration-driven online drive parameters as the browse root", async () => {
+    const browseOnlineDrive = vi.fn(() => chunks({ result: [] }));
+    const adapter = createDifyDatasourceInvocationClient({
+      client: difyClient({ browseOnlineDrive }),
+    });
+    const source: Source = {
+      ...SOURCE,
+      metadata: {
+        ...SOURCE.metadata,
+        datasource: "shared_drive",
+        parameters: {
+          bucket: "manuals",
+          max_keys: 50,
+          next_page_parameters: { cursor: "saved" },
+          prefix: "products/",
+        },
+        providerKind: "online-drive",
+      },
+      uri: "gdrive://shared-drive",
+    };
+
+    const connector = createApiOnlineDriveConnector({ client: adapter });
+
+    await connector.browse({ source, tenantId: "tenant-1" });
+
+    expect(browseOnlineDrive).toHaveBeenCalledWith({
+      bucket: "manuals",
+      credentialId: "dify-credential-1",
+      datasource: "shared_drive",
+      maxKeys: 50,
+      nextPageParameters: { cursor: "saved" },
+      pluginId: "langgenius/notion_datasource",
+      prefix: "products/",
+      provider: "notion_datasource",
+      tenantId: "tenant-1",
+    });
+  });
+
   it("rejects inline credentials in integrated mode", async () => {
     const adapter = createDifyDatasourceInvocationClient({ client: difyClient() });
     const source = {

knowledge-fs/apps/api/src/dify-datasource-invocation-client.ts

@@ -9,6 +9,8 @@ import type {
   ApiDatasourceInvocationInput,
 } from "./datasource-invocation-client";
 
+const DEFAULT_ONLINE_DRIVE_MAX_KEYS = 20;
+
 interface DifyDatasourceSourceConfig {
   readonly credentialId: string;
   readonly datasource: string;
@@ -39,7 +41,7 @@ export function createDifyDatasourceInvocationClient(input: {
           yield* input.client.getWebsiteCrawl({
             ...common,
             datasourceParameters: withCrawlOptions(
-              config.parameters,
+              config,
               invocation.source,
               invocation.source.uri,
             ),
@@ -61,17 +63,26 @@ export function createDifyDatasourceInvocationClient(input: {
             page: invocation.page,
           });
           return;
-        case "online_drive_browse_files":
+        case "online_drive_browse_files": {
+          const sourceBucket = optionalString(config.parameters.bucket);
+          const sourcePrefix = optionalString(config.parameters.prefix);
+          const sourceMaxKeys = positiveSafeInteger(config.parameters.max_keys);
+          const sourceNextPageParameters = plainObject(config.parameters.next_page_parameters);
           yield* input.client.browseOnlineDrive({
             ...common,
-            ...(invocation.bucket === undefined ? {} : { bucket: invocation.bucket }),
-            ...(invocation.continuationToken === undefined
+            ...(invocation.bucket === undefined && sourceBucket === undefined
               ? {}
+              : { bucket: invocation.bucket ?? sourceBucket }),
+            ...(invocation.continuationToken === undefined
+              ? Object.keys(sourceNextPageParameters).length
+                ? { nextPageParameters: sourceNextPageParameters }
+                : {}
               : { nextPageParameters: decodeNextPageParameters(invocation.continuationToken) }),
-            ...(invocation.maxKeys === undefined ? {} : { maxKeys: invocation.maxKeys }),
-            prefix: invocation.prefix ?? "",
+            maxKeys: invocation.maxKeys ?? sourceMaxKeys ?? DEFAULT_ONLINE_DRIVE_MAX_KEYS,
+            prefix: invocation.prefix || sourcePrefix || "",
           });
           return;
+        }
         case "online_drive_download_file":
           yield* input.client.downloadOnlineDriveFile({ ...common, file: invocation.file });
           return;
@@ -149,18 +160,37 @@ function withCrawlUrl(parameters: Record<string, unknown>, uri: string): Record<
     : { ...parameters, url: uri };
 }
 
+function optionalString(value: unknown): string | undefined {
+  return typeof value === "string" && value.trim() ? value : undefined;
+}
+
+function positiveSafeInteger(value: unknown): number | undefined {
+  return Number.isSafeInteger(value) && Number(value) > 0 ? Number(value) : undefined;
+}
+
 function withCrawlOptions(
-  parameters: Record<string, unknown>,
+  config: DifyDatasourceSourceConfig,
   source: Source,
   uri: string,
 ): Record<string, unknown> {
+  const parameters = config.parameters;
+  // New datasource-driven Sources persist the exact plugin parameters. Preserve the legacy
+  // URL, provider-specific subpage flag, and limit projection only for older Sources.
+  if (source.metadata.datasourceParameterMode === "exact") {
+    return parameters;
+  }
   const crawlOptions = plainObject(source.metadata.crawlOptions);
   const includeSubpages = crawlOptions.includeSubpages;
   const limit = crawlOptions.limit;
+  const subpagesParameter = /jina|watercrawl/u.test(
+    `${config.pluginId} ${config.provider} ${config.datasource}`.toLowerCase(),
+  )
+    ? "crawl_sub_pages"
+    : "crawl_subpages";
   return withCrawlUrl(
     {
       ...parameters,
-      ...(typeof includeSubpages === "boolean" ? { crawl_subpages: includeSubpages } : {}),
+      ...(typeof includeSubpages === "boolean" ? { [subpagesParameter]: includeSubpages } : {}),
       ...(Number.isSafeInteger(limit) && Number(limit) > 0 ? { limit } : {}),
     },
     uri,

knowledge-fs/apps/api/src/online-drive-options.test.ts

@@ -91,12 +91,12 @@ describe("createApiOnlineDriveConnector", () => {
     ]);
     expect(calls[0]).toMatchObject({
       bucket: "b1",
-      maxKeys: 20,
       operation: "online_drive_browse_files",
       prefix: "docs/",
       source: SOURCE,
       tenantId: "tenant-1",
     });
+    expect(calls[0]).not.toHaveProperty("maxKeys");
     expect(JSON.stringify(calls[0])).not.toContain("credentials");
   });
 

knowledge-fs/apps/api/src/online-drive-options.ts

@@ -6,8 +6,6 @@ import type {
 
 import type { ApiDatasourceInvocationClient } from "./datasource-invocation-client";
 
-const DEFAULT_MAX_KEYS = 20;
-
 /**
  * Online-drive connector backed by the deployment-selected datasource runtime. Downloaded file bytes arrive as
  * ToolInvokeMessage
@@ -33,7 +31,7 @@ export function createApiOnlineDriveConnector(input: {
       for await (const raw of input.client.dispatch({
         ...(bucket === undefined ? {} : { bucket }),
         ...(continuationToken === undefined ? {} : { continuationToken }),
-        maxKeys: maxKeys ?? DEFAULT_MAX_KEYS,
+        ...(maxKeys === undefined ? {} : { maxKeys }),
         operation: "online_drive_browse_files",
         prefix: prefix ?? "",
         source,

packages/contracts/generated/api/console/knowledge-fs/types.gen.ts

@@ -33,10 +33,34 @@ export type KnowledgeFsInitialSourcePreviewPayload = {
 export type KnowledgeFsInitialSourcePreviewResponse = {
   documents?: Array<KnowledgeFsInitialSourcePreviewDocumentResponse>
   files?: Array<KnowledgeFsInitialSourcePreviewFileResponse>
-  kind: 'online_document' | 'online_drive'
+  kind: 'online_document' | 'online_drive' | 'website_crawl'
   next_page_parameters?: {
     [key: string]: JsonValue
   } | null
+  pages?: Array<KnowledgeFsInitialSourcePreviewPageResponse>
+}
+
+export type KnowledgeFsInitialWebsiteSourcePreviewPayload = {
+  credentialId: string
+  datasource: string
+  kind: 'website_crawl'
+  parameters?: {
+    [key: string]: JsonValue
+  }
+  pluginId: string
+  provider: string
+  providerDisplayName?: string | null
+}
+
+export type KnowledgeFsInitialSourcePreviewJobCreateResponse = {
+  job_id: string
+  status?: 'pending'
+}
+
+export type KnowledgeFsInitialSourcePreviewJobResponse = {
+  job_id: string
+  result?: KnowledgeFsInitialSourcePreviewResponse | null
+  status: 'canceled' | 'completed' | 'failed' | 'pending' | 'running'
 }
 
 export type KnowledgeFsSpaceListResponse = {
@@ -1040,6 +1064,12 @@ export type KnowledgeFsInitialSourcePreviewFileResponse = {
   type: string
 }
 
+export type KnowledgeFsInitialSourcePreviewPageResponse = {
+  description?: string | null
+  source_url: string
+  title?: string | null
+}
+
 export type KnowledgeFsSpaceListItemResponse = {
   control_space_id: string
   created_at: string
@@ -1067,6 +1097,9 @@ export type KnowledgeFsInitialWebsiteSourcePayload = {
   datasource?: string
   kind: 'website_crawl'
   name: string
+  parameters?: {
+    [key: string]: JsonValue
+  }
   pluginId?: string | null
   provider: string
   providerDisplayName?: string | null
@@ -1080,6 +1113,9 @@ export type KnowledgeFsInitialOnlineDocumentSourcePayload = {
   datasource: string
   kind: 'online_document'
   name: string
+  parameters?: {
+    [key: string]: JsonValue
+  }
   pluginId: string
   provider: string
   providerDisplayName?: string | null
@@ -1092,6 +1128,9 @@ export type KnowledgeFsInitialOnlineDriveSourcePayload = {
   datasource: string
   kind: 'online_drive'
   name: string
+  parameters?: {
+    [key: string]: JsonValue
+  }
   pluginId: string
   provider: string
   providerDisplayName?: string | null
@@ -1820,6 +1859,52 @@ export type PostKnowledgeFsSourceProviderPreviewResponses = {
 export type PostKnowledgeFsSourceProviderPreviewResponse =
   PostKnowledgeFsSourceProviderPreviewResponses[keyof PostKnowledgeFsSourceProviderPreviewResponses]
 
+export type PostKnowledgeFsSourceProviderPreviewJobsData = {
+  body: KnowledgeFsInitialWebsiteSourcePreviewPayload
+  path?: never
+  query?: never
+  url: '/knowledge-fs/source-provider-preview/jobs'
+}
+
+export type PostKnowledgeFsSourceProviderPreviewJobsResponses = {
+  202: KnowledgeFsInitialSourcePreviewJobCreateResponse
+}
+
+export type PostKnowledgeFsSourceProviderPreviewJobsResponse =
+  PostKnowledgeFsSourceProviderPreviewJobsResponses[keyof PostKnowledgeFsSourceProviderPreviewJobsResponses]
+
+export type DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdData = {
+  body?: never
+  path: {
+    job_id: string
+  }
+  query?: never
+  url: '/knowledge-fs/source-provider-preview/jobs/{job_id}'
+}
+
+export type DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponses = {
+  200: KnowledgeFsInitialSourcePreviewJobResponse
+}
+
+export type DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponse =
+  DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponses[keyof DeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponses]
+
+export type GetKnowledgeFsSourceProviderPreviewJobsByJobIdData = {
+  body?: never
+  path: {
+    job_id: string
+  }
+  query?: never
+  url: '/knowledge-fs/source-provider-preview/jobs/{job_id}'
+}
+
+export type GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponses = {
+  200: KnowledgeFsInitialSourcePreviewJobResponse
+}
+
+export type GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponse =
+  GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponses[keyof GetKnowledgeFsSourceProviderPreviewJobsByJobIdResponses]
+
 export type GetKnowledgeFsSpacesData = {
   body?: never
   path?: never

packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts

@@ -2,6 +2,14 @@
 
 import * as z from 'zod'
 
+/**
+ * KnowledgeFSInitialSourcePreviewJobCreateResponse
+ */
+export const zKnowledgeFsInitialSourcePreviewJobCreateResponse = z.object({
+  job_id: z.string(),
+  status: z.literal('pending').optional().default('pending'),
+})
+
 /**
  * KnowledgeFSBulkJobResponse
  */
@@ -755,6 +763,19 @@ export const zKnowledgeFsInitialSourcePreviewPayload = z.object({
   providerDisplayName: z.string().min(1).max(255).nullish(),
 })
 
+/**
+ * KnowledgeFSInitialWebsiteSourcePreviewPayload
+ */
+export const zKnowledgeFsInitialWebsiteSourcePreviewPayload = z.object({
+  credentialId: z.string().min(1).max(255),
+  datasource: z.string().min(1).max(255),
+  kind: z.literal('website_crawl'),
+  parameters: z.record(z.string(), zJsonValue).optional(),
+  pluginId: z.string().min(1).max(255),
+  provider: z.string().min(1).max(255),
+  providerDisplayName: z.string().min(1).max(255).nullish(),
+})
+
 /**
  * KnowledgeFSInitialSourcePreviewDocumentResponse
  */
@@ -781,14 +802,33 @@ export const zKnowledgeFsInitialSourcePreviewFileResponse = z.object({
   type: z.string(),
 })
 
+/**
+ * KnowledgeFSInitialSourcePreviewPageResponse
+ */
+export const zKnowledgeFsInitialSourcePreviewPageResponse = z.object({
+  description: z.string().nullish(),
+  source_url: z.string(),
+  title: z.string().nullish(),
+})
+
 /**
  * KnowledgeFSInitialSourcePreviewResponse
  */
 export const zKnowledgeFsInitialSourcePreviewResponse = z.object({
   documents: z.array(zKnowledgeFsInitialSourcePreviewDocumentResponse).optional(),
   files: z.array(zKnowledgeFsInitialSourcePreviewFileResponse).optional(),
-  kind: z.enum(['online_document', 'online_drive']),
+  kind: z.enum(['online_document', 'online_drive', 'website_crawl']),
   next_page_parameters: z.record(z.string(), zJsonValue).nullish(),
+  pages: z.array(zKnowledgeFsInitialSourcePreviewPageResponse).optional(),
+})
+
+/**
+ * KnowledgeFSInitialSourcePreviewJobResponse
+ */
+export const zKnowledgeFsInitialSourcePreviewJobResponse = z.object({
+  job_id: z.string(),
+  result: zKnowledgeFsInitialSourcePreviewResponse.nullish(),
+  status: z.enum(['canceled', 'completed', 'failed', 'pending', 'running']),
 })
 
 /**
@@ -1742,6 +1782,7 @@ export const zKnowledgeFsInitialWebsiteSourcePayload = z.object({
   datasource: z.string().min(1).max(255).optional().default('crawl'),
   kind: z.literal('website_crawl'),
   name: z.string().min(1).max(200),
+  parameters: z.record(z.string(), zJsonValue).optional(),
   pluginId: z.string().min(1).max(255).nullish(),
   provider: z.string().min(1).max(255),
   providerDisplayName: z.string().min(1).max(255).nullish(),
@@ -1771,6 +1812,7 @@ export const zKnowledgeFsInitialOnlineDocumentSourcePayload = z.object({
   datasource: z.string().min(1).max(255),
   kind: z.literal('online_document'),
   name: z.string().min(1).max(200),
+  parameters: z.record(z.string(), zJsonValue).optional(),
   pluginId: z.string().min(1).max(255),
   provider: z.string().min(1).max(255),
   providerDisplayName: z.string().min(1).max(255).nullish(),
@@ -1806,6 +1848,7 @@ export const zKnowledgeFsInitialOnlineDriveSourcePayload = z.object({
   datasource: z.string().min(1).max(255),
   kind: z.literal('online_drive'),
   name: z.string().min(1).max(200),
+  parameters: z.record(z.string(), zJsonValue).optional(),
   pluginId: z.string().min(1).max(255),
   provider: z.string().min(1).max(255),
   providerDisplayName: z.string().min(1).max(255).nullish(),
@@ -2401,6 +2444,35 @@ export const zPostKnowledgeFsSourceProviderPreviewBody = zKnowledgeFsInitialSour
 export const zPostKnowledgeFsSourceProviderPreviewResponse =
   zKnowledgeFsInitialSourcePreviewResponse
 
+export const zPostKnowledgeFsSourceProviderPreviewJobsBody =
+  zKnowledgeFsInitialWebsiteSourcePreviewPayload
+
+/**
+ * Website datasource preview queued
+ */
+export const zPostKnowledgeFsSourceProviderPreviewJobsResponse =
+  zKnowledgeFsInitialSourcePreviewJobCreateResponse
+
+export const zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdPath = z.object({
+  job_id: z.string(),
+})
+
+/**
+ * Website datasource preview canceled
+ */
+export const zDeleteKnowledgeFsSourceProviderPreviewJobsByJobIdResponse =
+  zKnowledgeFsInitialSourcePreviewJobResponse
+
+export const zGetKnowledgeFsSourceProviderPreviewJobsByJobIdPath = z.object({
+  job_id: z.string(),
+})
+
+/**
+ * Website datasource preview status
+ */
+export const zGetKnowledgeFsSourceProviderPreviewJobsByJobIdResponse =
+  zKnowledgeFsInitialSourcePreviewJobResponse
+
 export const zGetKnowledgeFsSpacesQuery = z.object({
   creator_ids: z.array(z.string().min(1).max(255)).max(100).optional(),
   limit: z.int().gte(1).lte(100).optional().default(20),

web/features/new-rag/__tests__/add-source-page.spec.tsx

@@ -676,7 +676,6 @@ describe('AddSourcePage', () => {
   })
 
   it('restores a website draft handed off by the creation flow', async () => {
-    const user = userEvent.setup()
     queryState.connections.data = { pages: [{ items: [connection('active')] }] }
 
     render(
@@ -700,11 +699,12 @@ describe('AddSourcePage', () => {
     expect(screen.getByRole('textbox', { name: /dataset\.newKnowledge\.sourceName/ })).toHaveValue(
       'Dify docs',
     )
-    await user.click(screen.getByRole('button', { name: /dataset\.newKnowledge\.crawlOptions/ }))
     expect(
-      screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' }),
+      screen.getByRole('switch', { name: 'dataset.newKnowledge.includeSubpages' }),
     ).not.toBeChecked()
-    expect(screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })).toHaveValue('25')
+    expect(screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })).toHaveValue(
+      25,
+    )
   })
 
   it('keeps the exact website provider selected while loading website dependencies', async () => {
@@ -1061,6 +1061,57 @@ describe('AddSourcePage', () => {
     ).toBeEnabled()
   })
 
+  it('clears website parameters when switching between configured providers', async () => {
+    const user = userEvent.setup()
+    queryState.providers.data = { items: [difyManagedFirecrawlProvider] }
+    queryState.datasourcePlugins.data = [firecrawlDatasourcePlugin, jinaDatasourcePlugin]
+    queryState.datasourceAuth.data = { result: [firecrawlDatasourceAuth, jinaDatasourceAuth] }
+    queryState.connections.data = {
+      pages: [
+        {
+          items: [
+            {
+              ...connection('active'),
+              authKind: 'endpoint',
+              configuration: {
+                credentialId: 'firecrawl-credential-1',
+                datasource: 'crawl',
+                pluginId: 'langgenius/firecrawl_datasource',
+                provider: 'firecrawl',
+                providerKind: 'website',
+              },
+            },
+            {
+              ...connection('active'),
+              authKind: 'endpoint',
+              configuration: {
+                credentialId: 'jina-credential-1',
+                datasource: 'jina_reader',
+                pluginId: 'langgenius/jina_datasource',
+                provider: 'jinareader',
+                providerKind: 'website',
+              },
+              id: 'jina-connection-1',
+              name: 'Jina Reader',
+            },
+          ],
+        },
+      ],
+    }
+
+    render(<AddSourcePage knowledgeSpaceId="space-1" />)
+    const rootUrl = await screen.findByRole('textbox', {
+      name: /dataset\.newKnowledge\.rootUrl/,
+    })
+    await user.type(rootUrl, 'https://firecrawl.example.com')
+
+    await user.click(screen.getByRole('radio', { name: 'Jina Reader' }))
+
+    expect(
+      await screen.findByRole('textbox', { name: /dataset\.newKnowledge\.rootUrl/ }),
+    ).toHaveValue('')
+  })
+
   it('opens Data Source settings when Dify has no Firecrawl credential', async () => {
     const user = userEvent.setup()
     queryState.providers.data = { items: [difyManagedFirecrawlProvider] }

web/features/new-rag/__tests__/connected-source-setup.spec.tsx

@@ -18,6 +18,7 @@ import type { DataSourceItem } from '@/app/components/workflow/block-selector/ty
 import { QueryClientProvider } from '@tanstack/react-query'
 import { act, screen, waitFor } from '@testing-library/react'
 import userEvent from '@testing-library/user-event'
+import { useState } from 'react'
 import { render } from '@/test/console/render'
 import { createTestQueryClient } from '@/test/query-client'
 import { ConnectedSourceSetup } from '../connected-source-setup'
@@ -263,6 +264,26 @@ const notionDatasourcePlugin: DataSourceItem = {
   provider: 'notion',
 }
 
+const notionDatasourcePluginWithParameters: DataSourceItem = {
+  ...notionDatasourcePlugin,
+  declaration: {
+    ...notionDatasourcePlugin.declaration,
+    datasources: [
+      {
+        ...notionDatasourcePlugin.declaration.datasources[0]!,
+        parameters: [
+          {
+            label: { en_US: 'Workspace' },
+            name: 'workspace',
+            required: true,
+            type: 'string',
+          },
+        ],
+      },
+    ],
+  },
+}
+
 const s3Provider: KnowledgeFsSourceProviderResponse = {
   auth_kinds: ['endpoint'],
   available: true,
@@ -489,6 +510,35 @@ function renderSetup(draft: ConnectedDraft = defaultDraft) {
   }
 }
 
+function renderStatefulSetup(draft: ConnectedDraft = defaultDraft) {
+  const queryClient = createTestQueryClient()
+  const onCompleted = vi.fn()
+  const onDirtyChange = vi.fn()
+  const onDraftChange = vi.fn()
+  const onExit = vi.fn()
+
+  function StatefulSetup() {
+    const [currentDraft, setCurrentDraft] = useState(draft)
+    return (
+      <QueryClientProvider client={queryClient}>
+        <ConnectedSourceSetup
+          draft={currentDraft}
+          knowledgeSpaceId="space-1"
+          onCompleted={onCompleted}
+          onDirtyChange={onDirtyChange}
+          onDraftChange={(nextDraft) => {
+            onDraftChange(nextDraft)
+            setCurrentDraft(nextDraft as ConnectedDraft)
+          }}
+          onExit={onExit}
+        />
+      </QueryClientProvider>
+    )
+  }
+
+  return { ...render(<StatefulSetup />), onDraftChange }
+}
+
 describe('ConnectedSourceSetup', () => {
   beforeEach(() => {
     for (const mock of Object.values(clientMock)) mock.mockReset()
@@ -623,6 +673,8 @@ describe('ConnectedSourceSetup', () => {
         connectionId: 'connection-1',
         metadata: {
           clientRequestId: expect.any(String),
+          datasourceParameterMode: 'exact',
+          parameters: {},
           preview: true,
           providerId: 'notion-provider',
           providerKind: 'online-document',
@@ -657,6 +709,8 @@ describe('ConnectedSourceSetup', () => {
         expectedVersion: 3,
         metadata: {
           clientRequestId: expect.any(String),
+          datasourceParameterMode: 'exact',
+          parameters: {},
           preview: false,
           providerId: 'notion-provider',
           providerKind: 'online-document',
@@ -713,6 +767,35 @@ describe('ConnectedSourceSetup', () => {
     expect(clientMock.deleteSource).not.toHaveBeenCalled()
   })
 
+  it('applies datasource parameters once instead of rebuilding the preview on every keypress', async () => {
+    const user = userEvent.setup()
+    clientMock.listDatasourcePlugins.mockResolvedValue([notionDatasourcePluginWithParameters])
+    clientMock.listDatasourceAuth.mockResolvedValue({
+      result: [notionDatasourceAuth([notionCredential])],
+    })
+    clientMock.listConnections.mockResolvedValue({
+      data: [connectionResponse()],
+      next_cursor: null,
+    } satisfies KnowledgeFsSourceConnectionListResponse)
+
+    renderStatefulSetup({ ...defaultDraft, parameters: {} })
+
+    const workspace = await screen.findByRole('textbox', { name: 'Workspace' })
+    await user.type(workspace, 'product-docs')
+
+    expect(clientMock.createSource).not.toHaveBeenCalled()
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' }))
+
+    await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())
+    expect(clientMock.createSource).toHaveBeenCalledWith(
+      expect.objectContaining({
+        body: expect.objectContaining({
+          metadata: expect.objectContaining({ parameters: { workspace: 'product-docs' } }),
+        }),
+      }),
+    )
+  })
+
   it('creates the managed endpoint connection automatically from the default credential', async () => {
     clientMock.listDatasourceAuth.mockResolvedValue({
       result: [notionDatasourceAuth([notionCredential])],
@@ -966,6 +1049,8 @@ describe('ConnectedSourceSetup', () => {
         connectionId: googleConnection.id,
         metadata: {
           clientRequestId: expect.any(String),
+          datasourceParameterMode: 'exact',
+          parameters: {},
           preview: true,
           providerId: googleProvider.id,
           providerKind: 'online-drive',
@@ -1207,6 +1292,8 @@ describe('ConnectedSourceSetup', () => {
         connectionId: 's3-connection',
         metadata: {
           clientRequestId: expect.any(String),
+          datasourceParameterMode: 'exact',
+          parameters: {},
           preview: true,
           providerId: 's3-provider',
           providerKind: 'online-drive',

web/features/new-rag/__tests__/create-knowledge-page.spec.tsx

@@ -10,7 +10,10 @@ const serviceMock = vi.hoisted(() => ({
   createKfsSource: vi.fn(),
   getKfsSource: vi.fn(),
   getCrawlStatus: vi.fn(),
+  cancelWebsitePreview: vi.fn(),
+  getWebsitePreview: vi.fn(),
   previewInitialSource: vi.fn(),
+  startWebsitePreview: vi.fn(),
   getDefaultModel: vi.fn(),
   getSpace: vi.fn(),
   getSyncPolicy: vi.fn(),
@@ -102,6 +105,13 @@ vi.mock('@/service/client', () => ({
     knowledgeFs: {
       sourceProviderPreview: {
         post: serviceMock.previewInitialSource,
+        jobs: {
+          post: serviceMock.startWebsitePreview,
+          byJobId: {
+            delete: serviceMock.cancelWebsitePreview,
+            get: serviceMock.getWebsitePreview,
+          },
+        },
       },
       spaces: {
         byControlSpaceId: {
@@ -272,6 +282,66 @@ const jinaDatasourceAuth = {
   provider: 'jinareader',
 }
 
+const tavilyDatasourcePlugin = {
+  ...firecrawlDatasourcePlugin,
+  declaration: {
+    ...firecrawlDatasourcePlugin.declaration,
+    datasources: [
+      {
+        description: { en_US: 'Search and extract', zh_Hans: '搜索与提取' },
+        identity: {
+          author: 'langgenius',
+          label: { en_US: 'Tavily', zh_Hans: 'Tavily' },
+          name: 'search_extract',
+          provider: 'tavily',
+        },
+        parameters: [
+          {
+            label: { en_US: 'Search query', zh_Hans: '搜索词' },
+            name: 'query',
+            required: true,
+            type: 'string',
+          },
+          {
+            default: 'basic',
+            label: { en_US: 'Search depth', zh_Hans: '搜索深度' },
+            name: 'search_depth',
+            options: [
+              { label: { en_US: 'Basic', zh_Hans: '基础' }, value: 'basic' },
+              { label: { en_US: 'Advanced', zh_Hans: '高级' }, value: 'advanced' },
+            ],
+            type: 'select',
+          },
+        ],
+      },
+    ],
+    identity: {
+      ...firecrawlDatasourcePlugin.declaration.identity,
+      label: { en_US: 'Tavily', zh_Hans: 'Tavily' },
+      name: 'tavily',
+    },
+  },
+  plugin_id: 'langgenius/tavily_datasource',
+  plugin_unique_identifier: 'langgenius/tavily_datasource:1.0.0@local',
+  provider: 'tavily',
+}
+
+const tavilyDatasourceAuth = {
+  ...firecrawlDatasourceAuth,
+  credentials_list: [
+    {
+      ...firecrawlDatasourceAuth.credentials_list[0],
+      id: 'tavily-credential-1',
+      name: 'Default Tavily',
+    },
+  ],
+  label: { en_US: 'Tavily' },
+  name: 'tavily',
+  plugin_id: 'langgenius/tavily_datasource',
+  plugin_unique_identifier: 'langgenius/tavily_datasource:1.0.0@local',
+  provider: 'tavily',
+}
+
 const notionDatasourcePlugin = {
   ...firecrawlDatasourcePlugin,
   declaration: {
@@ -300,6 +370,26 @@ const notionDatasourcePlugin = {
   provider: 'notion',
 }
 
+const notionDatasourcePluginWithParameters = {
+  ...notionDatasourcePlugin,
+  declaration: {
+    ...notionDatasourcePlugin.declaration,
+    datasources: [
+      {
+        ...notionDatasourcePlugin.declaration.datasources[0],
+        parameters: [
+          {
+            label: { en_US: 'Workspace', zh_Hans: '工作区' },
+            name: 'workspace',
+            required: true,
+            type: 'string',
+          },
+        ],
+      },
+    ],
+  },
+}
+
 const notionDatasourceAuth = {
   ...firecrawlDatasourceAuth,
   credentials_list: [
@@ -316,6 +406,49 @@ const notionDatasourceAuth = {
   provider: 'notion',
 }
 
+const outlineDatasourcePlugin = {
+  ...notionDatasourcePlugin,
+  declaration: {
+    ...notionDatasourcePlugin.declaration,
+    datasources: [
+      {
+        ...notionDatasourcePlugin.declaration.datasources[0]!,
+        description: { en_US: 'Outline', zh_Hans: 'Outline' },
+        identity: {
+          ...notionDatasourcePlugin.declaration.datasources[0]!.identity,
+          label: { en_US: 'Outline', zh_Hans: 'Outline' },
+          name: 'outline',
+          provider: 'outline',
+        },
+      },
+    ],
+    identity: {
+      ...notionDatasourcePlugin.declaration.identity,
+      label: { en_US: 'Outline', zh_Hans: 'Outline' },
+      name: 'outline',
+    },
+  },
+  plugin_id: 'langgenius/outline_datasource',
+  plugin_unique_identifier: 'langgenius/outline_datasource:1.0.0@local',
+  provider: 'outline',
+}
+
+const outlineDatasourceAuth = {
+  ...notionDatasourceAuth,
+  credentials_list: [
+    {
+      ...notionDatasourceAuth.credentials_list[0],
+      id: 'outline-credential-1',
+      name: 'Default Outline',
+    },
+  ],
+  label: { en_US: 'Outline' },
+  name: 'outline',
+  plugin_id: 'langgenius/outline_datasource',
+  plugin_unique_identifier: 'langgenius/outline_datasource:1.0.0@local',
+  provider: 'outline',
+}
+
 const googleDriveDatasourcePlugin = {
   ...firecrawlDatasourcePlugin,
   declaration: {
@@ -465,6 +598,25 @@ describe('CreateKnowledgePage', () => {
     datasourceQueryMock.auth.error = null
     datasourceQueryMock.auth.isPending = false
     serviceMock.create.mockResolvedValue(createdKnowledge)
+    serviceMock.startWebsitePreview.mockResolvedValue({ job_id: 'website-preview-1' })
+    serviceMock.getWebsitePreview.mockResolvedValue({
+      job_id: 'website-preview-1',
+      status: 'completed',
+      result: {
+        kind: 'website_crawl',
+        pages: [
+          {
+            description: 'Getting started',
+            source_url: 'https://docs.dify.ai/getting-started',
+            title: 'Getting started',
+          },
+        ],
+      },
+    })
+    serviceMock.cancelWebsitePreview.mockResolvedValue({
+      job_id: 'website-preview-1',
+      status: 'canceled',
+    })
     serviceMock.createCrawl.mockResolvedValue({ job_id: 'crawl-job-1' })
     serviceMock.createKfsSource.mockResolvedValue(kfsSourceResponse())
     serviceMock.getKfsSource.mockResolvedValue(kfsSourceResponse({ status: 'active', version: 3 }))
@@ -1025,12 +1177,10 @@ describe('CreateKnowledgePage', () => {
     await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.websiteCrawl' }))
     expect(screen.getByRole('button', { name: 'dataset.newKnowledge.moreProviders' })).toBeEnabled()
     expect(screen.queryByRole('alert')).not.toBeInTheDocument()
-    expect(screen.getByText('dataset.newKnowledge.crawlOptions')).toBeInTheDocument()
     expect(
       screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }),
     ).toBeDisabled()
     expect(screen.getByText('dataset.newKnowledge.pagesAppearTitle')).toBeInTheDocument()
-    expect(screen.getByText('dataset.newKnowledge.usingDefaults')).toBeInTheDocument()
     const rootUrl = screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder')
     const sourceName = screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder')
     expect(rootUrl).toBeEnabled()
@@ -1146,27 +1296,23 @@ describe('CreateKnowledgePage', () => {
     )
     await user.keyboard('{Enter}')
     expect(serviceMock.create).not.toHaveBeenCalled()
-    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlOptions' }))
-    await user.click(screen.getByRole('checkbox', { name: 'dataset.newKnowledge.includeSubpages' }))
-    const maxPages = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })
+    await user.click(screen.getByRole('switch', { name: 'dataset.newKnowledge.includeSubpages' }))
+    const maxPages = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })
     await user.clear(maxPages)
     await user.type(maxPages, '25')
-    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlOptions' }))
-    expect(
-      screen.getByText(
-        'dataset.newKnowledge.includeSubpages: dataset.newKnowledge.booleanFalse · dataset.newKnowledge.maxPages: 25',
-      ),
-    ).toBeInTheDocument()
     await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
 
     expect(serviceMock.create).not.toHaveBeenCalled()
     expect(routerMock.replace).not.toHaveBeenCalled()
-    expect(serviceMock.createCrawl).toHaveBeenCalledWith({
-      options: expect.objectContaining({
-        crawl_sub_pages: false,
-        limit: 25,
+    expect(serviceMock.startWebsitePreview).toHaveBeenCalledWith({
+      body: expect.objectContaining({
+        kind: 'website_crawl',
+        parameters: expect.objectContaining({
+          crawl_subpages: false,
+          limit: 25,
+          url: 'https://docs.dify.ai',
+        }),
       }),
-      url: 'https://docs.dify.ai',
     })
     expect(await screen.findByText('Getting started')).toBeInTheDocument()
     expect(
@@ -1187,17 +1333,46 @@ describe('CreateKnowledgePage', () => {
     expect(screen.getByText('Getting started')).toBeInTheDocument()
   })
 
-  it('creates an initial source from a synchronous Jina Reader preview', async () => {
+  it('lets users edit website parameters after a successful preview', async () => {
+    const user = userEvent.setup()
+    navigationMock.startMode = 'source'
+    renderPage()
+    await fillRequiredFields(user)
+    const rootUrl = screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder')
+    await user.type(rootUrl, 'https://docs.dify.ai')
+    await user.type(
+      screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
+      'Dify docs',
+    )
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
+    await screen.findByText('Getting started')
+
+    expect(rootUrl).toBeEnabled()
+    await user.clear(rootUrl)
+
+    expect(screen.queryByText('Getting started')).not.toBeInTheDocument()
+    expect(
+      screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }),
+    ).toBeDisabled()
+  })
+
+  it('creates an initial source from a Jina Reader preview job', async () => {
     const user = userEvent.setup()
     navigationMock.startMode = 'source'
     datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, jinaDatasourcePlugin]
     datasourceQueryMock.auth.data = { result: [firecrawlDatasourceAuth, jinaDatasourceAuth] }
-    serviceMock.createCrawl.mockResolvedValueOnce({
-      data: {
-        content: '# Dify introduction',
-        description: 'Introduction',
-        title: 'Dify introduction',
-        url: 'https://docs.dify.ai/introduction',
+    serviceMock.getWebsitePreview.mockResolvedValueOnce({
+      job_id: 'website-preview-1',
+      status: 'completed',
+      result: {
+        kind: 'website_crawl',
+        pages: [
+          {
+            description: 'Introduction',
+            source_url: 'https://docs.dify.ai/introduction',
+            title: 'Dify introduction',
+          },
+        ],
       },
     })
     renderPage()
@@ -1235,10 +1410,71 @@ describe('CreateKnowledgePage', () => {
     })
   })
 
-  it('shows and can stop an ongoing website crawl', async () => {
+  it('previews and persists a declaration-driven website datasource without a root URL', async () => {
     const user = userEvent.setup()
     navigationMock.startMode = 'source'
-    serviceMock.getCrawlStatus.mockImplementation(() => new Promise(() => {}))
+    datasourceQueryMock.plugins.data = [firecrawlDatasourcePlugin, tavilyDatasourcePlugin]
+    datasourceQueryMock.auth.data = { result: [firecrawlDatasourceAuth, tavilyDatasourceAuth] }
+    serviceMock.getWebsitePreview.mockResolvedValueOnce({
+      job_id: 'website-preview-1',
+      status: 'completed',
+      result: {
+        kind: 'website_crawl',
+        pages: [
+          {
+            description: 'Tavily result',
+            source_url: 'https://example.com/result',
+            title: 'Tavily result',
+          },
+        ],
+      },
+    })
+    renderPage()
+    await fillRequiredFields(user)
+    await user.click(screen.getByRole('radio', { name: 'Tavily' }))
+    await user.type(screen.getByRole('textbox', { name: /Search query/ }), 'agentic RAG')
+    await user.type(
+      screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
+      'Tavily research',
+    )
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
+
+    expect(serviceMock.startWebsitePreview).toHaveBeenCalledWith({
+      body: {
+        credentialId: 'tavily-credential-1',
+        datasource: 'search_extract',
+        kind: 'website_crawl',
+        parameters: { query: 'agentic RAG', search_depth: 'basic' },
+        pluginId: 'langgenius/tavily_datasource',
+        provider: 'tavily',
+        providerDisplayName: 'Tavily',
+      },
+    })
+    await user.click(await screen.findByRole('checkbox', { name: 'Tavily result' }))
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.createTitle' }))
+
+    await waitFor(() => expect(serviceMock.create).toHaveBeenCalledOnce())
+    expect(serviceMock.create.mock.calls[0]?.[0].body.initial_source).toEqual(
+      expect.objectContaining({
+        parameters: { query: 'agentic RAG', search_depth: 'basic' },
+        root_url: expect.stringMatching(/^datasource:\/\//),
+      }),
+    )
+  })
+
+  it('shows and can stop an ongoing website crawl', async () => {
+    const user = userEvent.setup()
+    let resolveCancellation: ((value: { job_id: string; status: 'canceled' }) => void) | undefined
+    serviceMock.cancelWebsitePreview.mockReturnValue(
+      new Promise((resolve) => {
+        resolveCancellation = resolve
+      }),
+    )
+    navigationMock.startMode = 'source'
+    serviceMock.getWebsitePreview.mockResolvedValue({
+      job_id: 'website-preview-1',
+      status: 'running',
+    })
     renderPage()
     await fillRequiredFields(user)
     await user.type(
@@ -1259,6 +1495,54 @@ describe('CreateKnowledgePage', () => {
     expect(stopButton).toBeEnabled()
 
     await user.click(stopButton)
+    expect(serviceMock.cancelWebsitePreview).toHaveBeenCalledWith({
+      params: { job_id: 'website-preview-1' },
+    })
+    expect(screen.getByRole('status')).toHaveTextContent('dataset.newKnowledge.crawlingPages')
+
+    resolveCancellation?.({ job_id: 'website-preview-1', status: 'canceled' })
+    await waitFor(() =>
+      expect(screen.getByRole('status')).toHaveTextContent('dataset.newKnowledge.crawlStopped'),
+    )
+  })
+
+  it('keeps the preview job available when stopping fails so cancellation can be retried', async () => {
+    const user = userEvent.setup()
+    navigationMock.startMode = 'source'
+    serviceMock.getWebsitePreview.mockResolvedValue({
+      job_id: 'website-preview-1',
+      status: 'running',
+    })
+    serviceMock.cancelWebsitePreview
+      .mockRejectedValueOnce(new Error('cancel response lost'))
+      .mockResolvedValueOnce({
+        job_id: 'website-preview-1',
+        status: 'canceled',
+      })
+    renderPage()
+    await fillRequiredFields(user)
+    await user.type(
+      screen.getByPlaceholderText('dataset.newKnowledge.rootUrlPlaceholder'),
+      'https://docs.dify.ai',
+    )
+    await user.type(
+      screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
+      'Dify docs',
+    )
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
+
+    const stopButton = await screen.findByRole('button', {
+      name: 'dataset.newKnowledge.stopCrawl',
+    })
+    await user.click(stopButton)
+    await waitFor(() => expect(stopButton).toBeEnabled())
+
+    await user.click(stopButton)
+
+    await waitFor(() => expect(serviceMock.cancelWebsitePreview).toHaveBeenCalledTimes(2))
+    expect(serviceMock.cancelWebsitePreview).toHaveBeenNthCalledWith(2, {
+      params: { job_id: 'website-preview-1' },
+    })
     expect(screen.getByRole('status')).toHaveTextContent('dataset.newKnowledge.crawlStopped')
   })
 
@@ -1300,10 +1584,15 @@ describe('CreateKnowledgePage', () => {
           datasource: 'crawl',
           kind: 'website_crawl',
           name: 'Dify docs',
+          parameters: {
+            crawl_subpages: true,
+            limit: 100,
+            url: 'https://docs.dify.ai',
+          },
           pluginId: 'langgenius/firecrawl_datasource',
           provider: 'firecrawl',
           providerDisplayName: 'Firecrawl',
-          root_url: 'https://docs.dify.ai',
+          root_url: 'https://docs.dify.ai/',
           selection: [
             {
               source_url: 'https://docs.dify.ai/getting-started',
@@ -1385,6 +1674,7 @@ describe('CreateKnowledgePage', () => {
           datasource: 'notion',
           kind: 'online_document',
           name: 'Notion handbook',
+          parameters: {},
           pluginId: 'langgenius/notion_datasource',
           provider: 'notion',
           providerDisplayName: 'Notion',
@@ -1404,6 +1694,88 @@ describe('CreateKnowledgePage', () => {
     })
   })
 
+  it('does not inject website defaults when switching connected document providers', async () => {
+    const user = userEvent.setup()
+    navigationMock.startMode = 'source'
+    datasourceQueryMock.plugins.data = [
+      firecrawlDatasourcePlugin,
+      notionDatasourcePlugin,
+      outlineDatasourcePlugin,
+    ]
+    datasourceQueryMock.auth.data = {
+      result: [firecrawlDatasourceAuth, notionDatasourceAuth, outlineDatasourceAuth],
+    }
+    serviceMock.previewInitialSource.mockResolvedValue({
+      documents: [],
+      files: [],
+      kind: 'online_document',
+      next_page_parameters: null,
+    })
+    renderPage()
+    await fillRequiredFields(user)
+    await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' }))
+    await user.click(screen.getByRole('radio', { name: 'Outline' }))
+    await user.type(
+      screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
+      'Outline handbook',
+    )
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' }))
+
+    await waitFor(() => expect(serviceMock.previewInitialSource).toHaveBeenCalledOnce())
+    expect(serviceMock.previewInitialSource).toHaveBeenCalledWith({
+      body: expect.objectContaining({
+        datasource: 'outline',
+        parameters: {},
+        pluginId: 'langgenius/outline_datasource',
+        provider: 'outline',
+      }),
+    })
+  })
+
+  it('resets a connected-source preview when datasource parameters change', async () => {
+    const user = userEvent.setup()
+    navigationMock.startMode = 'source'
+    datasourceQueryMock.plugins.data = [
+      firecrawlDatasourcePlugin,
+      notionDatasourcePluginWithParameters,
+    ]
+    datasourceQueryMock.auth.data = {
+      result: [firecrawlDatasourceAuth, notionDatasourceAuth],
+    }
+    serviceMock.previewInitialSource.mockResolvedValue({
+      documents: [
+        {
+          last_edited_time: '2026-08-10T08:00:00Z',
+          name: 'Product handbook',
+          page_id: 'page-1',
+          provider_item_id: '["workspace-1","page-1"]',
+          type: 'page',
+          workspace_id: 'workspace-1',
+          workspace_name: 'Dify',
+        },
+      ],
+      kind: 'online_document',
+      next_page_parameters: null,
+    })
+    renderPage()
+    await fillRequiredFields(user)
+    await user.click(screen.getByRole('radio', { name: 'dataset.newKnowledge.onlineDocuments' }))
+    await user.type(screen.getByRole('textbox', { name: 'Workspace' }), 'Product')
+    await user.type(
+      screen.getByPlaceholderText('dataset.newKnowledge.sourceNamePlaceholder'),
+      'Notion handbook',
+    )
+    await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' }))
+    await screen.findByRole('checkbox', { name: 'Product handbook' })
+
+    const workspace = screen.getByRole('textbox', { name: 'Workspace' })
+    expect(workspace).toBeEnabled()
+    await user.clear(workspace)
+
+    expect(screen.queryByRole('checkbox', { name: 'Product handbook' })).not.toBeInTheDocument()
+    expect(screen.getByRole('button', { name: 'dataset.newKnowledge.preview' })).toBeDisabled()
+  })
+
   it('creates a knowledge space atomically with a selected drive file', async () => {
     const user = userEvent.setup()
     navigationMock.startMode = 'source'
@@ -1451,6 +1823,7 @@ describe('CreateKnowledgePage', () => {
           datasource: 'google_drive',
           kind: 'online_drive',
           name: 'Drive runbook',
+          parameters: {},
           pluginId: 'langgenius/google_drive',
           provider: 'google_drive',
           providerDisplayName: 'Google Drive',

web/features/new-rag/__tests__/datasource-parameter-form.spec.tsx

@@ -0,0 +1,136 @@
+import type { DatasourceParameters, DatasourceParameterSchema } from '../datasource-parameter-model'
+import { screen } from '@testing-library/react'
+import userEvent from '@testing-library/user-event'
+import { useState } from 'react'
+import { render } from '@/test/console/render'
+import { DatasourceParameterForm } from '../datasource-parameter-form'
+import {
+  datasourceParameterDefaults,
+  websiteDatasourceParameterSchemas,
+} from '../datasource-parameter-model'
+
+const limitSchema: DatasourceParameterSchema = {
+  label: { en_US: 'Limit' },
+  max: 200,
+  min: 100,
+  name: 'limit',
+  options: [],
+  required: true,
+  type: 'number',
+}
+
+const urlSchema: DatasourceParameterSchema = {
+  label: { en_US: 'Root URL' },
+  name: 'url',
+  options: [],
+  required: true,
+  type: 'string',
+}
+
+const decimalSchema: DatasourceParameterSchema = {
+  label: { en_US: 'Threshold' },
+  name: 'threshold',
+  options: [],
+  precision: 2,
+  required: false,
+  type: 'number',
+}
+
+function NumberParameterForm() {
+  const [parameters, setParameters] = useState<DatasourceParameters>({})
+  return (
+    <DatasourceParameterForm
+      parameters={parameters}
+      schemas={[limitSchema]}
+      onChange={setParameters}
+    />
+  )
+}
+
+function UrlParameterForm() {
+  const [parameters, setParameters] = useState<DatasourceParameters>({})
+  return (
+    <DatasourceParameterForm
+      parameters={parameters}
+      schemas={[urlSchema]}
+      onChange={setParameters}
+    />
+  )
+}
+
+function DecimalParameterForm() {
+  const [parameters, setParameters] = useState<DatasourceParameters>({})
+  return (
+    <DatasourceParameterForm
+      parameters={parameters}
+      schemas={[decimalSchema]}
+      onChange={setParameters}
+    />
+  )
+}
+
+function LegacyWebsiteParameterForm() {
+  const schemas = websiteDatasourceParameterSchemas({ parameters: [] } as never)
+  const [parameters, setParameters] = useState<DatasourceParameters>(() =>
+    datasourceParameterDefaults(schemas),
+  )
+  return (
+    <DatasourceParameterForm parameters={parameters} schemas={schemas} onChange={setParameters} />
+  )
+}
+
+describe('DatasourceParameterForm', () => {
+  it('keeps an out-of-range numeric value visible and reports it as invalid', async () => {
+    const user = userEvent.setup()
+    render(<NumberParameterForm />)
+
+    const input = screen.getByRole('spinbutton', { name: 'Limit' })
+    await user.type(input, '15')
+    expect(input).toHaveValue(15)
+
+    await user.tab()
+    expect(input).toHaveValue(15)
+    expect(input).toHaveAttribute('aria-invalid', 'true')
+  })
+
+  it('associates an actionable error with an invalid URL field', async () => {
+    const user = userEvent.setup()
+    render(<UrlParameterForm />)
+
+    const input = screen.getByRole('textbox', { name: 'Root URL' })
+    await user.type(input, 'ftp://example.com')
+    await user.tab()
+
+    expect(input).toHaveAttribute('aria-invalid', 'true')
+    expect(input).toHaveAccessibleDescription('dataset.newKnowledge.invalidRootUrl')
+    expect(screen.getByText('dataset.newKnowledge.invalidRootUrl')).toBeInTheDocument()
+  })
+
+  it('allows decimal input using the declaration precision', async () => {
+    const user = userEvent.setup()
+    render(<DecimalParameterForm />)
+
+    const input = screen.getByRole('spinbutton', { name: 'Threshold' })
+    expect(input).toHaveAttribute('inputmode', 'decimal')
+    expect(input).toHaveAttribute('step', '0.01')
+
+    await user.type(input, '0.25')
+    expect(input).toHaveValue(0.25)
+    expect(input).not.toHaveAttribute('aria-invalid', 'true')
+  })
+
+  it('resolves translation keys for legacy website parameter labels', () => {
+    render(<LegacyWebsiteParameterForm />)
+
+    expect(screen.getByRole('textbox', { name: 'dataset.newKnowledge.rootUrl' })).toHaveAttribute(
+      'placeholder',
+      'dataset.newKnowledge.rootUrlPlaceholder',
+    )
+    expect(
+      screen.getByRole('switch', { name: 'dataset.newKnowledge.includeSubpages' }),
+    ).toBeChecked()
+    expect(screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })).toHaveValue(
+      100,
+    )
+  })
+})

web/features/new-rag/__tests__/datasource-parameter-model.spec.ts

@@ -0,0 +1,163 @@
+import {
+  datasourceParameterDefaults,
+  datasourceParameterRecord,
+  datasourceParameterSchemas,
+  invalidDatasourceParameters,
+  missingRequiredDatasourceParameters,
+  websiteDatasourceParameterSchemas,
+  withDatasourceParameterDefaults,
+} from '../datasource-parameter-model'
+
+describe('datasource parameter model', () => {
+  it('parses plugin-declared fields and applies defaults without replacing saved values', () => {
+    const schemas = datasourceParameterSchemas({
+      parameters: [
+        {
+          default: 'basic',
+          label: { en_US: 'Search depth' },
+          name: 'search_depth',
+          options: [
+            { label: { en_US: 'Basic' }, value: 'basic' },
+            { label: { en_US: 'Advanced' }, value: 'advanced' },
+          ],
+          required: true,
+          type: 'select',
+        },
+        {
+          label: { en_US: 'Query' },
+          name: 'query',
+          required: true,
+          type: 'string',
+        },
+      ],
+    } as never)
+
+    expect(datasourceParameterDefaults(schemas)).toEqual({ search_depth: 'basic' })
+    expect(withDatasourceParameterDefaults(schemas, { search_depth: 'advanced' })).toEqual({
+      search_depth: 'advanced',
+    })
+    expect(missingRequiredDatasourceParameters(schemas, { search_depth: 'advanced' })).toEqual([
+      expect.objectContaining({ name: 'query' }),
+    ])
+  })
+
+  it('validates URL, numeric, and select constraints from a declaration', () => {
+    const schemas = datasourceParameterSchemas({
+      parameters: [
+        { label: { en_US: 'URL' }, name: 'url', required: true, type: 'string' },
+        { label: { en_US: 'Limit' }, max: 200, min: 1, name: 'limit', type: 'integer' },
+        {
+          label: { en_US: 'Mode' },
+          name: 'mode',
+          options: [{ label: { en_US: 'One' }, value: 'one' }],
+          type: 'select',
+        },
+      ],
+    } as never)
+
+    expect(
+      invalidDatasourceParameters(schemas, {
+        limit: 201,
+        mode: 'unknown',
+        url: 'ftp://example.com',
+      }).map((schema) => schema.name),
+    ).toEqual(['url', 'limit', 'mode'])
+    expect(
+      invalidDatasourceParameters(schemas, {
+        limit: 1.5,
+        mode: 'one',
+        url: 'https://example.com',
+      }).map((schema) => schema.name),
+    ).toEqual(['limit'])
+    expect(
+      invalidDatasourceParameters(schemas, {
+        limit: 25,
+        mode: 'one',
+        url: 'https://example.com',
+      }),
+    ).toEqual([])
+  })
+
+  it('preserves and validates decimal precision for numeric parameters', () => {
+    const schemas = datasourceParameterSchemas({
+      parameters: [
+        { label: { en_US: 'Threshold' }, name: 'threshold', precision: 2, type: 'number' },
+      ],
+    } as never)
+
+    expect(schemas[0]).toEqual(expect.objectContaining({ precision: 2, type: 'number' }))
+    expect(invalidDatasourceParameters(schemas, { threshold: 0.25 })).toEqual([])
+    expect(invalidDatasourceParameters(schemas, { threshold: 0.251 })).toEqual([
+      expect.objectContaining({ name: 'threshold' }),
+    ])
+  })
+
+  it('keeps legacy website providers functional when their declaration has no fields', () => {
+    const schemas = websiteDatasourceParameterSchemas({ parameters: [] } as never)
+
+    expect(schemas).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ name: 'url', required: true }),
+        expect.objectContaining({ defaultValue: true, name: 'crawl_subpages' }),
+        expect.objectContaining({ defaultValue: 100, name: 'limit' }),
+      ]),
+    )
+    expect(schemas[0]).toEqual(
+      expect.objectContaining({
+        label: {},
+        labelTranslationKey: 'newKnowledge.rootUrl',
+        placeholder: {},
+        placeholderTranslationKey: 'newKnowledge.rootUrlPlaceholder',
+      }),
+    )
+  })
+
+  it('uses the provider-specific legacy subpage parameter name', () => {
+    expect(
+      websiteDatasourceParameterSchemas({
+        identity: { name: 'jina_reader', provider: 'jinareader' },
+        parameters: [],
+      } as never),
+    ).toEqual(expect.arrayContaining([expect.objectContaining({ name: 'crawl_sub_pages' })]))
+  })
+
+  it('retains required unsupported parameters so the datasource cannot run without them', () => {
+    const schemas = datasourceParameterSchemas({
+      parameters: [
+        {
+          label: { en_US: 'Private token' },
+          name: 'private_token',
+          required: true,
+          type: 'secret-input',
+        },
+        {
+          label: { en_US: 'Attachment' },
+          name: 'attachment',
+          required: false,
+          type: 'file',
+        },
+      ],
+    } as never)
+
+    expect(schemas).toEqual([
+      expect.objectContaining({ name: 'private_token', type: 'unsupported' }),
+      expect.objectContaining({ name: 'attachment', type: 'unsupported' }),
+    ])
+    expect(missingRequiredDatasourceParameters(schemas, {})).toEqual([
+      expect.objectContaining({ name: 'private_token' }),
+    ])
+    expect(invalidDatasourceParameters(schemas, { attachment: 'file-1' })).toEqual([
+      expect.objectContaining({ name: 'attachment' }),
+    ])
+  })
+
+  it('accepts only bounded scalar draft parameters', () => {
+    expect(datasourceParameterRecord({ enabled: true, limit: 5, query: 'dify' })).toEqual({
+      enabled: true,
+      limit: 5,
+      query: 'dify',
+    })
+    expect(datasourceParameterRecord({ nested: { token: 'no' } })).toBeUndefined()
+    expect(datasourceParameterRecord({ value: Number.NaN })).toBeUndefined()
+  })
+})

web/features/new-rag/__tests__/routes.spec.ts

@@ -86,6 +86,7 @@ describe('New RAG routes', () => {
     ).toEqual({
       includeSubpages: false,
       maxPages: 25,
+      parameters: {},
       provider: 'Firecrawl',
       rootUrl: 'https://docs.dify.ai',
       sourceName: 'Dify docs',

web/features/new-rag/__tests__/website-crawl-preview.spec.tsx

@@ -278,6 +278,12 @@ describe('WebsiteCrawlPreview', () => {
         metadata: {
           clientRequestId: expect.any(String),
           crawlOptions: { includeSubpages: true, limit: 100 },
+          datasourceParameterMode: 'exact',
+          parameters: {
+            crawl_subpages: true,
+            limit: 100,
+            url: 'https://docs.dify.ai',
+          },
           preview: true,
           providerId: 'plugin-daemon-website',
           providerName: 'Firecrawl',
@@ -523,12 +529,11 @@ describe('WebsiteCrawlPreview', () => {
     async (invalidLimit) => {
       render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
       const user = await fillValidForm()
-      await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ }))
-      const pageLimit = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })
+      const pageLimit = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })
       await user.clear(pageLimit)
       await user.type(pageLimit, invalidLimit)
 
-      expect(pageLimit).toHaveValue(invalidLimit)
+      expect(pageLimit).toHaveValue(Number(invalidLimit))
       expect(pageLimit).toHaveAttribute('aria-invalid', 'true')
       expect(pageLimit).toHaveAccessibleDescription('dataset.newKnowledge.maxPages: 1–200')
       expect(
@@ -542,11 +547,10 @@ describe('WebsiteCrawlPreview', () => {
   it.each([1, 200])('submits the exact valid page limit %s', async (validLimit) => {
     render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
     const user = await fillValidForm()
-    await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ }))
-    const pageLimit = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })
+    const pageLimit = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })
     await user.clear(pageLimit)
     await user.type(pageLimit, String(validLimit))
-    expect(pageLimit).toHaveValue(String(validLimit))
+    expect(pageLimit).toHaveValue(validLimit)
     await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
 
     await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())
@@ -558,17 +562,10 @@ describe('WebsiteCrawlPreview', () => {
   it('preserves a replacement crawl page limit after clearing the input', async () => {
     render(<WebsiteCrawlPreview connection={connection} knowledgeSpaceId="space-1" />)
     const user = await fillValidForm()
-    await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ }))
-    const pageLimit = screen.getByRole('textbox', { name: 'dataset.newKnowledge.maxPages' })
+    const pageLimit = screen.getByRole('spinbutton', { name: 'dataset.newKnowledge.maxPages' })
     await user.clear(pageLimit)
     await user.type(pageLimit, '50')
-    expect(pageLimit).toHaveValue('50')
-    await user.click(screen.getByRole('button', { name: /^dataset\.newKnowledge\.crawlOptions/ }))
-    expect(
-      screen.getByText(
-        'dataset.newKnowledge.includeSubpages: dataset.newKnowledge.booleanTrue · dataset.newKnowledge.maxPages: 50',
-      ),
-    ).toBeInTheDocument()
+    expect(pageLimit).toHaveValue(50)
     await user.click(screen.getByRole('button', { name: 'dataset.newKnowledge.crawlAndPreview' }))
 
     await waitFor(() => expect(clientMock.createSource).toHaveBeenCalledOnce())

web/features/new-rag/add-source-page.tsx

@@ -28,6 +28,10 @@ import { consoleClient, consoleQuery } from '@/service/client'
 import { useDataSourceList } from '@/service/use-pipeline'
 import { AddSourceExitDialog } from './components/add-source-exit-dialog'
 import { ConnectedSourceSetup } from './connected-source-setup'
+import {
+  datasourceParameterDefaults,
+  websiteDatasourceParameterSchemas,
+} from './datasource-parameter-model'
 import {
   createNewKnowledgeSourceDraft,
   newKnowledgeDetailPath,
@@ -1276,8 +1280,14 @@ export function AddSourcePage({
                   if (!nextProvider) return
                   updateSourceDraft({
                     ...sourceDraft,
+                    parameters: nextProvider.installed
+                      ? datasourceParameterDefaults(
+                          websiteDatasourceParameterSchemas(nextProvider.datasource),
+                        )
+                      : {},
                     provider: nextProvider.label,
                     providerKey: nextProvider.key,
+                    rootUrl: '',
                   })
                 }}
               />
@@ -1325,13 +1335,14 @@ export function AddSourcePage({
                 </div>
               ) : activeConnection && websitePreviewReady ? (
                 <WebsiteCrawlPreview
-                  key={historyGuardReleaseVersion}
+                  key={`${historyGuardReleaseVersion}:${datasourceProvider.key}:${activeConnection.id}`}
                   connection={activeConnection}
                   initialDraft={sourceDraft}
                   knowledgeSpaceId={knowledgeSpaceId}
                   onDraftFinished={clearStoredSourceDraft}
                   onInteractionLockChange={setWebsiteSetupLocked}
                   providerName={websiteProviderName}
+                  providerOption={datasourceProvider}
                 />
               ) : activeConnection ? (
                 <div className="flex min-h-64 items-center justify-center">

web/features/new-rag/connected-source-setup.tsx

@@ -4,6 +4,7 @@ import type {
   KnowledgeFsSourceFileResponse,
   KnowledgeFsSourcePageResponse,
 } from '@dify/contracts/api/console/knowledge-fs/types.gen'
+import type { DatasourceParameters, DatasourceParameterSchema } from './datasource-parameter-model'
 import type {
   NewKnowledgeOnlineDocumentsSourceDraft,
   NewKnowledgeOnlineDriveSourceDraft,
@@ -27,6 +28,14 @@ import { consoleClient, consoleQuery } from '@/service/client'
 import { useGetDataSourceListAuth } from '@/service/use-datasource'
 import { useDataSourceList } from '@/service/use-pipeline'
 import { formatFileSize } from '@/utils/format'
+import { DatasourceParameterForm } from './datasource-parameter-form'
+import {
+  datasourceParameterDefaults,
+  datasourceParameterSchemas,
+  invalidDatasourceParameters,
+  missingRequiredDatasourceParameters,
+  withDatasourceParameterDefaults,
+} from './datasource-parameter-model'
 import { createRequestId } from './request-id'
 import { NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH } from './routes'
 import {
@@ -252,13 +261,16 @@ function sourceType() {
 
 function sourceMetadata(
   draft: ConnectedSourceDraft,
+  parameters: DatasourceParameters,
   provider: SourceProvider,
   clientRequestId: string,
   preview: boolean,
 ) {
   return {
     clientRequestId,
+    datasourceParameterMode: 'exact',
     preview,
+    parameters,
     providerId: provider.id,
     providerKind: usesDriveTransport(draft) ? 'online-drive' : 'online-document',
     providerName: draft.provider,
@@ -616,6 +628,7 @@ function ResourceConfiguration({
   onDraftChange,
   onDirtyChange,
   onExit,
+  parameters,
   provider,
   providerRegion,
 }: {
@@ -626,6 +639,7 @@ function ResourceConfiguration({
   onDraftChange: (draft: NewKnowledgeSourceDraft) => void
   onDirtyChange: (dirty: boolean) => void
   onExit: () => void
+  parameters: DatasourceParameters
   provider: SourceProvider
   providerRegion?: string
 }) {
@@ -690,6 +704,8 @@ function ResourceConfiguration({
                 connectionId: connection.id,
                 metadata: {
                   clientRequestId,
+                  datasourceParameterMode: 'exact',
+                  parameters,
                   preview: true,
                   providerId: provider.id,
                   providerKind: driveTransport ? 'online-drive' : 'online-document',
@@ -732,6 +748,7 @@ function ResourceConfiguration({
     draft.sourceType,
     driveTransport,
     knowledgeSpaceId,
+    parameters,
     previewUri,
     provider.id,
   ])
@@ -1271,7 +1288,13 @@ function ResourceConfiguration({
         await consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.patch({
           body: {
             expectedVersion: previewSource.version,
-            metadata: sourceMetadata(draft, provider, previewRequestIdRef.current, false),
+            metadata: sourceMetadata(
+              draft,
+              parameters,
+              provider,
+              previewRequestIdRef.current,
+              false,
+            ),
             name: draft.sourceName.trim(),
             status: 'active',
           },
@@ -1510,6 +1533,81 @@ function ResourceConfiguration({
   )
 }
 
+function AppliedResourceConfiguration({
+  connection,
+  draft,
+  knowledgeSpaceId,
+  onCompleted,
+  onDraftChange,
+  onDirtyChange,
+  onExit,
+  parameters,
+  parametersValid,
+  parameterSchemas,
+  provider,
+  providerRegion,
+}: {
+  connection: SourceConnection
+  draft: ConnectedSourceDraft
+  knowledgeSpaceId: string
+  onCompleted: () => void
+  onDraftChange: (draft: NewKnowledgeSourceDraft) => void
+  onDirtyChange: (dirty: boolean) => void
+  onExit: () => void
+  parameters: DatasourceParameters
+  parametersValid: boolean
+  parameterSchemas: DatasourceParameterSchema[]
+  provider: SourceProvider
+  providerRegion?: string
+}) {
+  const { t } = useTranslation('dataset')
+  const [appliedParameters, setAppliedParameters] = useState<DatasourceParameters | undefined>(
+    () => (parametersValid ? parameters : undefined),
+  )
+  const parametersApplied =
+    appliedParameters !== undefined &&
+    JSON.stringify(appliedParameters) === JSON.stringify(parameters)
+
+  return (
+    <>
+      <DatasourceParameterForm
+        parameters={parameters}
+        schemas={parameterSchemas}
+        onChange={(nextParameters) =>
+          onDraftChange({
+            ...draft,
+            parameters: nextParameters,
+          })
+        }
+      />
+      {parametersValid && !parametersApplied && (
+        <Button
+          type="button"
+          variant="primary"
+          className="w-full"
+          onClick={() => setAppliedParameters(parameters)}
+        >
+          {t(($) => $['newKnowledge.preview'])}
+        </Button>
+      )}
+      {parametersApplied && appliedParameters && (
+        <ResourceConfiguration
+          connection={connection}
+          draft={draft}
+          knowledgeSpaceId={knowledgeSpaceId}
+          onCompleted={onCompleted}
+          onDraftChange={onDraftChange}
+          onDirtyChange={onDirtyChange}
+          onExit={onExit}
+          parameters={appliedParameters}
+          provider={provider}
+          providerRegion={providerRegion}
+        />
+      )}
+    </>
+  )
+}
+
 export function ConnectedSourceSetup({
   draft,
   knowledgeSpaceId,
@@ -1558,6 +1656,18 @@ export function ConnectedSourceSetup({
   )
   const providerOption = sourceProviderOptionForDraft(providerOptions, draft)
   const installedProviderOption = providerOption?.installed ? providerOption : undefined
+  const parameterSchemas = useMemo(
+    () =>
+      installedProviderOption ? datasourceParameterSchemas(installedProviderOption.datasource) : [],
+    [installedProviderOption],
+  )
+  const parameters = useMemo(
+    () => withDatasourceParameterDefaults(parameterSchemas, draft.parameters),
+    [draft.parameters, parameterSchemas],
+  )
+  const parametersValid =
+    !missingRequiredDatasourceParameters(parameterSchemas, parameters).length &&
+    !invalidDatasourceParameters(parameterSchemas, parameters).length
   const provider = providerForDraft(providersQuery.data ?? [], draft, providerOption)
   const driveTransport = usesDriveTransport(draft)
   const datasourceProvider = datasourceProviderForOption(installedProviderOption)
@@ -1725,6 +1835,9 @@ export function ConnectedSourceSetup({
     if (draft.sourceType === 'onlineDocuments') {
       onDraftChange({
         ...draft,
+        parameters: nextProvider.installed
+          ? datasourceParameterDefaults(datasourceParameterSchemas(nextProvider.datasource))
+          : {},
         provider: nextProvider.label,
         providerKey: nextProvider.key,
         sourceName: '',
@@ -1733,6 +1846,9 @@ export function ConnectedSourceSetup({
     }
     onDraftChange({
       ...draft,
+      parameters: nextProvider.installed
+        ? datasourceParameterDefaults(datasourceParameterSchemas(nextProvider.datasource))
+        : {},
       provider: nextProvider.label,
       providerKey: nextProvider.key,
       sourceName: '',
@@ -1806,7 +1922,7 @@ export function ConnectedSourceSetup({
           </p>
         </div>
       ) : connection?.status === 'active' ? (
-        <ResourceConfiguration
+        <AppliedResourceConfiguration
           key={`${provider.id}:${connection.id}`}
           connection={connection}
           draft={draft}
@@ -1815,6 +1931,9 @@ export function ConnectedSourceSetup({
           onDraftChange={onDraftChange}
           onDirtyChange={onDirtyChange}
           onExit={onExit}
+          parameters={parameters}
+          parametersValid={parametersValid}
+          parameterSchemas={parameterSchemas}
           provider={provider}
           providerRegion={draft.provider === 'Amazon S3' ? credentialRegion(credential) : undefined}
         />

web/features/new-rag/crawl-selection-form.tsx

@@ -108,9 +108,8 @@ async function waitForImportTerminal(
   throw new Error('Source import did not reach a terminal state')
 }
 
-function pageSkipReason(page: PreviewPage, rootUrl: string): PageSkipReason | undefined {
+function pageSkipReason(page: PreviewPage, rootUrl?: string): PageSkipReason | undefined {
   try {
-    const root = new URL(rootUrl)
     const candidate = new URL(page.sourceUrl)
     if (
       !['http:', 'https:'].includes(candidate.protocol) ||
@@ -118,6 +117,8 @@ function pageSkipReason(page: PreviewPage, rootUrl: string): PageSkipReason | un
       candidate.password
     )
       return 'failed'
+    if (!rootUrl) return undefined
+    const root = new URL(rootUrl)
     if (candidate.hostname.toLocaleLowerCase() !== root.hostname.toLocaleLowerCase())
       return 'off-domain'
     return undefined
@@ -186,6 +187,7 @@ export function CrawlPreviewPageSelection({
   progressFailed = 0,
   recrawlDisabled,
   rootUrl,
+  sourceLabel,
   selectedPageIds,
 }: {
   busy?: boolean
@@ -195,7 +197,8 @@ export function CrawlPreviewPageSelection({
   pages: PreviewPage[]
   progressFailed?: number
   recrawlDisabled?: boolean
-  rootUrl: string
+  rootUrl?: string
+  sourceLabel?: string
   selectedPageIds: Set<string>
 }) {
   const { t } = useTranslation('dataset')
@@ -251,7 +254,7 @@ export function CrawlPreviewPageSelection({
         >
           {t(($) => $['newKnowledge.pagesCrawled'], {
             count: pages.length,
-            host: new URL(rootUrl).host,
+            host: sourceLabel ?? (rootUrl ? new URL(rootUrl).host : ''),
           })}
         </h3>
         <span className="system-xs-regular text-text-tertiary">
@@ -372,7 +375,7 @@ function ReadyCrawlSelectionForm({
   onWorkflowRun: (run: SourceWorkflowRun) => void
   pages: PreviewPage[]
   policy: SyncPolicy
-  rootUrl: string
+  rootUrl?: string
   run: SourceWorkflowRun
   source: Source
   workflowUncertain: boolean
@@ -765,7 +768,7 @@ export function CrawlSelectionForm({
   onWorkflowPending: (request: Promise<SourceWorkflowRun | undefined>) => void
   onWorkflowRun: (run: SourceWorkflowRun) => void
   pages: PreviewPage[]
-  rootUrl: string
+  rootUrl?: string
   run: SourceWorkflowRun
   source: Source
   workflowUncertain?: boolean

web/features/new-rag/create-connected-source-setup.tsx

@@ -18,6 +18,13 @@ import { useCallback, useEffect, useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import Loading from '@/app/components/base/loading'
 import { consoleClient } from '@/service/client'
+import { DatasourceParameterForm } from './datasource-parameter-form'
+import {
+  datasourceParameterSchemas,
+  invalidDatasourceParameters,
+  missingRequiredDatasourceParameters,
+  withDatasourceParameterDefaults,
+} from './datasource-parameter-model'
 import { SourceNameField, SourceSyncPolicyField } from './source-setup-fields'
 
 type ConnectedDraft = NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft
@@ -88,6 +95,17 @@ export function CreateConnectedSourceSetup({
     () => new Map(),
   )
   const driveTransport = providerOption.providerType === 'online_drive'
+  const parameterSchemas = useMemo(
+    () => datasourceParameterSchemas(providerOption.datasource),
+    [providerOption.datasource],
+  )
+  const parameters = useMemo(
+    () => withDatasourceParameterDefaults(parameterSchemas, draft.parameters),
+    [draft.parameters, parameterSchemas],
+  )
+  const parametersValid =
+    !missingRequiredDatasourceParameters(parameterSchemas, parameters).length &&
+    !invalidDatasourceParameters(parameterSchemas, parameters).length
   const selectionAtLimit = selected.size >= MAX_SELECTION
   const selectableResources = useMemo(
     () =>
@@ -121,6 +139,7 @@ export function CreateConnectedSourceSetup({
       pluginId: providerOption.plugin.plugin_id,
       provider: providerOption.plugin.provider,
       providerDisplayName: providerOption.label,
+      parameters,
     }
     if (!driveTransport) {
       onInitialSourceChange({
@@ -173,6 +192,7 @@ export function CreateConnectedSourceSetup({
     providerOption.label,
     providerOption.plugin.plugin_id,
     providerOption.plugin.provider,
+    parameters,
     selectableResources,
     selected,
   ])
@@ -193,6 +213,7 @@ export function CreateConnectedSourceSetup({
       parentKey?: string
       prefix?: string
     } = {}) => {
+      if (!parametersValid) return
       append ? setLoadingMore(true) : setLoading(true)
       setError(false)
       try {
@@ -202,6 +223,7 @@ export function CreateConnectedSourceSetup({
             datasource: providerOption.datasource.identity.name,
             kind: driveTransport ? 'online_drive' : 'online_document',
             parameters: {
+              ...parameters,
               ...(bucket ? { bucket } : {}),
               ...(prefix ? { prefix } : {}),
               ...(nextPage ? { next_page_parameters: nextPage } : {}),
@@ -255,7 +277,7 @@ export function CreateConnectedSourceSetup({
         setLoadingMore(false)
       }
     },
-    [credential.id, driveTransport, providerOption],
+    [credential.id, driveTransport, parameters, parametersValid, providerOption],
   )
 
   const toggle = (key: string) => {
@@ -321,13 +343,26 @@ export function CreateConnectedSourceSetup({
           onDraftChange={onDraftChange}
         />
       </div>
+      <DatasourceParameterForm
+        disabled={disabled || loading}
+        parameters={parameters}
+        schemas={parameterSchemas}
+        onChange={(nextParameters) => {
+          setResources([])
+          setSelected(new Set())
+          setExpanded(new Set())
+          setNextPageRequests(new Map())
+          setPreviewed(false)
+          onDraftChange({ ...draft, parameters: nextParameters })
+        }}
+      />
       {!previewed && (
         <Button
           type="button"
           variant="primary"
           className="w-full"
           loading={loading}
-          disabled={disabled || loading}
+          disabled={disabled || loading || !parametersValid}
           onClick={() => void requestPreview()}
         >
           {t(($) => $['newKnowledge.preview'])}

web/features/new-rag/create-source-setup.tsx

@@ -7,40 +7,29 @@ import type {
   DataSourceAuth,
   DataSourceCredential,
 } from '@/app/components/header/account-setting/data-source-page-new/types'
-import type { CrawlResultItem } from '@/models/datasets'
 import { Button } from '@langgenius/dify-ui/button'
-import { Checkbox } from '@langgenius/dify-ui/checkbox'
 import { cn } from '@langgenius/dify-ui/cn'
-import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from '@langgenius/dify-ui/collapsible'
-import { Field, FieldControl, FieldLabel } from '@langgenius/dify-ui/field'
 import { Fieldset, FieldsetLegend } from '@langgenius/dify-ui/fieldset'
-import {
-  NumberField,
-  NumberFieldControls,
-  NumberFieldDecrement,
-  NumberFieldGroup,
-  NumberFieldIncrement,
-  NumberFieldInput,
-} from '@langgenius/dify-ui/number-field'
 import { useEffect, useMemo, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { buildIntegrationPath } from '@/app/components/integrations/routes'
-import {
-  checkFirecrawlTaskStatus,
-  checkJinaReaderTaskStatus,
-  checkWatercrawlTaskStatus,
-  createFirecrawlTask,
-  createJinaReaderTask,
-  createWatercrawlTask,
-} from '@/service/datasets'
+import { consoleClient } from '@/service/client'
 import { useGetDataSourceListAuth } from '@/service/use-datasource'
 import { useDataSourceList } from '@/service/use-pipeline'
 import { CrawlPreviewPageSelection } from './crawl-selection-form'
 import { CreateConnectedSourceSetup } from './create-connected-source-setup'
-import { isValidWebsiteSourceDraft, NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH } from './routes'
+import { DatasourceParameterForm } from './datasource-parameter-form'
+import {
+  datasourceIncludeSubpages,
+  datasourceParameterDefaults,
+  datasourceParameterSchemas,
+  invalidDatasourceParameters,
+  missingRequiredDatasourceParameters,
+  websiteDatasourceParameterSchemas,
+  withDatasourceParameterDefaults,
+} from './datasource-parameter-model'
 import {
   discoverSourceProviderOptions,
-  normalizeSourceProviderName,
   sourceProviderOptionForDraft,
 } from './source-provider-options'
 import {
@@ -52,60 +41,17 @@ import {
   SourceTypeSelector,
 } from './source-setup-fields'
 
-const DEFAULT_INCLUDE_SUBPAGES = true
-const DEFAULT_MAX_PAGES = 100
-const CRAWL_POLL_INTERVAL_MS = 1500
 const CRAWL_PREVIEW_SKELETONS = [
   { id: 'short', sourceWidth: 'w-22.5', titleWidth: 'w-37.5' },
   { id: 'medium', sourceWidth: 'w-26', titleWidth: 'w-42.5' },
   { id: 'long', sourceWidth: 'w-29.5', titleWidth: 'w-47.5' },
   { id: 'longest', sourceWidth: 'w-33', titleWidth: 'w-52.5' },
 ] as const
+const CRAWL_POLL_INTERVAL_MS = 1500
 
 type LocalCrawlState = 'error' | 'idle' | 'running' | 'stopped' | 'success'
 type InitialSource = NonNullable<KnowledgeFsSpaceCreatePayload['initial_source']>
 
-function crawlPages(response: Record<string, unknown>): CrawlResultItem[] {
-  const items = Array.isArray(response.data)
-    ? response.data
-    : response.data && typeof response.data === 'object'
-      ? [response.data]
-      : []
-  return items.flatMap((item) => {
-    if (!item || typeof item !== 'object') return []
-    const page = item as Record<string, unknown>
-    const sourceUrl =
-      typeof page.source_url === 'string'
-        ? page.source_url
-        : typeof page.url === 'string'
-          ? page.url
-          : ''
-    if (!sourceUrl) return []
-    return [
-      {
-        description: typeof page.description === 'string' ? page.description : '',
-        markdown:
-          typeof page.markdown === 'string'
-            ? page.markdown
-            : typeof page.content === 'string'
-              ? page.content
-              : '',
-        source_url: sourceUrl,
-        title: typeof page.title === 'string' ? page.title : sourceUrl,
-      },
-    ]
-  })
-}
-
-function crawlPreviewPages(pages: CrawlResultItem[]): CrawlPreviewPage[] {
-  return pages.map((page) => ({
-    description: page.description,
-    pageId: page.source_url,
-    sourceUrl: page.source_url,
-    title: page.title,
-  }))
-}
-
 function datasourceAuthForProvider(
   authProviders: DataSourceAuth[],
   pluginId: string,
@@ -129,14 +75,18 @@ function providerIntegrationPath(packageId?: string) {
   return `${base}?${query.toString()}`
 }
 
-function websiteProviderTransport(provider: string) {
-  const normalized = normalizeSourceProviderName(provider)
-  if (normalized.includes('firecrawl'))
-    return { check: checkFirecrawlTaskStatus, create: createFirecrawlTask }
-  if (normalized.includes('jinareader') || normalized === 'jina')
-    return { check: checkJinaReaderTaskStatus, create: createJinaReaderTask }
-  if (normalized.includes('watercrawl'))
-    return { check: checkWatercrawlTaskStatus, create: createWatercrawlTask }
+function websiteSourceUri(parameters: Record<string, boolean | number | string>, fallback: string) {
+  const url = parameters.url
+  if (typeof url === 'string') {
+    try {
+      const parsed = new URL(url)
+      if (['http:', 'https:'].includes(parsed.protocol) && !parsed.username && !parsed.password)
+        return parsed.toString()
+    } catch {
+      // Non-URL website datasources use a stable synthetic URI.
+    }
+  }
+  return `datasource://${encodeURIComponent(fallback)}`
 }
 
 export function CreateSourceSetup({
@@ -155,13 +105,12 @@ export function CreateSourceSetup({
   const { t } = useTranslation('dataset')
   const datasourcePluginsQuery = useDataSourceList(true)
   const datasourceAuthQuery = useGetDataSourceListAuth()
-  const [optionsExpanded, setOptionsExpanded] = useState(false)
   const [crawlState, setCrawlState] = useState<LocalCrawlState>('idle')
-  const [previewPages, setPreviewPages] = useState<CrawlResultItem[]>([])
+  const [stoppingPreview, setStoppingPreview] = useState(false)
+  const [previewPages, setPreviewPages] = useState<CrawlPreviewPage[]>([])
   const [selectedPageIds, setSelectedPageIds] = useState<Set<string>>(() => new Set())
   const crawlAttemptRef = useRef(0)
-  const pollResolveRef = useRef<(() => void) | undefined>(undefined)
-  const pollTimerRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined)
+  const previewJobIdRef = useRef<string | undefined>(undefined)
   const sourceType = draft.sourceType
   const providerOptions = useMemo(
     () => discoverSourceProviderOptions(sourceType, datasourcePluginsQuery.data ?? []),
@@ -177,31 +126,95 @@ export function CreateSourceSetup({
       )
     : undefined
   const credential = preferredCredential(datasourceAuth)
-  const websiteTransport =
-    draft.sourceType === 'websiteCrawl' && installedProviderOption
-      ? websiteProviderTransport(installedProviderOption.plugin.provider)
-      : undefined
-  const previewReady = Boolean(
-    websiteTransport &&
-    credential &&
-    draft.sourceType === 'websiteCrawl' &&
-    isValidWebsiteSourceDraft(draft),
+  const parameterSchemas = useMemo(
+    () =>
+      draft.sourceType === 'websiteCrawl' && installedProviderOption
+        ? websiteDatasourceParameterSchemas(installedProviderOption.datasource)
+        : [],
+    [draft.sourceType, installedProviderOption],
   )
-  const previewRootUrl = draft.sourceType === 'websiteCrawl' ? draft.rootUrl : ''
-  const selectionPages = useMemo(() => crawlPreviewPages(previewPages), [previewPages])
-  const crawlOptionsAreDefault =
-    draft.sourceType !== 'websiteCrawl' ||
-    (draft.includeSubpages === DEFAULT_INCLUDE_SUBPAGES && draft.maxPages === DEFAULT_MAX_PAGES)
-  const stopPreview = (state: LocalCrawlState = 'stopped') => {
+  const parameters = useMemo(() => {
+    const current = withDatasourceParameterDefaults(parameterSchemas, draft.parameters)
+    if (
+      draft.sourceType === 'websiteCrawl' &&
+      draft.rootUrl &&
+      parameterSchemas.some((parameter) => parameter.name === 'url') &&
+      current.url === undefined
+    )
+      current.url = draft.rootUrl
+    return current
+  }, [draft, parameterSchemas])
+  const parametersValid =
+    !missingRequiredDatasourceParameters(parameterSchemas, parameters).length &&
+    !invalidDatasourceParameters(parameterSchemas, parameters).length
+  const selectionPages = previewPages
+  const previewReady = Boolean(
+    draft.sourceType === 'websiteCrawl' &&
+    credential &&
+    installedProviderOption &&
+    parametersValid &&
+    draft.sourceName.trim(),
+  )
+  const sourceUri = installedProviderOption
+    ? websiteSourceUri(parameters, installedProviderOption.key)
+    : ''
+  const selectionRootUrl =
+    typeof parameters.url === 'string' && sourceUri.startsWith('http') ? sourceUri : undefined
+  const cancelPreviewBestEffort = () => {
     crawlAttemptRef.current += 1
-    globalThis.clearTimeout(pollTimerRef.current)
-    pollTimerRef.current = undefined
-    pollResolveRef.current?.()
-    pollResolveRef.current = undefined
-    setCrawlState(state)
+    const jobId = previewJobIdRef.current
+    previewJobIdRef.current = undefined
+    if (jobId)
+      void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId
+        .delete({
+          params: { job_id: jobId },
+        })
+        .catch(() => {})
+  }
+  const stopPreview = async () => {
+    if (stoppingPreview) return
+    crawlAttemptRef.current += 1
+    const jobId = previewJobIdRef.current
+    if (!jobId) {
+      setCrawlState('stopped')
+      return
+    }
+    setStoppingPreview(true)
+    try {
+      const response = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId.delete({
+        params: { job_id: jobId },
+      })
+      if (response.status === 'completed' && response.result) {
+        if (previewJobIdRef.current === jobId) previewJobIdRef.current = undefined
+        setPreviewPages(
+          (response.result.pages ?? []).map((page) => ({
+            description: page.description ?? undefined,
+            pageId: page.source_url,
+            sourceUrl: page.source_url,
+            title: page.title ?? page.source_url,
+          })),
+        )
+        setCrawlState('success')
+      } else if (response.status === 'canceled') {
+        if (previewJobIdRef.current === jobId) previewJobIdRef.current = undefined
+        setCrawlState('stopped')
+      } else if (response.status === 'failed') {
+        if (previewJobIdRef.current === jobId) previewJobIdRef.current = undefined
+        setCrawlState('error')
+      } else {
+        setCrawlState('running')
+      }
+    } catch {
+      // Keep the job handle and running state so Stop can be retried and unmount
+      // cleanup can still revoke a task whose cancellation response was lost.
+      setCrawlState('running')
+    } finally {
+      setStoppingPreview(false)
+    }
   }
   const resetPreview = () => {
-    stopPreview('idle')
+    cancelPreviewBestEffort()
+    setCrawlState('idle')
     setPreviewPages([])
     setSelectedPageIds(new Set())
     onInitialSourceChange(undefined)
@@ -218,9 +231,17 @@ export function CreateSourceSetup({
     if (!nextProvider) return
     updateDraft({
       ...draft,
+      parameters: nextProvider.installed
+        ? datasourceParameterDefaults(
+            draft.sourceType === 'websiteCrawl'
+              ? websiteDatasourceParameterSchemas(nextProvider.datasource)
+              : datasourceParameterSchemas(nextProvider.datasource),
+          )
+        : {},
       provider: nextProvider.label,
       providerKey: nextProvider.key,
       sourceName: '',
+      ...(draft.sourceType === 'websiteCrawl' ? { rootUrl: '' } : {}),
       ...(draft.sourceType === 'onlineDrive' &&
       nextProvider.label === 'Amazon S3' &&
       draft.syncPolicy === 'provider'
@@ -232,8 +253,13 @@ export function CreateSourceSetup({
   useEffect(
     () => () => {
       crawlAttemptRef.current += 1
-      globalThis.clearTimeout(pollTimerRef.current)
-      pollResolveRef.current?.()
+      const jobId = previewJobIdRef.current
+      if (jobId)
+        void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId
+          .delete({
+            params: { job_id: jobId },
+          })
+          .catch(() => {})
     },
     [],
   )
@@ -241,62 +267,77 @@ export function CreateSourceSetup({
   const startPreview = async () => {
     if (
       draft.sourceType !== 'websiteCrawl' ||
-      !isValidWebsiteSourceDraft(draft) ||
-      !websiteTransport ||
+      !previewReady ||
       !credential ||
       !installedProviderOption
     )
       return
     const attempt = crawlAttemptRef.current + 1
     crawlAttemptRef.current = attempt
-    globalThis.clearTimeout(pollTimerRef.current)
     setPreviewPages([])
     setSelectedPageIds(new Set())
     setCrawlState('running')
     try {
-      const created = (await websiteTransport.create({
-        options: {
-          crawl_sub_pages: draft.includeSubpages,
-          excludes: '',
-          includes: '',
-          limit: draft.maxPages,
-          max_depth: '',
-          only_main_content: true,
-          use_sitemap: true,
+      const job = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.post({
+        body: {
+          credentialId: credential.id,
+          datasource: installedProviderOption.datasource.identity.name,
+          kind: 'website_crawl',
+          parameters,
+          pluginId: installedProviderOption.plugin.plugin_id,
+          provider: installedProviderOption.plugin.provider,
+          providerDisplayName: installedProviderOption.label,
         },
-        url: draft.rootUrl,
-      })) as Record<string, unknown>
-      const synchronousPages = crawlPages(created)
-      if (synchronousPages.length) {
-        setPreviewPages(synchronousPages)
-        setCrawlState('success')
+      })
+      if (crawlAttemptRef.current !== attempt) {
+        void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId
+          .delete({
+            params: { job_id: job.job_id },
+          })
+          .catch(() => {})
         return
       }
-      const jobId = typeof created.job_id === 'string' ? created.job_id : undefined
-      if (!jobId) throw new Error('Website crawl did not return a job id')
-
-      while (crawlAttemptRef.current === attempt) {
-        const response = (await websiteTransport.check(jobId)) as Record<string, unknown>
+      previewJobIdRef.current = job.job_id
+      let response = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId.get({
+        params: { job_id: job.job_id },
+      })
+      while (
+        crawlAttemptRef.current === attempt &&
+        !['completed', 'failed', 'canceled'].includes(response.status)
+      ) {
+        await new Promise((resolve) => globalThis.setTimeout(resolve, CRAWL_POLL_INTERVAL_MS))
         if (crawlAttemptRef.current !== attempt) return
-        setPreviewPages(crawlPages(response))
-        if (response.status === 'completed') {
-          setCrawlState('success')
-          return
-        }
-        if (response.status === 'error' || !response.status) {
-          setCrawlState('error')
-          return
-        }
-        await new Promise<void>((resolve) => {
-          pollResolveRef.current = resolve
-          pollTimerRef.current = globalThis.setTimeout(() => {
-            pollResolveRef.current = undefined
-            resolve()
-          }, CRAWL_POLL_INTERVAL_MS)
+        response = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId.get({
+          params: { job_id: job.job_id },
         })
       }
+      if (crawlAttemptRef.current !== attempt) return
+      previewJobIdRef.current = undefined
+      if (response.status !== 'completed' || !response.result) {
+        setCrawlState(response.status === 'canceled' ? 'stopped' : 'error')
+        return
+      }
+      setPreviewPages(
+        (response.result.pages ?? []).map((page) => ({
+          description: page.description ?? undefined,
+          pageId: page.source_url,
+          sourceUrl: page.source_url,
+          title: page.title ?? page.source_url,
+        })),
+      )
+      setCrawlState('success')
     } catch {
-      if (crawlAttemptRef.current === attempt) setCrawlState('error')
+      if (crawlAttemptRef.current === attempt) {
+        const jobId = previewJobIdRef.current
+        previewJobIdRef.current = undefined
+        if (jobId)
+          void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId
+            .delete({
+              params: { job_id: jobId },
+            })
+            .catch(() => {})
+        setCrawlState('error')
+      }
     }
   }
 
@@ -322,8 +363,8 @@ export function CreateSourceSetup({
     }
     onInitialSourceChange({
       crawl_options: {
-        include_subpages: draft.includeSubpages,
-        limit: draft.maxPages,
+        include_subpages: datasourceIncludeSubpages(parameters),
+        limit: typeof parameters.limit === 'number' ? parameters.limit : 200,
       },
       credentialId: credential.id,
       datasource: installedProviderOption.datasource.identity.name,
@@ -332,7 +373,8 @@ export function CreateSourceSetup({
       pluginId: installedProviderOption.plugin.plugin_id,
       provider: installedProviderOption.plugin.provider,
       providerDisplayName: installedProviderOption.label,
-      root_url: draft.rootUrl,
+      parameters,
+      root_url: sourceUri,
       selection: selectedPages.map((page) => ({
         source_url: page.sourceUrl,
         ...(page.title ? { title: page.title } : {}),
@@ -345,8 +387,10 @@ export function CreateSourceSetup({
     draft,
     installedProviderOption,
     onInitialSourceChange,
+    parameters,
     selectedPageIds,
     selectionPages,
+    sourceUri,
   ])
 
   return (
@@ -447,124 +491,28 @@ export function CreateSourceSetup({
             )
           }
         />
-      ) : draft.sourceType === 'websiteCrawl' &&
-        installedProviderOption &&
-        credential &&
-        !websiteTransport ? (
-        <div className="rounded-xl bg-background-section p-4">
-          <p className="system-sm-semibold text-text-primary">{installedProviderOption.label}</p>
-          <p className="mt-1 system-xs-regular text-text-tertiary">
-            {t(($) => $['newKnowledge.providerUnavailable'])}
-          </p>
-        </div>
       ) : draft.sourceType === 'websiteCrawl' && installedProviderOption && credential ? (
         <div className="space-y-4">
-          <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
-            <Field name="rootUrl" className="gap-1.5">
-              <FieldLabel className="py-0.5">
-                {t(($) => $['newKnowledge.rootUrl'])}
-                <span aria-hidden className="ml-0.5 text-text-destructive">
-                  *
-                </span>
-              </FieldLabel>
-              <FieldControl
-                type="url"
-                inputMode="url"
-                autoComplete="off"
-                disabled={disabled}
-                maxLength={NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH}
-                value={draft.rootUrl}
-                placeholder={t(($) => $['newKnowledge.rootUrlPlaceholder'])}
-                size="large"
-                onValueChange={(value) => {
-                  updateDraft({ ...draft, rootUrl: value })
-                }}
-                onKeyDown={(event) => {
-                  if (event.key === 'Enter') event.preventDefault()
-                }}
-              />
-            </Field>
-            <SourceNameField
-              disabled={disabled}
-              draft={draft}
-              labelClassName="py-0.5"
-              preventSubmitOnEnter
-              size="large"
-              onDraftChange={updateDraft}
-            />
-          </div>
-          <Collapsible
-            open={optionsExpanded}
-            onOpenChange={setOptionsExpanded}
-            className="overflow-hidden rounded-lg bg-background-section"
-          >
-            <CollapsibleTrigger
-              aria-label={t(($) => $['newKnowledge.crawlOptions'])}
-              disabled={disabled}
-              className="min-h-9 justify-start px-3 system-xs-medium"
-            >
-              <span
-                aria-hidden
-                className="i-ri-arrow-right-s-line size-4 transition-transform group-data-panel-open:rotate-90 motion-reduce:transition-none"
-              />
-              {t(($) => $['newKnowledge.crawlOptions'])}
-              {!optionsExpanded && (
-                <span className="ml-auto system-xs-regular text-text-tertiary">
-                  {crawlOptionsAreDefault
-                    ? t(($) => $['newKnowledge.usingDefaults'])
-                    : `${t(($) => $['newKnowledge.includeSubpages'])}: ${t(($) =>
-                        draft.includeSubpages
-                          ? $['newKnowledge.booleanTrue']
-                          : $['newKnowledge.booleanFalse'],
-                      )} · ${t(($) => $['newKnowledge.maxPages'])}: ${draft.maxPages}`}
-                </span>
-              )}
-            </CollapsibleTrigger>
-            <CollapsiblePanel>
-              <Fieldset
-                disabled={disabled}
-                className="grid grid-cols-1 gap-3 px-3 pb-3 sm:grid-cols-2"
-              >
-                <label className="flex items-center gap-2 system-xs-regular text-text-secondary">
-                  <Checkbox
-                    name="includeSubpages"
-                    checked={draft.includeSubpages}
-                    disabled={disabled}
-                    onCheckedChange={(checked) =>
-                      updateDraft({ ...draft, includeSubpages: checked })
-                    }
-                  />
-                  {t(($) => $['newKnowledge.includeSubpages'])}
-                </label>
-                <div>
-                  <span className="system-xs-medium text-text-secondary">
-                    {t(($) => $['newKnowledge.maxPages'])}
-                  </span>
-                  <NumberField
-                    disabled={disabled}
-                    name="maxPages"
-                    min={1}
-                    max={200}
-                    value={draft.maxPages}
-                    onValueChange={(value) => updateDraft({ ...draft, maxPages: value ?? 0 })}
-                  >
-                    <NumberFieldGroup className="mt-1.5">
-                      <NumberFieldInput
-                        aria-label={t(($) => $['newKnowledge.maxPages'])}
-                        onKeyDown={(event) => {
-                          if (event.key === 'Enter') event.preventDefault()
-                        }}
-                      />
-                      <NumberFieldControls>
-                        <NumberFieldIncrement />
-                        <NumberFieldDecrement />
-                      </NumberFieldControls>
-                    </NumberFieldGroup>
-                  </NumberField>
-                </div>
-              </Fieldset>
-            </CollapsiblePanel>
-          </Collapsible>
+          <SourceNameField
+            disabled={disabled}
+            draft={draft}
+            labelClassName="py-0.5"
+            preventSubmitOnEnter
+            size="large"
+            onDraftChange={updateDraft}
+          />
+          <DatasourceParameterForm
+            disabled={disabled || crawlState === 'running'}
+            parameters={parameters}
+            schemas={parameterSchemas}
+            onChange={(nextParameters) =>
+              updateDraft({
+                ...draft,
+                parameters: nextParameters,
+                rootUrl: typeof nextParameters.url === 'string' ? nextParameters.url : '',
+              })
+            }
+          />
           {crawlState !== 'success' && (
             <Button
               type="button"
@@ -614,7 +562,7 @@ export function CreateSourceSetup({
                 >
                   {t(($) => $['newKnowledge.crawlingPages'], {
                     count: previewPages.length,
-                    host: new URL(draft.rootUrl).host,
+                    host: installedProviderOption.label,
                   })}
                 </p>
                 <Button
@@ -622,7 +570,9 @@ export function CreateSourceSetup({
                   variant="ghost-accent"
                   size="small"
                   className="ml-auto shrink-0"
-                  onClick={() => stopPreview()}
+                  disabled={stoppingPreview}
+                  loading={stoppingPreview}
+                  onClick={() => void stopPreview()}
                 >
                   {t(($) => $['newKnowledge.stopCrawl'])}
                 </Button>
@@ -664,8 +614,9 @@ export function CreateSourceSetup({
                 onRecrawl={() => void startPreview()}
                 onSelectionChange={updateSelectedPageIds}
                 pages={selectionPages}
-                rootUrl={previewRootUrl}
+                rootUrl={selectionRootUrl}
                 selectedPageIds={selectedPageIds}
+                sourceLabel={installedProviderOption.label}
               />
             )}
           </section>

web/features/new-rag/datasource-parameter-form.tsx

@@ -0,0 +1,263 @@
+'use client'
+
+import type {
+  DatasourceParameters,
+  DatasourceParameterSchema,
+  DatasourceParameterValue,
+} from './datasource-parameter-model'
+import {
+  Field,
+  FieldControl,
+  FieldDescription,
+  FieldError,
+  FieldLabel,
+} from '@langgenius/dify-ui/field'
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectItemIndicator,
+  SelectItemText,
+  SelectLabel,
+  SelectTrigger,
+} from '@langgenius/dify-ui/select'
+import { Switch } from '@langgenius/dify-ui/switch'
+import { useId, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { invalidDatasourceParameters, localizedDatasourceText } from './datasource-parameter-model'
+
+function ParameterLabel({ label, required }: { label: string; required: boolean }) {
+  return (
+    <>
+      {label}
+      {required && (
+        <span aria-hidden className="ml-0.5 text-text-destructive">
+          *
+        </span>
+      )}
+    </>
+  )
+}
+
+function DatasourceParameterField({
+  disabled,
+  parameter,
+  value,
+  onChange,
+}: {
+  disabled: boolean
+  parameter: DatasourceParameterSchema
+  value: DatasourceParameterValue | undefined
+  onChange: (value: DatasourceParameterValue | undefined) => void
+}) {
+  const { i18n, t } = useTranslation('dataset')
+  const generatedId = useId()
+  const [numberDraft, setNumberDraft] = useState<string>()
+  const description = localizedDatasourceText(parameter.description, i18n.language, '')
+  const descriptionId = description ? `${generatedId}-description` : undefined
+  const errorId = `${generatedId}-error`
+  const label =
+    parameter.labelTranslationKey === 'newKnowledge.rootUrl'
+      ? t(($) => $['newKnowledge.rootUrl'])
+      : parameter.labelTranslationKey === 'newKnowledge.includeSubpages'
+        ? t(($) => $['newKnowledge.includeSubpages'])
+        : parameter.labelTranslationKey === 'newKnowledge.maxPages'
+          ? t(($) => $['newKnowledge.maxPages'])
+          : localizedDatasourceText(parameter.label, i18n.language, parameter.name)
+  const placeholder = parameter.placeholderTranslationKey
+    ? t(($) => $['newKnowledge.rootUrlPlaceholder'])
+    : localizedDatasourceText(parameter.placeholder, i18n.language, '')
+  const invalid = Boolean(
+    (parameter.required && (value === undefined || (typeof value === 'string' && !value.trim()))) ||
+    invalidDatasourceParameters([parameter], {
+      ...(value === undefined ? {} : { [parameter.name]: value }),
+    }).length,
+  )
+  const error = invalid
+    ? parameter.name === 'url' && typeof value === 'string' && value
+      ? t(($) => $['newKnowledge.invalidRootUrl'])
+      : parameter.labelTranslationKey === 'newKnowledge.maxPages' &&
+          parameter.min !== undefined &&
+          parameter.max !== undefined
+        ? `${label}: ${parameter.min}–${parameter.max}`
+        : t(($) => $['newKnowledge.invalidDatasourceParameter'], { parameter: label })
+    : undefined
+  const describedBy =
+    [descriptionId, error ? errorId : undefined].filter(Boolean).join(' ') || undefined
+  const numberStep = parameter.integer
+    ? 1
+    : parameter.precision === undefined
+      ? 'any'
+      : 10 ** -parameter.precision
+
+  if (parameter.type === 'unsupported')
+    return (
+      <Field name={parameter.name} className="gap-1.5" invalid>
+        <FieldLabel>
+          <ParameterLabel label={label} required={parameter.required} />
+        </FieldLabel>
+        <FieldDescription>{t(($) => $['newKnowledge.providerUnavailable'])}</FieldDescription>
+      </Field>
+    )
+
+  if (parameter.type === 'boolean')
+    return (
+      <Field name={parameter.name} className="gap-1.5" invalid={invalid}>
+        <div className="flex min-h-8 items-center justify-between gap-3">
+          <FieldLabel htmlFor={generatedId}>
+            <ParameterLabel label={label} required={parameter.required} />
+          </FieldLabel>
+          <Switch
+            id={generatedId}
+            checked={value === true}
+            disabled={disabled}
+            aria-describedby={describedBy}
+            onCheckedChange={(checked) => onChange(checked)}
+          />
+        </div>
+        {description && <FieldDescription id={descriptionId}>{description}</FieldDescription>}
+        {error && (
+          <FieldError id={errorId} match>
+            {error}
+          </FieldError>
+        )}
+      </Field>
+    )
+
+  if (parameter.type === 'select') {
+    const selectedValue = typeof value === 'string' ? value : null
+    return (
+      <Field name={parameter.name} className="gap-1.5" invalid={invalid}>
+        <Select<string | null>
+          name={parameter.name}
+          disabled={disabled}
+          required={parameter.required}
+          value={selectedValue}
+          onValueChange={(nextValue) => onChange(nextValue ?? undefined)}
+        >
+          <SelectLabel>
+            <ParameterLabel label={label} required={parameter.required} />
+          </SelectLabel>
+          <SelectTrigger aria-describedby={describedBy} size="large">
+            {parameter.options.find((option) => option.value === selectedValue)
+              ? localizedDatasourceText(
+                  parameter.options.find((option) => option.value === selectedValue)?.label,
+                  i18n.language,
+                  selectedValue ?? '—',
+                )
+              : '—'}
+          </SelectTrigger>
+          <SelectContent>
+            {!parameter.required && (
+              <SelectItem value={null}>
+                <SelectItemText>—</SelectItemText>
+                <SelectItemIndicator />
+              </SelectItem>
+            )}
+            {parameter.options.map((option) => (
+              <SelectItem key={option.value} value={option.value}>
+                <SelectItemText>
+                  {localizedDatasourceText(option.label, i18n.language, option.value)}
+                </SelectItemText>
+                <SelectItemIndicator />
+              </SelectItem>
+            ))}
+          </SelectContent>
+        </Select>
+        {description && <FieldDescription id={descriptionId}>{description}</FieldDescription>}
+        {error && (
+          <FieldError id={errorId} match>
+            {error}
+          </FieldError>
+        )}
+      </Field>
+    )
+  }
+
+  return (
+    <Field name={parameter.name} className="gap-1.5" invalid={invalid}>
+      <FieldLabel>
+        <ParameterLabel label={label} required={parameter.required} />
+      </FieldLabel>
+      <FieldControl
+        aria-describedby={describedBy}
+        autoComplete="off"
+        disabled={disabled}
+        inputMode={
+          parameter.type === 'number'
+            ? parameter.integer || parameter.precision === 0
+              ? 'numeric'
+              : 'decimal'
+            : undefined
+        }
+        max={parameter.max}
+        maxLength={parameter.name === 'url' ? 2048 : undefined}
+        min={parameter.min}
+        placeholder={placeholder || undefined}
+        required={parameter.required}
+        size="large"
+        step={parameter.type === 'number' ? numberStep : undefined}
+        type={parameter.type === 'number' ? 'number' : parameter.name === 'url' ? 'url' : 'text'}
+        value={
+          parameter.type === 'number' && numberDraft !== undefined
+            ? numberDraft
+            : value === undefined
+              ? ''
+              : String(value)
+        }
+        onValueChange={(nextValue) => {
+          if (parameter.type === 'number') {
+            setNumberDraft(nextValue)
+            const number = Number(nextValue)
+            onChange(nextValue && Number.isFinite(number) ? number : undefined)
+            return
+          }
+          onChange(nextValue || undefined)
+        }}
+        onBlur={() => {
+          if (parameter.type !== 'number' || numberDraft === undefined) return
+          setNumberDraft(undefined)
+        }}
+      />
+      {description && <FieldDescription id={descriptionId}>{description}</FieldDescription>}
+      {error && (
+        <FieldError id={errorId} match>
+          {error}
+        </FieldError>
+      )}
+    </Field>
+  )
+}
+
+export function DatasourceParameterForm({
+  disabled = false,
+  parameters,
+  schemas,
+  onChange,
+}: {
+  disabled?: boolean
+  parameters: DatasourceParameters
+  schemas: DatasourceParameterSchema[]
+  onChange: (parameters: DatasourceParameters) => void
+}) {
+  if (!schemas.length) return null
+
+  return (
+    <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
+      {schemas.map((parameter) => (
+        <DatasourceParameterField
+          key={parameter.name}
+          disabled={disabled}
+          parameter={parameter}
+          value={parameters[parameter.name]}
+          onChange={(value) => {
+            const next = { ...parameters }
+            if (value === undefined) delete next[parameter.name]
+            else next[parameter.name] = value
+            onChange(next)
+          }}
+        />
+      ))}
+    </div>
+  )
+}

web/features/new-rag/datasource-parameter-model.ts

@@ -0,0 +1,269 @@
+import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
+
+type Datasource = DataSourceItem['declaration']['datasources'][number]
+
+const LEGACY_WEBSITE_PARAMETER_SCHEMAS: DatasourceParameterSchema[] = [
+  {
+    label: {},
+    labelTranslationKey: 'newKnowledge.rootUrl',
+    name: 'url',
+    options: [],
+    placeholder: {},
+    placeholderTranslationKey: 'newKnowledge.rootUrlPlaceholder',
+    required: true,
+    type: 'string',
+  },
+  {
+    defaultValue: true,
+    label: {},
+    labelTranslationKey: 'newKnowledge.includeSubpages',
+    name: 'crawl_subpages',
+    options: [],
+    required: false,
+    type: 'boolean',
+  },
+  {
+    defaultValue: 100,
+    integer: true,
+    label: {},
+    labelTranslationKey: 'newKnowledge.maxPages',
+    max: 200,
+    min: 1,
+    name: 'limit',
+    options: [],
+    required: false,
+    type: 'number',
+  },
+]
+
+export type DatasourceParameterValue = boolean | number | string
+export type DatasourceParameters = Record<string, DatasourceParameterValue>
+
+export type DatasourceParameterOption = {
+  label: Record<string, string | undefined>
+  value: string
+}
+
+export type DatasourceParameterSchema = {
+  defaultValue?: DatasourceParameterValue
+  description?: Record<string, string | undefined>
+  integer?: boolean
+  label: Record<string, string | undefined>
+  labelTranslationKey?:
+    | 'newKnowledge.includeSubpages'
+    | 'newKnowledge.maxPages'
+    | 'newKnowledge.rootUrl'
+  max?: number
+  min?: number
+  name: string
+  options: DatasourceParameterOption[]
+  placeholder?: Record<string, string | undefined>
+  placeholderTranslationKey?: 'newKnowledge.rootUrlPlaceholder'
+  precision?: number
+  required: boolean
+  type: 'boolean' | 'number' | 'select' | 'string' | 'unsupported'
+}
+
+function record(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === 'object' && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined
+}
+
+function localizedText(value: unknown): Record<string, string | undefined> {
+  const candidate = record(value)
+  if (!candidate) return {}
+  return Object.fromEntries(
+    Object.entries(candidate).flatMap(([key, text]) =>
+      typeof text === 'string' ? [[key, text] as const] : [],
+    ),
+  )
+}
+
+function finiteNumber(value: unknown) {
+  return typeof value === 'number' && Number.isFinite(value) ? value : undefined
+}
+
+function boundedPrecision(value: unknown) {
+  return Number.isSafeInteger(value) && Number(value) >= 0 && Number(value) <= 20
+    ? Number(value)
+    : undefined
+}
+
+function parameterType(value: unknown): DatasourceParameterSchema['type'] | undefined {
+  if (value === 'boolean') return 'boolean'
+  if (value === 'number' || value === 'integer') return 'number'
+  if (value === 'select') return 'select'
+  if (value === 'string' || value === 'text-input') return 'string'
+  if (['file', 'files', 'secret-input', 'system-files'].includes(String(value)))
+    return 'unsupported'
+}
+
+function defaultValue(
+  value: unknown,
+  type: DatasourceParameterSchema['type'],
+): DatasourceParameterValue | undefined {
+  if (type === 'boolean' && typeof value === 'boolean') return value
+  if (type === 'number' && typeof value === 'number' && Number.isFinite(value)) return value
+  if ((type === 'select' || type === 'string') && typeof value === 'string') return value
+}
+
+export function datasourceParameterSchemas(datasource: Datasource): DatasourceParameterSchema[] {
+  return datasource.parameters.flatMap((rawParameter) => {
+    const parameter = record(rawParameter)
+    if (!parameter || typeof parameter.name !== 'string' || !parameter.name.trim()) return []
+    const type = parameterType(parameter.type)
+    if (!type) return []
+    const options = Array.isArray(parameter.options)
+      ? parameter.options.flatMap((rawOption) => {
+          const option = record(rawOption)
+          if (!option || typeof option.value !== 'string') return []
+          return [
+            {
+              label: localizedText(option.label),
+              value: option.value,
+            },
+          ]
+        })
+      : []
+    return [
+      {
+        ...(defaultValue(parameter.default, type) === undefined
+          ? {}
+          : { defaultValue: defaultValue(parameter.default, type) }),
+        description: localizedText(parameter.description),
+        ...(parameter.type === 'integer' ? { integer: true } : {}),
+        label: localizedText(parameter.label),
+        ...(finiteNumber(parameter.max) === undefined ? {} : { max: finiteNumber(parameter.max) }),
+        ...(finiteNumber(parameter.min) === undefined ? {} : { min: finiteNumber(parameter.min) }),
+        name: parameter.name.trim(),
+        options,
+        placeholder: localizedText(parameter.placeholder),
+        ...(boundedPrecision(parameter.precision) === undefined
+          ? {}
+          : { precision: boundedPrecision(parameter.precision) }),
+        required: parameter.required === true,
+        type,
+      },
+    ]
+  })
+}
+
+export function websiteDatasourceParameterSchemas(datasource?: Datasource) {
+  const schemas = datasource ? datasourceParameterSchemas(datasource) : []
+  if (schemas.length) return schemas
+  const providerIdentity = [datasource?.identity?.name, datasource?.identity?.provider]
+    .filter(Boolean)
+    .join(' ')
+    .toLocaleLowerCase()
+  const crawlSubpagesName = /jina|watercrawl/.test(providerIdentity)
+    ? 'crawl_sub_pages'
+    : 'crawl_subpages'
+  return LEGACY_WEBSITE_PARAMETER_SCHEMAS.map((schema) =>
+    schema.name === 'crawl_subpages' ? { ...schema, name: crawlSubpagesName } : schema,
+  )
+}
+
+export function datasourceIncludeSubpages(parameters: DatasourceParameters, fallback = true) {
+  const value = parameters.crawl_subpages ?? parameters.crawl_sub_pages
+  return typeof value === 'boolean' ? value : fallback
+}
+
+export function datasourceParameterDefaults(schemas: DatasourceParameterSchema[]) {
+  return Object.fromEntries(
+    schemas.flatMap((schema) =>
+      schema.defaultValue === undefined ? [] : ([[schema.name, schema.defaultValue]] as const),
+    ),
+  ) satisfies DatasourceParameters
+}
+
+export function withDatasourceParameterDefaults(
+  schemas: DatasourceParameterSchema[],
+  parameters: DatasourceParameters | undefined,
+) {
+  return {
+    ...datasourceParameterDefaults(schemas),
+    ...parameters,
+  }
+}
+
+export function missingRequiredDatasourceParameters(
+  schemas: DatasourceParameterSchema[],
+  parameters: DatasourceParameters,
+) {
+  return schemas.filter((schema) => {
+    if (!schema.required) return false
+    const value = parameters[schema.name]
+    if (schema.type === 'unsupported') return true
+    return value === undefined || (typeof value === 'string' && !value.trim())
+  })
+}
+
+export function invalidDatasourceParameters(
+  schemas: DatasourceParameterSchema[],
+  parameters: DatasourceParameters,
+) {
+  return schemas.filter((schema) => {
+    const value = parameters[schema.name]
+    if (value === undefined || value === '') return false
+    if (schema.type === 'unsupported') return true
+    if (schema.type === 'boolean') return typeof value !== 'boolean'
+    if (schema.type === 'number')
+      return (
+        typeof value !== 'number' ||
+        !Number.isFinite(value) ||
+        (schema.integer === true && !Number.isInteger(value)) ||
+        (schema.precision !== undefined &&
+          Math.abs(value - Number(value.toFixed(schema.precision))) >
+            Number.EPSILON * Math.max(1, Math.abs(value))) ||
+        (schema.min !== undefined && value < schema.min) ||
+        (schema.max !== undefined && value > schema.max)
+      )
+    if (typeof value !== 'string') return true
+    if (schema.type === 'select')
+      return (
+        Boolean(schema.options.length) && !schema.options.some((option) => option.value === value)
+      )
+    if (schema.name !== 'url') return false
+    try {
+      const parsed = new URL(value)
+      return (
+        !['http:', 'https:'].includes(parsed.protocol) ||
+        Boolean(parsed.username || parsed.password)
+      )
+    } catch {
+      return true
+    }
+  })
+}
+
+export function datasourceParameterRecord(value: unknown): DatasourceParameters | undefined {
+  const candidate = record(value)
+  if (!candidate || Object.keys(candidate).length > 50) return undefined
+  const entries: Array<[string, DatasourceParameterValue]> = []
+  for (const [key, parameter] of Object.entries(candidate)) {
+    if (!key || key.length > 255) return undefined
+    if (
+      typeof parameter !== 'string' &&
+      typeof parameter !== 'boolean' &&
+      (typeof parameter !== 'number' || !Number.isFinite(parameter))
+    )
+      return undefined
+    entries.push([key, parameter])
+  }
+  return Object.fromEntries(entries)
+}
+
+export function localizedDatasourceText(
+  text: Record<string, string | undefined> | undefined,
+  language: string,
+  fallback: string,
+) {
+  return (
+    text?.[language] ??
+    text?.[language.replaceAll('-', '_')] ??
+    text?.en_US ??
+    text?.zh_Hans ??
+    fallback
+  )
+}

web/features/new-rag/routes.ts

@@ -1,3 +1,6 @@
+import type { DatasourceParameters } from './datasource-parameter-model'
+import { datasourceParameterRecord } from './datasource-parameter-model'
+
 export type NewKnowledgeStartMode = 'empty' | 'source' | 'upload'
 export type NewKnowledgeSourceType = 'onlineDocuments' | 'onlineDrive' | 'websiteCrawl'
 type NewKnowledgeSyncPolicy = 'daily' | 'manual' | 'provider'
@@ -10,6 +13,7 @@ export type NewKnowledgeSourceProvider =
   | NewKnowledgeWebsiteProvider
 
 type NewKnowledgeSourceDraftBase = {
+  parameters?: DatasourceParameters
   sourceName: string
   syncPolicy: NewKnowledgeSyncPolicy
   providerKey?: string
@@ -51,6 +55,7 @@ export function createNewKnowledgeSourceDraft(
   if (sourceType === 'onlineDocuments')
     return {
       provider: initialProvider?.trim() || 'Notion',
+      parameters: {},
       sourceName: '',
       sourceType,
       syncPolicy: 'provider',
@@ -58,6 +63,7 @@ export function createNewKnowledgeSourceDraft(
   if (sourceType === 'onlineDrive')
     return {
       provider: initialProvider?.trim() || 'Google Drive',
+      parameters: {},
       sourceName: '',
       sourceType,
       syncPolicy: 'provider',
@@ -65,6 +71,7 @@ export function createNewKnowledgeSourceDraft(
   return {
     includeSubpages: true,
     maxPages: 100,
+    parameters: {},
     provider: initialProvider?.trim() || 'Firecrawl',
     rootUrl: '',
     sourceName: '',
@@ -127,6 +134,8 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
       : candidate.syncPolicy === undefined
         ? 'provider'
         : undefined
+    const parameters =
+      candidate.parameters === undefined ? {} : datasourceParameterRecord(candidate.parameters)
     if (
       typeof candidate.sourceName !== 'string' ||
       candidate.sourceName.length > NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH ||
@@ -137,12 +146,14 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
         (typeof candidate.providerKey !== 'string' ||
           !candidate.providerKey ||
           candidate.providerKey.length > NEW_KNOWLEDGE_PROVIDER_KEY_MAX_LENGTH)) ||
-      !syncPolicy
+      !syncPolicy ||
+      !parameters
     )
       return undefined
     if (candidate.sourceType === 'onlineDocuments') {
       return {
         provider: candidate.provider,
+        parameters,
         ...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}),
         sourceName: candidate.sourceName,
         sourceType: candidate.sourceType,
@@ -152,6 +163,7 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
     if (candidate.sourceType === 'onlineDrive') {
       return {
         provider: candidate.provider,
+        parameters,
         ...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}),
         sourceName: candidate.sourceName,
         sourceType: candidate.sourceType,
@@ -172,6 +184,7 @@ export function parseNewKnowledgeSourceDraft(value: string): NewKnowledgeSourceD
     return {
       includeSubpages: candidate.includeSubpages,
       maxPages: candidate.maxPages,
+      parameters,
       provider: candidate.provider,
       ...(candidate.providerKey ? { providerKey: candidate.providerKey } : {}),
       rootUrl: candidate.rootUrl,

web/features/new-rag/website-crawl-preview.tsx

@@ -1,7 +1,9 @@
 'use client'
 
+import type { DatasourceParameters } from './datasource-parameter-model'
 import type { NewKnowledgeWebsiteSourceDraft } from './routes'
 import type { CrawlPreviewPage as PreviewPage, Source, SourceWorkflowRun } from './source-models'
+import type { InstalledSourceProviderOption } from './source-provider-options'
 import {
   AlertDialog,
   AlertDialogActions,
@@ -14,27 +16,26 @@ import {
 import { Button } from '@langgenius/dify-ui/button'
 import { Checkbox } from '@langgenius/dify-ui/checkbox'
 import { cn } from '@langgenius/dify-ui/cn'
-import { Collapsible, CollapsiblePanel, CollapsibleTrigger } from '@langgenius/dify-ui/collapsible'
-import { Field, FieldControl, FieldError, FieldLabel } from '@langgenius/dify-ui/field'
+import { Field, FieldControl, FieldLabel } from '@langgenius/dify-ui/field'
 import { Fieldset } from '@langgenius/dify-ui/fieldset'
 import { Form } from '@langgenius/dify-ui/form'
-import {
-  NumberField,
-  NumberFieldControls,
-  NumberFieldDecrement,
-  NumberFieldGroup,
-  NumberFieldIncrement,
-  NumberFieldInput,
-} from '@langgenius/dify-ui/number-field'
-import { memo, useCallback, useEffect, useId, useMemo, useRef, useState } from 'react'
+import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useRouter } from '@/next/navigation'
 import { consoleClient } from '@/service/client'
 import { CrawlSelectionForm } from './crawl-selection-form'
+import { DatasourceParameterForm } from './datasource-parameter-form'
+import {
+  datasourceIncludeSubpages,
+  datasourceParameterDefaults,
+  invalidDatasourceParameters,
+  missingRequiredDatasourceParameters,
+  websiteDatasourceParameterSchemas,
+  withDatasourceParameterDefaults,
+} from './datasource-parameter-model'
 import { createRequestId } from './request-id'
 import {
   NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH,
-  NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH,
   newKnowledgeDetailPath,
   normalizeWebsiteSourceUrl,
 } from './routes'
@@ -46,10 +47,10 @@ type ConnectionReference = {
 }
 
 type CrawlConfiguration = {
-  includeSubpages: boolean
-  limit: number
   name: string
-  url: string
+  parameters: DatasourceParameters
+  rootUrl?: string
+  uri: string
 }
 
 type PreviewDraft = {
@@ -79,8 +80,6 @@ function previewPagesEqual(left: PreviewPage, right: PreviewPage) {
 const PAGE_SIZE = 200
 const MAX_CURSOR_PAGES = 100
 const POLL_INTERVAL_MS = 1500
-const DEFAULT_PAGE_LIMIT = 100
-const MAX_PAGE_LIMIT = 200
 const MAX_PREVIEW_SELECTION = 200
 const SUCCESS_STATES = new Set([
   'complete',
@@ -191,6 +190,15 @@ function configurationKey(configuration: CrawlConfiguration) {
   return JSON.stringify(configuration)
 }
 
+function datasourceSourceUri(parameters: DatasourceParameters, fallback: string) {
+  const url = parameters.url
+  if (typeof url === 'string') {
+    const normalized = normalizeWebsiteSourceUrl(url)
+    if (normalized) return normalized.toString()
+  }
+  return `datasource://${encodeURIComponent(fallback)}`
+}
+
 function workflowAttemptKey(run: SourceWorkflowRun) {
   return `${run.id}:${run.executionAttempts}`
 }
@@ -418,6 +426,7 @@ export function WebsiteCrawlPreview({
   knowledgeSpaceId,
   onDraftFinished,
   onInteractionLockChange,
+  providerOption,
   providerName = 'Firecrawl',
 }: {
   connection: ConnectionReference
@@ -425,25 +434,44 @@ export function WebsiteCrawlPreview({
   knowledgeSpaceId: string
   onDraftFinished?: () => void
   onInteractionLockChange?: (locked: boolean) => void
+  providerOption?: InstalledSourceProviderOption
   providerName?: string
 }) {
   const { t } = useTranslation('dataset')
   const router = useRouter()
-  const rootUrlErrorId = useId()
-  const pageLimitErrorId = useId()
-  const [rootUrl, setRootUrl] = useState(initialDraft?.rootUrl ?? '')
-  const [sourceName, setSourceName] = useState(initialDraft?.sourceName ?? '')
-  const [urlTouched, setUrlTouched] = useState(false)
-  const [optionsExpanded, setOptionsExpanded] = useState(false)
-  const [includeSubpages, setIncludeSubpages] = useState(initialDraft?.includeSubpages ?? true)
-  const [pageLimit, setPageLimit] = useState<number | ''>(() => {
-    const initialLimit = initialDraft?.maxPages
-    return initialLimit && initialLimit > 0 && initialLimit <= MAX_PAGE_LIMIT
-      ? initialLimit
-      : DEFAULT_PAGE_LIMIT
+  const parameterSchemas = useMemo(
+    () =>
+      providerOption
+        ? websiteDatasourceParameterSchemas(providerOption.datasource)
+        : websiteDatasourceParameterSchemas(),
+    [providerOption],
+  )
+  const defaultParameters = useMemo(
+    () => datasourceParameterDefaults(parameterSchemas),
+    [parameterSchemas],
+  )
+  const [parameters, setParameters] = useState<DatasourceParameters>(() => {
+    const initialParameters = withDatasourceParameterDefaults(
+      parameterSchemas,
+      initialDraft?.parameters,
+    )
+    if (!providerOption || providerOption.datasource.parameters.length === 0) {
+      const crawlSubpagesParameter = parameterSchemas.find((parameter) =>
+        ['crawl_sub_pages', 'crawl_subpages'].includes(parameter.name),
+      )
+      if (crawlSubpagesParameter)
+        initialParameters[crawlSubpagesParameter.name] = initialDraft?.includeSubpages ?? true
+      initialParameters.limit = initialDraft?.maxPages ?? 100
+    }
+    if (
+      initialDraft?.rootUrl &&
+      parameterSchemas.some((parameter) => parameter.name === 'url') &&
+      initialParameters.url === undefined
+    )
+      initialParameters.url = initialDraft.rootUrl
+    return initialParameters
   })
-  const crawlOptionsAreDefault =
-    includeSubpages && (pageLimit === '' || pageLimit === DEFAULT_PAGE_LIMIT)
+  const [sourceName, setSourceName] = useState(initialDraft?.sourceName ?? '')
   const [run, setRun] = useState<SourceWorkflowRun>()
   const [pages, setPages] = useState<PreviewPage[]>([])
   const [selectedPageIds, setSelectedPageIds] = useState<Set<string>>(() => new Set())
@@ -464,7 +492,6 @@ export function WebsiteCrawlPreview({
   const actionPendingRef = useRef(false)
   const retryFingerprintRef = useRef<string | undefined>(undefined)
   const cancelFingerprintRef = useRef<string | undefined>(undefined)
-  const rootUrlInputRef = useRef<HTMLInputElement>(null)
   const sourceNameInputRef = useRef<HTMLInputElement>(null)
   const pageMapRef = useRef(new Map<string, PreviewPage>())
   const pageCursorRef = useRef<string | undefined>(undefined)
@@ -568,27 +595,29 @@ export function WebsiteCrawlPreview({
     setSelectionUncertain(uncertain)
   }, [])
 
-  const normalizedURL = useMemo(() => normalizeWebsiteSourceUrl(rootUrl), [rootUrl])
-  const pageLimitValid =
-    typeof pageLimit === 'number' &&
-    Number.isInteger(pageLimit) &&
-    pageLimit >= 1 &&
-    pageLimit <= MAX_PAGE_LIMIT
+  const normalizedURL = useMemo(
+    () =>
+      typeof parameters.url === 'string' ? normalizeWebsiteSourceUrl(parameters.url) : undefined,
+    [parameters.url],
+  )
+  const hasUrlParameter = parameterSchemas.some((parameter) => parameter.name === 'url')
+  const parametersValid =
+    !missingRequiredDatasourceParameters(parameterSchemas, parameters).length &&
+    !invalidDatasourceParameters(parameterSchemas, parameters).length &&
+    !(hasUrlParameter && parameters.url && !normalizedURL)
   const configuration = useMemo<CrawlConfiguration | undefined>(
     () =>
-      normalizedURL &&
+      parametersValid &&
       sourceName.trim() &&
-      sourceName.trim().length <= NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH &&
-      typeof pageLimit === 'number' &&
-      pageLimitValid
+      sourceName.trim().length <= NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH
         ? {
-            includeSubpages,
-            limit: pageLimit,
             name: sourceName.trim(),
-            url: normalizedURL.toString(),
+            parameters,
+            ...(normalizedURL ? { rootUrl: normalizedURL.toString() } : {}),
+            uri: datasourceSourceUri(parameters, providerOption?.key ?? providerName),
           }
         : undefined,
-    [includeSubpages, pageLimit, pageLimitValid, normalizedURL, sourceName],
+    [normalizedURL, parameters, parametersValid, providerName, providerOption?.key, sourceName],
   )
   const currentConfigurationKey = configuration ? configurationKey(configuration) : undefined
   const previewConfigurationMatches = Boolean(
@@ -612,9 +641,9 @@ export function WebsiteCrawlPreview({
   const runId = run?.id
   const locked = starting || stopping || active || uncertainOperation || selectionInteractionLocked
   const dirty = Boolean(
-    rootUrl || sourceName || run || !includeSubpages || pageLimit !== DEFAULT_PAGE_LIMIT,
+    sourceName || run || JSON.stringify(parameters) !== JSON.stringify(defaultParameters),
   )
-  const host = normalizedURL?.host ?? ''
+  const host = normalizedURL?.host ?? providerName
   const completedCount = Math.max(run?.progressCompleted ?? 0, pages.length)
   const crawlingStatusText = t(($) => $['newKnowledge.crawlingPages'], {
     count: completedCount,
@@ -795,9 +824,14 @@ export function WebsiteCrawlPreview({
               metadata: {
                 clientRequestId: draft.clientRequestId,
                 crawlOptions: {
-                  includeSubpages: nextConfiguration.includeSubpages,
-                  limit: nextConfiguration.limit,
+                  includeSubpages: datasourceIncludeSubpages(nextConfiguration.parameters),
+                  limit:
+                    typeof nextConfiguration.parameters.limit === 'number'
+                      ? nextConfiguration.parameters.limit
+                      : 200,
                 },
+                datasourceParameterMode: 'exact',
+                parameters: nextConfiguration.parameters,
                 preview: true,
                 providerId: connection.providerId,
                 providerName,
@@ -805,7 +839,7 @@ export function WebsiteCrawlPreview({
               name: nextConfiguration.name,
               status: 'disabled',
               type: 'web',
-              uri: nextConfiguration.url,
+              uri: nextConfiguration.uri,
             },
             params: { control_space_id: knowledgeSpaceId },
           }),
@@ -1136,9 +1170,7 @@ export function WebsiteCrawlPreview({
 
   const handlePrimaryAction = () => {
     if (!configuration) {
-      setUrlTouched(true)
-      if (!normalizedURL) rootUrlInputRef.current?.focus()
-      else sourceNameInputRef.current?.focus()
+      sourceNameInputRef.current?.focus()
       return
     }
     if (requestError === 'POLL_FAILED' && run) {
@@ -1294,31 +1326,13 @@ export function WebsiteCrawlPreview({
       </p>
       <Form onFormSubmit={handleSubmit}>
         <Fieldset disabled={locked} className="space-y-4">
+          <DatasourceParameterForm
+            disabled={locked}
+            parameters={parameters}
+            schemas={parameterSchemas}
+            onChange={setParameters}
+          />
           <div className="grid grid-cols-1 items-start gap-3 sm:grid-cols-2">
-            <Field name="rootUrl" invalid={urlTouched && !normalizedURL} className="gap-1.5">
-              <FieldLabel>
-                {t(($) => $['newKnowledge.rootUrl'])}
-                <span className="ml-0.5 text-text-destructive">*</span>
-              </FieldLabel>
-              <FieldControl
-                ref={rootUrlInputRef}
-                type="url"
-                inputMode="url"
-                autoComplete="off"
-                required
-                maxLength={NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH}
-                value={rootUrl}
-                placeholder={t(($) => $['newKnowledge.rootUrlPlaceholder'])}
-                aria-describedby={urlTouched && !normalizedURL ? rootUrlErrorId : undefined}
-                onBlur={() => setUrlTouched(true)}
-                onValueChange={setRootUrl}
-              />
-              {urlTouched && !normalizedURL && (
-                <FieldError id={rootUrlErrorId} match>
-                  {t(($) => $['newKnowledge.invalidRootUrl'])}
-                </FieldError>
-              )}
-            </Field>
             <Field name="sourceName" className="gap-1.5">
               <FieldLabel>
                 {t(($) => $['newKnowledge.sourceName'])}
@@ -1336,81 +1350,6 @@ export function WebsiteCrawlPreview({
               />
             </Field>
           </div>
-          <Collapsible
-            open={optionsExpanded}
-            onOpenChange={setOptionsExpanded}
-            className="overflow-hidden rounded-lg border border-components-option-card-option-border bg-background-default"
-          >
-            <CollapsibleTrigger className="h-8.5 min-h-8.5 justify-start rounded-none px-3 focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:outline-hidden focus-visible:ring-inset">
-              <span
-                aria-hidden
-                className="i-ri-arrow-right-s-line size-4 text-text-tertiary transition-transform group-data-panel-open:rotate-90 motion-reduce:transition-none"
-              />
-              <span className="system-xs-medium text-text-primary">
-                {t(($) => $['newKnowledge.crawlOptions'])}
-              </span>
-              {!optionsExpanded && (
-                <span className="ml-auto system-xs-regular text-text-tertiary">
-                  {crawlOptionsAreDefault
-                    ? t(($) => $['newKnowledge.usingDefaults'])
-                    : `${t(($) => $['newKnowledge.includeSubpages'])}: ${t(($) =>
-                        includeSubpages
-                          ? $['newKnowledge.booleanTrue']
-                          : $['newKnowledge.booleanFalse'],
-                      )} · ${t(($) => $['newKnowledge.maxPages'])}: ${
-                        pageLimit === '' ? DEFAULT_PAGE_LIMIT : pageLimit
-                      }`}
-                </span>
-              )}
-            </CollapsibleTrigger>
-            <CollapsiblePanel>
-              <div className="grid grid-cols-1 gap-3 border-t border-divider-subtle p-3 sm:grid-cols-2">
-                <label className="flex h-9 items-center gap-2 system-xs-regular text-text-secondary">
-                  <Checkbox checked={includeSubpages} onCheckedChange={setIncludeSubpages} />
-                  {t(($) => $['newKnowledge.includeSubpages'])}
-                </label>
-                <div className="flex flex-col gap-1">
-                  <div className="flex items-center gap-2">
-                    <span className="system-xs-regular text-text-secondary">
-                      {t(($) => $['newKnowledge.maxPages'])}
-                    </span>
-                    <NumberField
-                      allowOutOfRange
-                      min={1}
-                      max={MAX_PAGE_LIMIT}
-                      step={1}
-                      value={pageLimit === '' ? null : pageLimit}
-                      onValueChange={(value) => setPageLimit(value === null ? '' : value)}
-                    >
-                      <NumberFieldGroup className="ml-auto w-28">
-                        <NumberFieldInput
-                          aria-label={t(($) => $['newKnowledge.maxPages'])}
-                          aria-describedby={!pageLimitValid ? pageLimitErrorId : undefined}
-                          aria-invalid={!pageLimitValid}
-                          onBlur={() => {
-                            if (pageLimit === '') setPageLimit(DEFAULT_PAGE_LIMIT)
-                          }}
-                        />
-                        <NumberFieldControls>
-                          <NumberFieldIncrement />
-                          <NumberFieldDecrement />
-                        </NumberFieldControls>
-                      </NumberFieldGroup>
-                    </NumberField>
-                  </div>
-                  {!pageLimitValid && (
-                    <p
-                      id={pageLimitErrorId}
-                      className="text-right system-xs-regular text-text-destructive"
-                      role="alert"
-                    >
-                      {t(($) => $['newKnowledge.maxPages'])}: 1–{MAX_PAGE_LIMIT}
-                    </p>
-                  )}
-                </div>
-              </div>
-            </CollapsiblePanel>
-          </Collapsible>
         </Fieldset>
 
         {!showSuccess && (
@@ -1506,7 +1445,7 @@ export function WebsiteCrawlPreview({
               pendingCancelRunRef.current = nextRun
             }}
             pages={pages}
-            rootUrl={configuration.url}
+            rootUrl={configuration.rootUrl}
             run={run}
             source={draftRef.current.source}
             workflowUncertain={workflowUncertain}

web/i18n/ar-TN/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "تضمين الصفحات الفرعية",
   "newKnowledge.indexInformation": "الفهرسة",
   "newKnowledge.interruptTask": "إيقاف",
+  "newKnowledge.invalidDatasourceParameter": "أدخل قيمة صالحة لـ {{parameter}}.",
   "newKnowledge.invalidRootUrl": "أدخل عنوان URL صالحًا يبدأ بـ http(s).",
   "newKnowledge.keepEditing": "متابعة التحرير",
   "newKnowledge.lastIndexed": "آخر فهرسة",

web/i18n/de-DE/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Unterseiten einbeziehen",
   "newKnowledge.indexInformation": "Index",
   "newKnowledge.interruptTask": "Unterbrechen",
+  "newKnowledge.invalidDatasourceParameter": "Gib einen gültigen Wert für {{parameter}} ein.",
   "newKnowledge.invalidRootUrl": "Gib eine gültige http(s)-URL ein.",
   "newKnowledge.keepEditing": "Weiter bearbeiten",
   "newKnowledge.lastIndexed": "Zuletzt indexiert",

web/i18n/en-US/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Include sub-pages",
   "newKnowledge.indexInformation": "Index",
   "newKnowledge.interruptTask": "Interrupt",
+  "newKnowledge.invalidDatasourceParameter": "Enter a valid value for {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Enter a valid http(s) URL.",
   "newKnowledge.keepEditing": "Keep editing",
   "newKnowledge.lastIndexed": "Last indexed",

web/i18n/es-ES/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Incluir subpáginas",
   "newKnowledge.indexInformation": "Índice",
   "newKnowledge.interruptTask": "Interrumpir",
+  "newKnowledge.invalidDatasourceParameter": "Introduce un valor válido para {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Introduce una URL http(s) válida.",
   "newKnowledge.keepEditing": "Seguir editando",
   "newKnowledge.lastIndexed": "Última indexación",

web/i18n/fa-IR/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "شامل صفحه‌های فرعی",
   "newKnowledge.indexInformation": "نمایه",
   "newKnowledge.interruptTask": "توقف",
+  "newKnowledge.invalidDatasourceParameter": "یک مقدار معتبر برای {{parameter}} وارد کنید.",
   "newKnowledge.invalidRootUrl": "یک نشانی معتبر http(s) وارد کنید.",
   "newKnowledge.keepEditing": "ادامه ویرایش",
   "newKnowledge.lastIndexed": "آخرین نمایه‌سازی",

web/i18n/fr-FR/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Inclure les sous-pages",
   "newKnowledge.indexInformation": "Index",
   "newKnowledge.interruptTask": "Interrompre",
+  "newKnowledge.invalidDatasourceParameter": "Saisissez une valeur valide pour {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Saisissez une URL http(s) valide.",
   "newKnowledge.keepEditing": "Continuer la modification",
   "newKnowledge.lastIndexed": "Dernière indexation",

web/i18n/hi-IN/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "उप-पेज शामिल करें",
   "newKnowledge.indexInformation": "अनुक्रमणिका",
   "newKnowledge.interruptTask": "रोकें",
+  "newKnowledge.invalidDatasourceParameter": "{{parameter}} के लिए मान्य मान दर्ज करें।",
   "newKnowledge.invalidRootUrl": "मान्य http(s) URL दर्ज करें।",
   "newKnowledge.keepEditing": "संपादन जारी रखें",
   "newKnowledge.lastIndexed": "पिछली बार अनुक्रमित",

web/i18n/id-ID/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Sertakan subhalaman",
   "newKnowledge.indexInformation": "Indeks",
   "newKnowledge.interruptTask": "Hentikan",
+  "newKnowledge.invalidDatasourceParameter": "Masukkan nilai yang valid untuk {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Masukkan URL http(s) yang valid.",
   "newKnowledge.keepEditing": "Lanjutkan mengedit",
   "newKnowledge.lastIndexed": "Terakhir diindeks",

web/i18n/it-IT/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Includi sottopagine",
   "newKnowledge.indexInformation": "Indice",
   "newKnowledge.interruptTask": "Interrompere",
+  "newKnowledge.invalidDatasourceParameter": "Inserisci un valore valido per {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Inserisci un URL http(s) valido.",
   "newKnowledge.keepEditing": "Continua a modificare",
   "newKnowledge.lastIndexed": "Ultima indicizzazione",

web/i18n/ja-JP/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "サブページを含める",
   "newKnowledge.indexInformation": "インデックス",
   "newKnowledge.interruptTask": "中断",
+  "newKnowledge.invalidDatasourceParameter": "{{parameter}} に有効な値を入力してください。",
   "newKnowledge.invalidRootUrl": "有効なhttp(s) URLを入力してください。",
   "newKnowledge.keepEditing": "編集を続ける",
   "newKnowledge.lastIndexed": "最終インデックス日時",

web/i18n/ko-KR/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "하위 페이지 포함",
   "newKnowledge.indexInformation": "인덱스",
   "newKnowledge.interruptTask": "중단",
+  "newKnowledge.invalidDatasourceParameter": "{{parameter}}에 유효한 값을 입력하세요.",
   "newKnowledge.invalidRootUrl": "유효한 http(s) URL을 입력하세요.",
   "newKnowledge.keepEditing": "계속 편집",
   "newKnowledge.lastIndexed": "마지막 인덱싱",

web/i18n/lo-LA/dataset.json

@@ -286,6 +286,7 @@
   "newKnowledge.includeSubpages": "ລວມເອົາໜ້າຍ່ອຍ",
   "newKnowledge.indexInformation": "ຂໍ້ມູນດັດຊະນີ",
   "newKnowledge.interruptTask": "ຂັດຂວາງ",
+  "newKnowledge.invalidDatasourceParameter": "ປ້ອນຄ່າທີ່ຖືກຕ້ອງສຳລັບ {{parameter}}.",
   "newKnowledge.invalidRootUrl": "ລະບຸ URL http(s) ທີ່ຖືກຕ້ອງ.",
   "newKnowledge.keepEditing": "ສືບຕໍ່ແກ້ໄຂ",
   "newKnowledge.lastIndexed": "ດັດສະນີຫຼ້າສຸດ",

web/i18n/nl-NL/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Subpagina’s opnemen",
   "newKnowledge.indexInformation": "Index",
   "newKnowledge.interruptTask": "Onderbreken",
+  "newKnowledge.invalidDatasourceParameter": "Voer een geldige waarde in voor {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Voer een geldige http(s)-URL in.",
   "newKnowledge.keepEditing": "Doorgaan met bewerken",
   "newKnowledge.lastIndexed": "Laatst geïndexeerd",

web/i18n/pl-PL/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Uwzględnij podstrony",
   "newKnowledge.indexInformation": "Indeks",
   "newKnowledge.interruptTask": "Przerwij",
+  "newKnowledge.invalidDatasourceParameter": "Wprowadź prawidłową wartość dla {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Wprowadź prawidłowy adres URL http(s).",
   "newKnowledge.keepEditing": "Kontynuuj edycję",
   "newKnowledge.lastIndexed": "Ostatnio zindeksowano",

web/i18n/pt-BR/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Incluir subpáginas",
   "newKnowledge.indexInformation": "Índice",
   "newKnowledge.interruptTask": "Interromper",
+  "newKnowledge.invalidDatasourceParameter": "Insira um valor válido para {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Insira uma URL http(s) válida.",
   "newKnowledge.keepEditing": "Continuar editando",
   "newKnowledge.lastIndexed": "Última indexação",

web/i18n/ro-RO/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Include subpaginile",
   "newKnowledge.indexInformation": "Index",
   "newKnowledge.interruptTask": "Întrerupe",
+  "newKnowledge.invalidDatasourceParameter": "Introdu o valoare validă pentru {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Introdu o adresă URL http(s) validă.",
   "newKnowledge.keepEditing": "Continuă editarea",
   "newKnowledge.lastIndexed": "Ultima indexare",

web/i18n/ru-RU/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Включать вложенные страницы",
   "newKnowledge.indexInformation": "Индекс",
   "newKnowledge.interruptTask": "Прерывание",
+  "newKnowledge.invalidDatasourceParameter": "Введите допустимое значение для {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Введите корректный URL с http(s).",
   "newKnowledge.keepEditing": "Продолжить редактирование",
   "newKnowledge.lastIndexed": "Последняя индексация",

web/i18n/sl-SI/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Vključi podstrani",
   "newKnowledge.indexInformation": "Indeks",
   "newKnowledge.interruptTask": "Prekini",
+  "newKnowledge.invalidDatasourceParameter": "Vnesite veljavno vrednost za {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Vnesite veljaven URL http(s).",
   "newKnowledge.keepEditing": "Nadaljuj urejanje",
   "newKnowledge.lastIndexed": "Nazadnje indeksirano",

web/i18n/th-TH/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "รวมหน้าย่อย",
   "newKnowledge.indexInformation": "ดัชนี",
   "newKnowledge.interruptTask": "ขัดจังหวะ",
+  "newKnowledge.invalidDatasourceParameter": "ป้อนค่าที่ถูกต้องสำหรับ {{parameter}}",
   "newKnowledge.invalidRootUrl": "ป้อน URL http(s) ที่ถูกต้อง",
   "newKnowledge.keepEditing": "แก้ไขต่อ",
   "newKnowledge.lastIndexed": "จัดทำดัชนีล่าสุด",

web/i18n/tr-TR/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Alt sayfaları dahil et",
   "newKnowledge.indexInformation": "Dizin",
   "newKnowledge.interruptTask": "Kesinti",
+  "newKnowledge.invalidDatasourceParameter": "{{parameter}} için geçerli bir değer girin.",
   "newKnowledge.invalidRootUrl": "Geçerli bir http(s) URL’si girin.",
   "newKnowledge.keepEditing": "Düzenlemeye devam et",
   "newKnowledge.lastIndexed": "Son dizinleme",

web/i18n/uk-UA/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Включати підсторінки",
   "newKnowledge.indexInformation": "Індекс",
   "newKnowledge.interruptTask": "Переривати",
+  "newKnowledge.invalidDatasourceParameter": "Введіть припустиме значення для {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Введіть дійсну URL-адресу http(s).",
   "newKnowledge.keepEditing": "Продовжити редагування",
   "newKnowledge.lastIndexed": "Останнє індексування",

web/i18n/vi-VN/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "Bao gồm các trang con",
   "newKnowledge.indexInformation": "Chỉ mục",
   "newKnowledge.interruptTask": "Ngắt",
+  "newKnowledge.invalidDatasourceParameter": "Nhập giá trị hợp lệ cho {{parameter}}.",
   "newKnowledge.invalidRootUrl": "Nhập URL http(s) hợp lệ.",
   "newKnowledge.keepEditing": "Tiếp tục chỉnh sửa",
   "newKnowledge.lastIndexed": "Lập chỉ mục gần nhất",

web/i18n/zh-Hans/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "包含子页面",
   "newKnowledge.indexInformation": "索引",
   "newKnowledge.interruptTask": "中断",
+  "newKnowledge.invalidDatasourceParameter": "请输入有效的 {{parameter}}。",
   "newKnowledge.invalidRootUrl": "请输入有效的 http(s) URL。",
   "newKnowledge.keepEditing": "继续编辑",
   "newKnowledge.lastIndexed": "最近索引时间",

web/i18n/zh-Hant/dataset.json

@@ -294,6 +294,7 @@
   "newKnowledge.includeSubpages": "包含子頁面",
   "newKnowledge.indexInformation": "索引",
   "newKnowledge.interruptTask": "中斷",
+  "newKnowledge.invalidDatasourceParameter": "請輸入有效的 {{parameter}}。",
   "newKnowledge.invalidRootUrl": "請輸入有效的 http(s) URL。",
   "newKnowledge.keepEditing": "繼續編輯",
   "newKnowledge.lastIndexed": "最近建立索引時間",