'use client' import type { KnowledgeFsSpaceCreatePayload } from '@dify/contracts/api/console/knowledge-fs/types.gen' import type { NewKnowledgeSourceDraft } from './routes' import type { CrawlPreviewPage } from './source-models' import type { DataSourceAuth, DataSourceCredential, } from '@/app/components/header/account-setting/data-source-page-new/types' import { Button } from '@langgenius/dify-ui/button' import { cn } from '@langgenius/dify-ui/cn' import { Fieldset, FieldsetLegend } from '@langgenius/dify-ui/fieldset' import { useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { buildIntegrationPath } from '@/app/components/integrations/routes' import { consoleClient } from '@/service/client' import { useGetDataSourceListAuth } from '@/service/use-datasource' import { useDataSourceList } from '@/service/use-pipeline' import { CrawlPreviewPageSelection } from './crawl-selection-form' import { CreateConnectedSourceSetup } from './create-connected-source-setup' import { WebsiteDatasourceParameterForm } from './datasource-parameter-form' import { datasourceIncludeSubpages, datasourceParameterDefaults, datasourceParameterSchemas, invalidDatasourceParameters, missingRequiredDatasourceParameters, websiteDatasourceParameterSchemas, withDatasourceParameterDefaults, } from './datasource-parameter-model' import { discoverSourceProviderOptions, sourceProviderOptionForDraft, } from './source-provider-options' import { SourceNameField, SourceProviderCredentialRequiredCard, SourceProviderIcon, SourceProviderNotInstalledCard, SourceProviderRadioGroup, SourceSyncPolicyField, SourceTypeSelector, } from './source-setup-fields' const CRAWL_PREVIEW_SKELETONS = [ { id: 'short', sourceWidth: 'w-22.5', titleWidth: 'w-37.5' }, { id: 'medium', sourceWidth: 'w-26', titleWidth: 'w-42.5' }, { id: 'long', sourceWidth: 'w-29.5', titleWidth: 'w-47.5' }, { id: 'longest', sourceWidth: 'w-33', titleWidth: 'w-52.5' }, ] as const const CRAWL_POLL_INTERVAL_MS = 1500 type LocalCrawlState = 'error' | 'idle' | 'running' | 'stopped' | 'success' type InitialSource = NonNullable function datasourceAuthForProvider( authProviders: DataSourceAuth[], pluginId: string, provider: string, ) { return authProviders.find( (candidate) => candidate.plugin_id === pluginId && candidate.provider === provider, ) } function preferredCredential(auth?: DataSourceAuth): DataSourceCredential | undefined { return ( auth?.credentials_list.find((credential) => credential.is_default) ?? auth?.credentials_list[0] ) } function providerIntegrationPath(packageId?: string) { const base = buildIntegrationPath('data-source') if (!packageId) return base const query = new URLSearchParams({ 'package-ids': JSON.stringify([packageId]) }) return `${base}?${query.toString()}` } function websiteSourceUri(parameters: Record, fallback: string) { const url = parameters.url if (typeof url === 'string') { try { const parsed = new URL(url) if (['http:', 'https:'].includes(parsed.protocol) && !parsed.username && !parsed.password) return parsed.toString() } catch { // Non-URL website datasources use a stable synthetic URI. } } return `datasource://${encodeURIComponent(fallback)}` } export function CreateSourceSetup({ disabled, draft, onDraftChange, onInitialSourceChange, onSourceTypeChange, }: { disabled: boolean draft: NewKnowledgeSourceDraft onDraftChange: (draft: NewKnowledgeSourceDraft) => void onInitialSourceChange: (source?: InitialSource) => void onSourceTypeChange: (sourceType: NewKnowledgeSourceDraft['sourceType']) => void }) { const { t } = useTranslation('dataset') const datasourcePluginsQuery = useDataSourceList(true) const datasourceAuthQuery = useGetDataSourceListAuth() const [crawlState, setCrawlState] = useState('idle') const [stoppingPreview, setStoppingPreview] = useState(false) const [previewPages, setPreviewPages] = useState([]) const [selectedPageIds, setSelectedPageIds] = useState>(() => new Set()) const crawlAttemptRef = useRef(0) const previewJobIdRef = useRef(undefined) const sourceType = draft.sourceType const providerOptions = useMemo( () => discoverSourceProviderOptions(sourceType, datasourcePluginsQuery.data ?? []), [datasourcePluginsQuery.data, sourceType], ) const providerOption = sourceProviderOptionForDraft(providerOptions, draft) const installedProviderOption = providerOption?.installed ? providerOption : undefined const datasourceAuth = installedProviderOption ? datasourceAuthForProvider( datasourceAuthQuery.data?.result ?? [], installedProviderOption.plugin.plugin_id, installedProviderOption.plugin.provider, ) : undefined const credential = preferredCredential(datasourceAuth) const parameterSchemas = useMemo( () => draft.sourceType === 'websiteCrawl' && installedProviderOption ? websiteDatasourceParameterSchemas(installedProviderOption.datasource) : [], [draft.sourceType, installedProviderOption], ) const parameters = useMemo(() => { const current = withDatasourceParameterDefaults(parameterSchemas, draft.parameters) if ( draft.sourceType === 'websiteCrawl' && draft.rootUrl && parameterSchemas.some((parameter) => parameter.name === 'url') && current.url === undefined ) current.url = draft.rootUrl return current }, [draft, parameterSchemas]) const parametersValid = !missingRequiredDatasourceParameters(parameterSchemas, parameters).length && !invalidDatasourceParameters(parameterSchemas, parameters).length const selectionPages = previewPages const previewReady = Boolean( draft.sourceType === 'websiteCrawl' && credential && installedProviderOption && parametersValid && draft.sourceName.trim(), ) const sourceUri = installedProviderOption ? websiteSourceUri(parameters, installedProviderOption.key) : '' const selectionRootUrl = typeof parameters.url === 'string' && sourceUri.startsWith('http') ? sourceUri : undefined const cancelPreviewBestEffort = () => { crawlAttemptRef.current += 1 const jobId = previewJobIdRef.current previewJobIdRef.current = undefined if (jobId) void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId .delete({ params: { job_id: jobId }, }) .catch(() => {}) } const stopPreview = async () => { if (stoppingPreview) return crawlAttemptRef.current += 1 const jobId = previewJobIdRef.current if (!jobId) { setCrawlState('stopped') return } setStoppingPreview(true) try { const response = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId.delete({ params: { job_id: jobId }, }) if (response.status === 'completed' && response.result) { if (previewJobIdRef.current === jobId) previewJobIdRef.current = undefined setPreviewPages( (response.result.pages ?? []).map((page) => ({ description: page.description ?? undefined, pageId: page.source_url, sourceUrl: page.source_url, title: page.title ?? page.source_url, })), ) setCrawlState('success') } else if (response.status === 'canceled') { if (previewJobIdRef.current === jobId) previewJobIdRef.current = undefined setCrawlState('stopped') } else if (response.status === 'failed') { if (previewJobIdRef.current === jobId) previewJobIdRef.current = undefined setCrawlState('error') } else { setCrawlState('running') } } catch { // Keep the job handle and running state so Stop can be retried and unmount // cleanup can still revoke a task whose cancellation response was lost. setCrawlState('running') } finally { setStoppingPreview(false) } } const resetPreview = () => { cancelPreviewBestEffort() setCrawlState('idle') setPreviewPages([]) setSelectedPageIds(new Set()) onInitialSourceChange(undefined) } const updateDraft = (nextDraft: NewKnowledgeSourceDraft) => { onDraftChange(nextDraft) resetPreview() } const updateDraftWithoutReset = (nextDraft: NewKnowledgeSourceDraft) => { onDraftChange(nextDraft) } const selectProvider = (providerKey: string) => { const nextProvider = providerOptions.find((option) => option.key === providerKey) if (!nextProvider) return updateDraft({ ...draft, parameters: nextProvider.installed ? datasourceParameterDefaults( draft.sourceType === 'websiteCrawl' ? websiteDatasourceParameterSchemas(nextProvider.datasource) : datasourceParameterSchemas(nextProvider.datasource), ) : {}, provider: nextProvider.label, providerKey: nextProvider.key, sourceName: '', ...(draft.sourceType === 'websiteCrawl' ? { rootUrl: '' } : {}), ...(draft.sourceType === 'onlineDrive' && nextProvider.label === 'Amazon S3' && draft.syncPolicy === 'provider' ? { syncPolicy: 'daily' as const } : {}), }) } useEffect( () => () => { crawlAttemptRef.current += 1 const jobId = previewJobIdRef.current if (jobId) void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId .delete({ params: { job_id: jobId }, }) .catch(() => {}) }, [], ) const startPreview = async () => { if ( draft.sourceType !== 'websiteCrawl' || !previewReady || !credential || !installedProviderOption ) return const attempt = crawlAttemptRef.current + 1 crawlAttemptRef.current = attempt setPreviewPages([]) setSelectedPageIds(new Set()) setCrawlState('running') try { const job = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.post({ body: { credentialId: credential.id, datasource: installedProviderOption.datasource.identity.name, kind: 'website_crawl', parameters, pluginId: installedProviderOption.plugin.plugin_id, provider: installedProviderOption.plugin.provider, providerDisplayName: installedProviderOption.label, }, }) if (crawlAttemptRef.current !== attempt) { void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId .delete({ params: { job_id: job.job_id }, }) .catch(() => {}) return } previewJobIdRef.current = job.job_id let response = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId.get({ params: { job_id: job.job_id }, }) while ( crawlAttemptRef.current === attempt && !['completed', 'failed', 'canceled'].includes(response.status) ) { await new Promise((resolve) => globalThis.setTimeout(resolve, CRAWL_POLL_INTERVAL_MS)) if (crawlAttemptRef.current !== attempt) return response = await consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId.get({ params: { job_id: job.job_id }, }) } if (crawlAttemptRef.current !== attempt) return previewJobIdRef.current = undefined if (response.status !== 'completed' || !response.result) { setCrawlState(response.status === 'canceled' ? 'stopped' : 'error') return } setPreviewPages( (response.result.pages ?? []).map((page) => ({ description: page.description ?? undefined, pageId: page.source_url, sourceUrl: page.source_url, title: page.title ?? page.source_url, })), ) setCrawlState('success') } catch { if (crawlAttemptRef.current === attempt) { const jobId = previewJobIdRef.current previewJobIdRef.current = undefined if (jobId) void consoleClient.knowledgeFs.sourceProviderPreview.jobs.byJobId .delete({ params: { job_id: jobId }, }) .catch(() => {}) setCrawlState('error') } } } const updateSelectedPageIds = (pageIds: Set) => { setSelectedPageIds(pageIds) } useEffect(() => { if (draft.sourceType !== 'websiteCrawl') { if (!installedProviderOption || !credential) onInitialSourceChange(undefined) return } if ( crawlState !== 'success' || !selectionPages.length || !installedProviderOption || !credential ) { onInitialSourceChange(undefined) return } const selectedPages = selectionPages.filter((page) => selectedPageIds.has(page.pageId)) if (!selectedPages.length) { onInitialSourceChange(undefined) return } onInitialSourceChange({ crawl_options: { include_subpages: datasourceIncludeSubpages(parameters), limit: typeof parameters.limit === 'number' ? parameters.limit : 200, }, credentialId: credential.id, datasource: installedProviderOption.datasource.identity.name, kind: 'website_crawl', name: draft.sourceName.trim(), pluginId: installedProviderOption.plugin.plugin_id, provider: installedProviderOption.plugin.provider, providerDisplayName: installedProviderOption.label, parameters, root_url: sourceUri, selection: selectedPages.map((page) => ({ source_url: page.sourceUrl, ...(page.title ? { title: page.title } : {}), })), ...(draft.syncPolicy === 'custom' && draft.customIntervalSeconds ? { custom_interval_seconds: draft.customIntervalSeconds } : {}), sync_policy: draft.syncPolicy, }) }, [ crawlState, credential, draft, installedProviderOption, onInitialSourceChange, parameters, selectedPageIds, selectionPages, sourceUri, ]) return (
{ onSourceTypeChange(value) }} />
{t(($) => $['newKnowledge.providerLabel'])}
{t(($) => $['newKnowledge.providerLabel'])}
({ icon: , label: option.label, value: option.key, }))} surface="default" onChange={selectProvider} />
{datasourcePluginsQuery.isPending || datasourceAuthQuery.isPending ? (
) : datasourcePluginsQuery.error || datasourceAuthQuery.error ? (

{t(($) => $['newKnowledge.providerLoadFailed'])}

) : providerOption && !providerOption.installed ? ( } provider={providerOption.label} onInstall={() => globalThis.open( providerIntegrationPath(providerOption.packageId), '_blank', 'noopener,noreferrer', ) } /> ) : installedProviderOption && !credential ? ( } provider={installedProviderOption.label} onConnect={() => globalThis.open( providerIntegrationPath(installedProviderOption.packageId), '_blank', 'noopener,noreferrer', ) } /> ) : draft.sourceType === 'websiteCrawl' && installedProviderOption && credential ? (
} disabled={disabled || crawlState === 'running'} parameters={parameters} schemas={parameterSchemas} onChange={(nextParameters) => updateDraft({ ...draft, parameters: nextParameters, rootUrl: typeof nextParameters.url === 'string' ? nextParameters.url : '', }) } /> {crawlState !== 'success' && ( )}
$['newKnowledge.crawlPreview'])} className={cn( crawlState === 'running' ? 'flex h-60 flex-col gap-3.5 overflow-hidden rounded-xl border border-divider-deep bg-background-default-subtle p-4' : crawlState === 'success' ? '' : 'min-h-40 rounded-lg border border-dashed border-divider-regular px-4 py-4', )} > {crawlState === 'idle' && (

{t(($) => $['newKnowledge.pagesAppearTitle'])}

{t(($) => $['newKnowledge.pagesAppearDescription'])}

)} {crawlState === 'running' && (

{t(($) => $['newKnowledge.crawlingPages'], { count: previewPages.length, host: installedProviderOption.label, })}

)} {crawlState === 'running' && CRAWL_PREVIEW_SKELETONS.map(({ id, sourceWidth, titleWidth }) => (
))} {crawlState === 'error' && (

{t(($) => $['newKnowledge.crawlStartFailed'])}

)} {crawlState === 'stopped' && (

{t(($) => $['newKnowledge.crawlStopped'])}

)} {crawlState === 'success' && ( void startPreview()} onSelectionChange={updateSelectedPageIds} pages={selectionPages} rootUrl={selectionRootUrl} selectedPageIds={selectedPageIds} sourceLabel={installedProviderOption.label} /> )}
) : draft.sourceType !== 'websiteCrawl' && installedProviderOption && credential ? ( ) : null}
) }