'use client' import type { GetKnowledgeSpacesByIdSourceConnectionsResponse, GetSourceProvidersResponse, } from '@dify/contracts/knowledge-fs/types.gen' import type { NewKnowledgeSourceDraft, NewKnowledgeSourceType, NewKnowledgeWebsiteProvider, } from './routes' import { Button } from '@langgenius/dify-ui/button' import { cn } from '@langgenius/dify-ui/cn' import { useInfiniteQuery, useQuery, useQueryClient } from '@tanstack/react-query' import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import Loading from '@/app/components/base/loading' import { useRouter } from '@/next/navigation' import { consoleClient, consoleQuery } from '@/service/client' import { PendingWebsiteSetup, UnavailableConnectedSourceSetup } from './add-source-placeholder' import { AddSourceExitDialog } from './components/add-source-exit-dialog' import { createNewKnowledgeSourceDraft, newKnowledgeDetailPath, newKnowledgeSourceDraftStorageKey, parseNewKnowledgeSourceDraft, } from './routes' import { WebsiteCrawlPreview } from './website-crawl-preview' type Provider = GetSourceProvidersResponse['items'][number] type ProviderField = Provider['configuration'][number] type Connection = GetKnowledgeSpacesByIdSourceConnectionsResponse['items'][number] type ConnectionAuthKind = 'api-key' | 'endpoint' type SourceType = NewKnowledgeSourceType const CONNECTION_PAGE_SIZE = 200 const FIRECRAWL_PROVIDER_ID = 'plugin-daemon-website' const FIRECRAWL_CONNECTION_NAME = 'Firecrawl' const FIRECRAWL_CONFIGURATION = { datasource: 'crawl', pluginId: 'langgenius/firecrawl_datasource', provider: 'firecrawl', } as const const WEBSITE_PROVIDER_OPTIONS: Array<{ icon: string value: NewKnowledgeWebsiteProvider }> = [ { icon: 'i-ri-fire-fill text-orange-500', value: 'Firecrawl' }, { icon: 'i-custom-public-llm-jina', value: 'Jina Reader' }, { icon: 'i-ri-water-flash-line', value: 'WaterCrawl' }, ] const FIRECRAWL_FIXED_FIELD_NAMES = new Set(Object.keys(FIRECRAWL_CONFIGURATION)) const CONNECTION_STATUS_PRIORITY: Record = { active: 0, provisioning: 1, error: 2, expired: 3, revoked: 4, } function humanizeFieldName(name: string) { return name .replace(/([a-z0-9])([A-Z])/g, '$1 $2') .replace(/[-_]+/g, ' ') .replace(/^./, (character) => character.toUpperCase()) } function fieldValue(value: string, type: ProviderField['type']) { if (type === 'boolean') return value === 'true' if (type === 'integer') return Number.parseInt(value, 10) return value.trim() } function findFirecrawl(providers: Provider[]) { return providers.find((provider) => provider.id === FIRECRAWL_PROVIDER_ID) } function findProviderConnection(connections: Connection[], providerId?: string) { if (!providerId) return undefined return [ ...connections.filter( (connection) => connection.providerId === providerId && connection.status !== 'revoked', ), ].sort( (left, right) => CONNECTION_STATUS_PRIORITY[left.status] - CONNECTION_STATUS_PRIORITY[right.status] || right.updatedAt.localeCompare(left.updatedAt), )[0] } function findConnectionById(connections: Connection[], connectionId: string) { return [...connections.filter((connection) => connection.id === connectionId)].sort( (left, right) => right.version - left.version || right.updatedAt.localeCompare(left.updatedAt) || CONNECTION_STATUS_PRIORITY[left.status] - CONNECTION_STATUS_PRIORITY[right.status], )[0] } function normalizeSourceType(value: string | null): SourceType { if (value === 'onlineDocuments' || value === 'onlineDrive') return value return 'websiteCrawl' } function getSupportedAuthKinds(provider: Provider) { const fields = provider.configuration.filter( (field) => !FIRECRAWL_FIXED_FIELD_NAMES.has(field.name), ) const supported: ConnectionAuthKind[] = [] if (provider.authKinds.includes('api-key') && fields.some((field) => field.secret)) supported.push('api-key') if ( provider.authKinds.includes('endpoint') && fields.some((field) => !field.secret && field.format === 'uri') ) supported.push('endpoint') return supported } function SourceTypeSelector({ value, onChange, }: { value: SourceType onChange: (value: SourceType) => void }) { const { t } = useTranslation('dataset') const options = [ { icon: 'i-ri-global-line', key: 'websiteCrawl' as const }, { icon: 'i-ri-file-text-line', key: 'onlineDocuments' as const }, { icon: 'i-ri-hard-drive-3-line', key: 'onlineDrive' as const }, ] return (
{t(($) => $['newKnowledge.sourceTypeLabel'])}
{options.map((option) => ( ))}
) } function ProviderSelector({ provider, onChange, }: { provider: NewKnowledgeWebsiteProvider onChange: (provider: NewKnowledgeWebsiteProvider) => void }) { const { t } = useTranslation('datasetCreation') return (
{t(($) => $['stepOne.website.chooseProvider'])}
{WEBSITE_PROVIDER_OPTIONS.map((option) => ( ))}
) } function ProviderFieldControl({ field, setValues, values, }: { field: ProviderField setValues: React.Dispatch>> values: Record }) { const { t } = useTranslation('dataset') const generatedId = useId() const inputId = `${generatedId}-input` const descriptionId = field.description ? `${generatedId}-description` : undefined const label = humanizeFieldName(field.name) const sharedProps = { 'aria-describedby': descriptionId, id: inputId, required: field.required, value: values[field.name] ?? '', onChange: (event: React.ChangeEvent) => setValues((current) => ({ ...current, [field.name]: event.target.value })), } return (
{field.type === 'boolean' ? ( ) : ( )} {field.description && (

{field.description}

)}
) } function ConnectionForm({ knowledgeSpaceId, onConnected, onDraftChange, onReconcile, provider, }: { knowledgeSpaceId: string onConnected: (connection: Connection) => void onDraftChange: (dirty: boolean) => void onReconcile: () => Promise provider: Provider }) { const { t } = useTranslation('dataset') const supportedAuthKinds = getSupportedAuthKinds(provider) const [authKind, setAuthKind] = useState(supportedAuthKinds[0] ?? 'api-key') const [configuration, setConfiguration] = useState>({}) const [credentials, setCredentials] = useState>({}) const [error, setError] = useState(false) const [pending, setPending] = useState(false) const configurableFields = provider.configuration.filter( (field) => !FIRECRAWL_FIXED_FIELD_NAMES.has(field.name), ) const visibleFields = configurableFields.filter( (field) => authKind === 'api-key' || (!field.secret && field.format === 'uri'), ) const hasDraftChanges = authKind !== (supportedAuthKinds[0] ?? 'api-key') || Object.values(configuration).some((value) => Boolean(value.trim())) || Object.values(credentials).some((value) => Boolean(value.trim())) useEffect(() => { onDraftChange(hasDraftChanges) return () => onDraftChange(false) }, [hasDraftChanges, onDraftChange]) const changeAuthKind = (nextAuthKind: ConnectionAuthKind) => { if (nextAuthKind !== authKind) setCredentials({}) setAuthKind(nextAuthKind) } const handleSubmit = async (event: React.FormEvent) => { event.preventDefault() if (pending) return const missingRequiredField = visibleFields.some((field) => { const values = field.secret ? credentials : configuration return field.required && !values[field.name]?.trim() }) if (missingRequiredField) return setError(false) setPending(true) try { const fixedConfiguration = Object.fromEntries( provider.configuration .filter((field) => FIRECRAWL_FIXED_FIELD_NAMES.has(field.name)) .map((field) => [ field.name, FIRECRAWL_CONFIGURATION[field.name as keyof typeof FIRECRAWL_CONFIGURATION], ]), ) const safeConfiguration = { ...fixedConfiguration, ...Object.fromEntries( visibleFields .filter((field) => !field.secret && configuration[field.name]?.trim()) .map((field) => [field.name, fieldValue(configuration[field.name] ?? '', field.type)]), ), } const secretCredentials = Object.fromEntries( visibleFields .filter((field) => field.secret && credentials[field.name]?.trim()) .map((field) => [field.name, fieldValue(credentials[field.name] ?? '', field.type)]), ) const createdConnection = await consoleClient.knowledgeFs.postKnowledgeSpacesByIdSourceConnections({ body: { authKind, configuration: safeConfiguration, credentials: secretCredentials, name: FIRECRAWL_CONNECTION_NAME, providerId: provider.id, }, params: { id: knowledgeSpaceId }, }) setCredentials({}) onDraftChange(false) onConnected(createdConnection) } catch { setCredentials({}) let reconciledConnection: Connection | undefined try { reconciledConnection = await onReconcile() } catch { reconciledConnection = undefined } if (!reconciledConnection) setError(true) } finally { setPending(false) } } return (
{supportedAuthKinds.length > 1 && (
{t(($) => $['newKnowledge.authenticationMethod'])}
{supportedAuthKinds.map((kind) => ( ))}
)}
{visibleFields.map((field) => ( ))}
{error && (

{t(($) => $['newKnowledge.connectionFailed'])}

)}
) } function UnconfiguredProvider({ knowledgeSpaceId, onConnected, onDraftChange, onReconcile, provider, }: { knowledgeSpaceId: string onConnected: (connection: Connection) => void onDraftChange: (dirty: boolean) => void onReconcile: () => Promise provider: Provider }) { const { t } = useTranslation('dataset') const [configuring, setConfiguring] = useState(false) if (configuring) return ( ) return (

{t(($) => $['newKnowledge.providerNotConfigured'], { provider: FIRECRAWL_CONNECTION_NAME, })}

{t(($) => $['newKnowledge.providerNotConfiguredDescription'], { provider: FIRECRAWL_CONNECTION_NAME, })}

) } function ConnectionProblem({ connection, knowledgeSpaceId, onConnected, onReconcile, }: { connection: Connection knowledgeSpaceId: string onConnected: (connection: Connection) => void onReconcile: () => Promise }) { const { t } = useTranslation('dataset') const { t: tCommon } = useTranslation('common') const [pending, setPending] = useState(false) const [error, setError] = useState(false) const refresh = async () => { if (pending) return setPending(true) setError(false) try { const refreshed = await consoleClient.knowledgeFs.postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh( { body: { expectedVersion: connection.version }, params: { connectionId: connection.id, id: knowledgeSpaceId }, }, ) onConnected(refreshed) } catch { let reconciledConnection: Connection | undefined try { reconciledConnection = await onReconcile() } catch { reconciledConnection = undefined } if (!reconciledConnection) setError(true) } finally { setPending(false) } } return (

{t(($) => $['newKnowledge.connectionNeedsAttention'])}

{t(($) => $['newKnowledge.connectionNeedsAttentionDescription'])}

{error && (

{t(($) => $['newKnowledge.connectionRefreshFailed'])}

)}
) } function ProvisioningConnection({ onReconcile, }: { onReconcile: () => Promise }) { const { t } = useTranslation('dataset') const [pending, setPending] = useState(false) const [error, setError] = useState(false) const refresh = async () => { if (pending) return setPending(true) setError(false) try { const reconciledConnection = await onReconcile() if (!reconciledConnection) setError(true) } catch { setError(true) } finally { setPending(false) } } return (

{t(($) => $['newKnowledge.connectionProvisioning'])}

{error && (

{t(($) => $['newKnowledge.connectionRefreshFailed'])}

)}
) } export function AddSourcePage({ initialSourceDraft, initialSourceType, knowledgeSpaceId, sourceDraftKey, }: { initialSourceDraft?: NewKnowledgeSourceDraft initialSourceType?: string knowledgeSpaceId: string sourceDraftKey?: string }) { const { t } = useTranslation('dataset') const router = useRouter() const queryClient = useQueryClient() const initialDraftRef = useRef( initialSourceDraft ?? createNewKnowledgeSourceDraft(normalizeSourceType(initialSourceType ?? null)), ) const [sourceDraft, setSourceDraft] = useState(initialDraftRef.current) const sourceDraftBaselineRef = useRef( JSON.stringify(createNewKnowledgeSourceDraft(initialDraftRef.current.sourceType)), ) const [sourceDraftResolved, setSourceDraftResolved] = useState(!sourceDraftKey) const [connectedSourceBoundaryVisible, setConnectedSourceBoundaryVisible] = useState(false) const [connectionDraftDirty, setConnectionDraftDirty] = useState(false) const [exitOpen, setExitOpen] = useState(false) const [discarding, setDiscarding] = useState(false) const [discardError, setDiscardError] = useState(false) const [historyGuardReleaseVersion, setHistoryGuardReleaseVersion] = useState(0) const historyGuardArmedRef = useRef(false) const historyGuardReleaseRef = useRef(false) const browserBackExitRef = useRef(false) const pendingNavigationRef = useRef(undefined) const exitDestinationRef = useRef(newKnowledgeDetailPath(knowledgeSpaceId)) const sourceDraftsRef = useRef< Partial> >({ [sourceDraft.sourceType]: sourceDraft }) const sourceType = sourceDraft.sourceType const websiteSourceSelected = sourceDraft.sourceType === 'websiteCrawl' && sourceDraft.provider === FIRECRAWL_CONNECTION_NAME const detailPath = newKnowledgeDetailPath(knowledgeSpaceId) const updateSourceDraft = (draft: NewKnowledgeSourceDraft) => { sourceDraftsRef.current[draft.sourceType] = draft setSourceDraft(draft) } useEffect(() => { if (!sourceDraftKey) return undefined let active = true globalThis.queueMicrotask(() => { if (!active) return let draft: NewKnowledgeSourceDraft | undefined try { const storageKey = newKnowledgeSourceDraftStorageKey(sourceDraftKey) const storedDraft = globalThis.sessionStorage.getItem(storageKey) if (storedDraft) draft = parseNewKnowledgeSourceDraft(storedDraft) } catch { // Continue without the optional draft when browser storage is unavailable. } if (active) { const nextDraft = draft ?? createNewKnowledgeSourceDraft(normalizeSourceType(initialSourceType ?? null)) sourceDraftsRef.current[nextDraft.sourceType] = nextDraft setSourceDraft(nextDraft) setSourceDraftResolved(true) } }) return () => { active = false } }, [initialSourceType, sourceDraftKey]) const clearStoredSourceDraft = useCallback(() => { if (!sourceDraftKey) return try { globalThis.sessionStorage.removeItem(newKnowledgeSourceDraftStorageKey(sourceDraftKey)) } catch { // The draft remains scoped to this browser session when storage cleanup is unavailable. } }, [sourceDraftKey]) const providersQuery = useQuery( consoleQuery.knowledgeFs.getSourceProviders.queryOptions({ input: {}, context: { silent: true }, enabled: websiteSourceSelected, retry: false, }), ) const connectionsQuery = useInfiniteQuery( consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSourceConnections.infiniteOptions({ context: { silent: true }, enabled: websiteSourceSelected, input: (pageParam) => ({ params: { id: knowledgeSpaceId }, query: { limit: CONNECTION_PAGE_SIZE, ...(typeof pageParam === 'string' ? { cursor: pageParam } : {}), }, }), getNextPageParam: (lastPage) => lastPage.nextCursor, initialPageParam: null as string | null, retry: false, }), ) const provider = findFirecrawl(providersQuery.data?.items ?? []) const remoteConnections = connectionsQuery.data?.pages.flatMap((page) => page.items) ?? [] const remoteConnection = findProviderConnection(remoteConnections, provider?.id) const [connectionOverride, setConnectionOverride] = useState() const matchingRemoteConnection = connectionOverride ? remoteConnections.find((candidate) => candidate.id === connectionOverride.id) : undefined const connection = useMemo(() => { const localConnection = connectionOverride if (!localConnection || localConnection.providerId !== provider?.id) return remoteConnection if (!matchingRemoteConnection) return localConnection if (matchingRemoteConnection.id === localConnection.id) { if (matchingRemoteConnection.version > localConnection.version) return matchingRemoteConnection if (matchingRemoteConnection.version < localConnection.version) return localConnection if (matchingRemoteConnection.updatedAt > localConnection.updatedAt) return matchingRemoteConnection if (matchingRemoteConnection.updatedAt < localConnection.updatedAt) return localConnection if ( CONNECTION_STATUS_PRIORITY[matchingRemoteConnection.status] < CONNECTION_STATUS_PRIORITY[localConnection.status] ) return matchingRemoteConnection } return localConnection }, [connectionOverride, matchingRemoteConnection, provider?.id, remoteConnection]) const supportsDirectConnection = provider ? getSupportedAuthKinds(provider).length > 0 : false const { fetchNextPage: fetchNextConnectionPage, hasNextPage: hasNextConnectionPage, isFetchingNextPage: isFetchingNextConnectionPage, refetch: refetchConnections, } = connectionsQuery useEffect(() => { if ( websiteSourceSelected && hasNextConnectionPage && !isFetchingNextConnectionPage && !connectionsQuery.isFetchNextPageError ) void fetchNextConnectionPage() }, [ connectionsQuery.isFetchNextPageError, fetchNextConnectionPage, hasNextConnectionPage, isFetchingNextConnectionPage, websiteSourceSelected, ]) const rememberConnection = useCallback( (updatedConnection: Connection) => { setConnectionOverride(updatedConnection) void queryClient.invalidateQueries({ queryKey: consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSourceConnections.key(), }) }, [queryClient], ) const reconcileConnection = useCallback(async () => { if (connection) setConnectionOverride(connection) const refreshed = await refetchConnections() if (refreshed.error) throw refreshed.error const refreshedConnections = refreshed.data?.pages.flatMap((page) => page.items) ?? [] const refreshedCurrentConnection = connection ? findConnectionById(refreshedConnections, connection.id) : undefined const updatedConnection = connection ? refreshedCurrentConnection ? findConnectionById([connection, refreshedCurrentConnection], connection.id) : undefined : findProviderConnection(refreshedConnections, provider?.id) if (updatedConnection) setConnectionOverride(updatedConnection) return updatedConnection }, [connection, provider?.id, refetchConnections]) const loadingConnections = connectionsQuery.isPending || (!connectionsQuery.isFetchNextPageError && (connectionsQuery.hasNextPage || connectionsQuery.isFetchingNextPage)) const queryError = providersQuery.error || connectionsQuery.error || connectionsQuery.isFetchNextPageError const websiteReady = Boolean( websiteSourceSelected && !queryError && provider?.available && supportsDirectConnection && connection?.status === 'active', ) const websitePreviewReady = websiteReady && !historyGuardArmedRef.current && !historyGuardReleaseRef.current const hasUnsavedChanges = sourceDraftResolved && !websiteReady && (connectionDraftDirty || JSON.stringify(sourceDraft) !== sourceDraftBaselineRef.current) const armHistoryGuard = useCallback(() => { globalThis.history.pushState(globalThis.history.state, '', globalThis.location.href) historyGuardArmedRef.current = true }, []) const replaceAfterHistoryGuard = useCallback( (path: string) => { if (!historyGuardArmedRef.current) { router.replace(path) return } pendingNavigationRef.current = path globalThis.history.back() }, [router], ) const requestNavigation = useCallback( (path: string) => { if (discarding) return if (hasUnsavedChanges) { exitDestinationRef.current = path browserBackExitRef.current = false setDiscardError(false) setExitOpen(true) return } clearStoredSourceDraft() replaceAfterHistoryGuard(path) }, [clearStoredSourceDraft, discarding, hasUnsavedChanges, replaceAfterHistoryGuard], ) useEffect(() => { if ( !hasUnsavedChanges || historyGuardArmedRef.current || browserBackExitRef.current || pendingNavigationRef.current ) return armHistoryGuard() }, [armHistoryGuard, hasUnsavedChanges]) useEffect(() => { const handlePopState = () => { if (!historyGuardArmedRef.current) return historyGuardArmedRef.current = false if (historyGuardReleaseRef.current) { historyGuardReleaseRef.current = false setHistoryGuardReleaseVersion((version) => version + 1) return } const pendingNavigation = pendingNavigationRef.current if (pendingNavigation) { pendingNavigationRef.current = undefined router.replace(pendingNavigation) return } if (!hasUnsavedChanges) { clearStoredSourceDraft() router.replace(detailPath) return } browserBackExitRef.current = true exitDestinationRef.current = detailPath setDiscardError(false) setExitOpen(true) } globalThis.addEventListener('popstate', handlePopState) return () => globalThis.removeEventListener('popstate', handlePopState) }, [clearStoredSourceDraft, detailPath, hasUnsavedChanges, router]) useEffect(() => { if ( !websiteReady || !historyGuardArmedRef.current || historyGuardReleaseRef.current || pendingNavigationRef.current ) return historyGuardReleaseRef.current = true globalThis.history.back() }, [websiteReady]) useEffect(() => { if (!hasUnsavedChanges) return const handleBeforeUnload = (event: BeforeUnloadEvent) => { event.preventDefault() event.returnValue = '' } globalThis.addEventListener('beforeunload', handleBeforeUnload) return () => globalThis.removeEventListener('beforeunload', handleBeforeUnload) }, [hasUnsavedChanges]) useEffect(() => { if (!hasUnsavedChanges) return const handleDocumentClick = (event: MouseEvent) => { if ( event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey ) return const anchor = event .composedPath() .find((target): target is HTMLAnchorElement => target instanceof HTMLAnchorElement) if ( !anchor || anchor.hasAttribute('download') || (anchor.target && anchor.target !== '_self') ) return const destination = new URL(anchor.href, globalThis.location.href) if (destination.origin !== globalThis.location.origin) return const current = new URL(globalThis.location.href) if (destination.pathname === current.pathname && destination.search === current.search) return event.preventDefault() requestNavigation(`${destination.pathname}${destination.search}${destination.hash}`) } document.addEventListener('click', handleDocumentClick, true) return () => document.removeEventListener('click', handleDocumentClick, true) }, [hasUnsavedChanges, requestNavigation]) const requestExit = () => requestNavigation(detailPath) const cancelExit = () => { setExitOpen(false) setDiscardError(false) if (!browserBackExitRef.current) return browserBackExitRef.current = false armHistoryGuard() } const confirmExit = () => { if (discarding) return setDiscarding(true) setDiscardError(false) try { clearStoredSourceDraft() sourceDraftBaselineRef.current = JSON.stringify(sourceDraft) setConnectionDraftDirty(false) setExitOpen(false) browserBackExitRef.current = false const destination = exitDestinationRef.current exitDestinationRef.current = detailPath replaceAfterHistoryGuard(destination) } catch { setDiscardError(true) } finally { setDiscarding(false) } } if ( !sourceDraftResolved || (websiteSourceSelected && (providersQuery.isPending || loadingConnections)) ) return (
) return ( <>

{t(($) => $['newKnowledge.addSource'])}

{t(($) => $['newKnowledge.addSourceDescription'])}

{ sourceDraftsRef.current[sourceDraft.sourceType] = sourceDraft updateSourceDraft( sourceDraftsRef.current[value] ?? createNewKnowledgeSourceDraft(value), ) setConnectedSourceBoundaryVisible(false) }} /> {sourceDraft.sourceType === 'websiteCrawl' ? ( <> { updateSourceDraft({ ...sourceDraft, provider }) setConnectedSourceBoundaryVisible(false) }} /> {!websiteSourceSelected ? (

{sourceDraft.provider}

{t(($) => $['newKnowledge.providerUnavailable'])}

) : queryError ? (

{t(($) => $['newKnowledge.providerLoadFailed'])}

) : !provider ? (
{t(($) => $['newKnowledge.firecrawlUnavailable'])}
) : !provider.available || !supportsDirectConnection ? (

{FIRECRAWL_CONNECTION_NAME}

{provider.unavailableReason ?? t(($) => $['newKnowledge.providerUnavailable'])}

) : connection?.status === 'active' && websitePreviewReady ? ( ) : connection?.status === 'active' ? (
) : connection?.status === 'provisioning' ? ( ) : connection ? ( ) : ( )} {!websiteReady && ( )} ) : ( { updateSourceDraft(draft) setConnectedSourceBoundaryVisible(false) }} /> )} {sourceType === 'websiteCrawl' && !websiteReady && (
{t(($) => $['newKnowledge.addSourceRequiresSelection'])}
)} {sourceType !== 'websiteCrawl' && (
)} {connectedSourceBoundaryVisible && (

{t(($) => $['newKnowledge.sourceSetupBackendDependency'])}

)}
) }