- {t(($) => $['newKnowledge.connectionProvisioning'], {
- provider: provider.displayName,
- })}
+ {t(($) => $['newKnowledge.connectionProvisioning'])}
{error && (
@@ -651,42 +577,97 @@ function ProvisioningConnection({
)
}
-export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) {
+export function AddSourcePage({
+ initialSourceDraft,
+ initialSourceType,
+ knowledgeSpaceId,
+ sourceDraftKey,
+}: {
+ initialSourceDraft?: NewKnowledgeSourceDraft
+ initialSourceType?: string
+ knowledgeSpaceId: string
+ sourceDraftKey?: string
+}) {
const { t } = useTranslation('dataset')
const router = useRouter()
const queryClient = useQueryClient()
- const [sourceType, setSourceType] = useState('websiteCrawl')
- const [selectedProviderId, setSelectedProviderId] = useState()
- const [ignoredConnectionIds, setIgnoredConnectionIds] = useState>(() => new Set())
- const [crawlDraftDirty, setCrawlDraftDirty] = useState(false)
- const [provisionalSourceCount, setProvisionalSourceCount] = useState(0)
+ const initialDraftRef = useRef(
+ initialSourceDraft ??
+ createNewKnowledgeSourceDraft(normalizeSourceType(initialSourceType ?? null)),
+ )
+ const [sourceDraft, setSourceDraft] = useState(initialDraftRef.current)
+ const sourceDraftBaselineRef = useRef(
+ JSON.stringify(createNewKnowledgeSourceDraft(initialDraftRef.current.sourceType)),
+ )
+ const [sourceDraftResolved, setSourceDraftResolved] = useState(!sourceDraftKey)
+ const [connectedSourceBoundaryVisible, setConnectedSourceBoundaryVisible] = useState(false)
+ const [connectionDraftDirty, setConnectionDraftDirty] = useState(false)
const [exitOpen, setExitOpen] = useState(false)
const [discarding, setDiscarding] = useState(false)
const [discardError, setDiscardError] = useState(false)
- const detailPath = newKnowledgeDetailPath(knowledgeSpaceId)
- const provisionalSourcesRef = useRef(new Map())
- const pendingPreviewOperationsRef = useRef(new Set>())
- const deletionKeysRef = useRef(new Map())
+ const [historyGuardReleaseVersion, setHistoryGuardReleaseVersion] = useState(0)
const historyGuardArmedRef = useRef(false)
+ const historyGuardReleaseRef = useRef(false)
const browserBackExitRef = useRef(false)
const pendingNavigationRef = useRef(undefined)
- const exitDestinationRef = useRef(detailPath)
- const hasUnsavedChanges = Boolean(
- crawlDraftDirty ||
- provisionalSourceCount ||
- sourceType !== 'websiteCrawl' ||
- selectedProviderId,
- )
+ const exitDestinationRef = useRef(newKnowledgeDetailPath(knowledgeSpaceId))
+ const sourceDraftsRef = useRef<
+ Partial>
+ >({ [sourceDraft.sourceType]: sourceDraft })
+ const sourceType = sourceDraft.sourceType
+ const websiteSourceSelected =
+ sourceDraft.sourceType === 'websiteCrawl' && sourceDraft.provider === FIRECRAWL_CONNECTION_NAME
+ const detailPath = newKnowledgeDetailPath(knowledgeSpaceId)
+ const updateSourceDraft = (draft: NewKnowledgeSourceDraft) => {
+ sourceDraftsRef.current[draft.sourceType] = draft
+ setSourceDraft(draft)
+ }
+
+ useEffect(() => {
+ if (!sourceDraftKey) return undefined
+ let active = true
+ globalThis.queueMicrotask(() => {
+ if (!active) return
+ let draft: NewKnowledgeSourceDraft | undefined
+ try {
+ const storageKey = newKnowledgeSourceDraftStorageKey(sourceDraftKey)
+ const storedDraft = globalThis.sessionStorage.getItem(storageKey)
+ if (storedDraft) draft = parseNewKnowledgeSourceDraft(storedDraft)
+ } catch {
+ // Continue without the optional draft when browser storage is unavailable.
+ }
+ if (active) {
+ const nextDraft =
+ draft ?? createNewKnowledgeSourceDraft(normalizeSourceType(initialSourceType ?? null))
+ sourceDraftsRef.current[nextDraft.sourceType] = nextDraft
+ setSourceDraft(nextDraft)
+ setSourceDraftResolved(true)
+ }
+ })
+ return () => {
+ active = false
+ }
+ }, [initialSourceType, sourceDraftKey])
+ const clearStoredSourceDraft = useCallback(() => {
+ if (!sourceDraftKey) return
+ try {
+ globalThis.sessionStorage.removeItem(newKnowledgeSourceDraftStorageKey(sourceDraftKey))
+ } catch {
+ // The draft remains scoped to this browser session when storage cleanup is unavailable.
+ }
+ }, [sourceDraftKey])
const providersQuery = useQuery(
consoleQuery.knowledgeFs.getSourceProviders.queryOptions({
input: {},
context: { silent: true },
+ enabled: websiteSourceSelected,
retry: false,
}),
)
const connectionsQuery = useInfiniteQuery(
consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSourceConnections.infiniteOptions({
context: { silent: true },
+ enabled: websiteSourceSelected,
input: (pageParam) => ({
params: { id: knowledgeSpaceId },
query: {
@@ -699,25 +680,12 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
retry: false,
}),
)
- const providers = useMemo(
- () =>
- (providersQuery.data?.items ?? []).filter((candidate) =>
- candidate.capabilities.includes('website-crawl'),
- ),
- [providersQuery.data?.items],
- )
- const provider =
- providers.find((candidate) => candidate.id === selectedProviderId) ??
- providers.find((candidate) => candidate.id === FIRECRAWL_PROVIDER_ID) ??
- providers[0]
+ const provider = findFirecrawl(providersQuery.data?.items ?? [])
const remoteConnections = connectionsQuery.data?.pages.flatMap((page) => page.items) ?? []
- const usableRemoteConnections = remoteConnections.filter(
- (candidate) => candidate.status !== 'revoked' && !ignoredConnectionIds.has(candidate.id),
- )
- const remoteConnection = findProviderConnection(usableRemoteConnections, provider?.id)
+ const remoteConnection = findProviderConnection(remoteConnections, provider?.id)
const [connectionOverride, setConnectionOverride] = useState()
const matchingRemoteConnection = connectionOverride
- ? usableRemoteConnections.find((candidate) => candidate.id === connectionOverride.id)
+ ? remoteConnections.find((candidate) => candidate.id === connectionOverride.id)
: undefined
const connection = useMemo(() => {
const localConnection = connectionOverride
@@ -746,231 +714,9 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
refetch: refetchConnections,
} = connectionsQuery
- const armHistoryGuard = useCallback(() => {
- globalThis.history.pushState(globalThis.history.state, '', globalThis.location.href)
- historyGuardArmedRef.current = true
- }, [])
-
- const replaceAfterHistoryGuard = useCallback(
- (path: string) => {
- if (!historyGuardArmedRef.current) {
- router.replace(path)
- return
- }
-
- pendingNavigationRef.current = path
- globalThis.history.back()
- },
- [router],
- )
-
- const requestNavigation = useCallback(
- (path: string) => {
- if (discarding) return
- if (hasUnsavedChanges) {
- exitDestinationRef.current = path
- browserBackExitRef.current = false
- setDiscardError(false)
- setExitOpen(true)
- return
- }
- replaceAfterHistoryGuard(path)
- },
- [discarding, hasUnsavedChanges, replaceAfterHistoryGuard],
- )
-
- useEffect(() => {
- if (
- !hasUnsavedChanges ||
- historyGuardArmedRef.current ||
- browserBackExitRef.current ||
- pendingNavigationRef.current
- )
- return
-
- armHistoryGuard()
- }, [armHistoryGuard, hasUnsavedChanges])
-
- useEffect(() => {
- const handlePopState = () => {
- if (!historyGuardArmedRef.current) return
-
- historyGuardArmedRef.current = false
- const pendingNavigation = pendingNavigationRef.current
- if (pendingNavigation) {
- pendingNavigationRef.current = undefined
- router.replace(pendingNavigation)
- return
- }
- if (!hasUnsavedChanges) {
- router.replace(detailPath)
- return
- }
-
- browserBackExitRef.current = true
- exitDestinationRef.current = detailPath
- setDiscardError(false)
- setExitOpen(true)
- }
-
- globalThis.addEventListener('popstate', handlePopState)
- return () => globalThis.removeEventListener('popstate', handlePopState)
- }, [detailPath, hasUnsavedChanges, router])
-
- useEffect(() => {
- if (!hasUnsavedChanges) return
-
- const handleBeforeUnload = (event: BeforeUnloadEvent) => {
- event.preventDefault()
- event.returnValue = ''
- }
-
- globalThis.addEventListener('beforeunload', handleBeforeUnload)
- return () => globalThis.removeEventListener('beforeunload', handleBeforeUnload)
- }, [hasUnsavedChanges])
-
- useEffect(() => {
- if (!hasUnsavedChanges) return
-
- const handleDocumentClick = (event: MouseEvent) => {
- if (
- event.defaultPrevented ||
- event.button !== 0 ||
- event.metaKey ||
- event.ctrlKey ||
- event.shiftKey ||
- event.altKey
- )
- return
-
- const anchor = event
- .composedPath()
- .find((target): target is HTMLAnchorElement => target instanceof HTMLAnchorElement)
- if (
- !anchor ||
- anchor.hasAttribute('download') ||
- (anchor.target && anchor.target !== '_self')
- )
- return
-
- const destination = new URL(anchor.href, globalThis.location.href)
- if (destination.origin !== globalThis.location.origin) return
-
- const current = new URL(globalThis.location.href)
- if (destination.pathname === current.pathname && destination.search === current.search) return
-
- event.preventDefault()
- requestNavigation(`${destination.pathname}${destination.search}${destination.hash}`)
- }
-
- document.addEventListener('click', handleDocumentClick, true)
- return () => document.removeEventListener('click', handleDocumentClick, true)
- }, [hasUnsavedChanges, requestNavigation])
-
- const rememberProvisionalSource = useCallback((source: Source) => {
- provisionalSourcesRef.current.set(source.id, source)
- setProvisionalSourceCount(provisionalSourcesRef.current.size)
- }, [])
-
- const getDeletionKey = useCallback((sourceId: string) => {
- let idempotencyKey = deletionKeysRef.current.get(sourceId)
- if (!idempotencyKey) {
- idempotencyKey = globalThis.crypto.randomUUID()
- deletionKeysRef.current.set(sourceId, idempotencyKey)
- }
- return idempotencyKey
- }, [])
-
- const rememberPendingPreviewOperation = useCallback((operation: Promise) => {
- pendingPreviewOperationsRef.current.add(operation)
- void operation.then(
- () => pendingPreviewOperationsRef.current.delete(operation),
- () => pendingPreviewOperationsRef.current.delete(operation),
- )
- }, [])
-
- const requestExit = () => requestNavigation(detailPath)
-
- const cancelExit = () => {
- setExitOpen(false)
- setDiscardError(false)
- exitDestinationRef.current = detailPath
- if (!browserBackExitRef.current) return
-
- browserBackExitRef.current = false
- armHistoryGuard()
- }
-
- const confirmExit = async () => {
- if (discarding) return
- setDiscarding(true)
- setDiscardError(false)
- try {
- await Promise.allSettled([...pendingPreviewOperationsRef.current])
- for (const source of provisionalSourcesRef.current.values()) {
- try {
- await consoleClient.knowledgeFs.deleteKnowledgeSpacesByIdSourcesBySourceId({
- body: { expectedRevision: source.version ?? 1 },
- headers: { 'idempotency-key': getDeletionKey(source.id) },
- params: { id: knowledgeSpaceId, sourceId: source.id },
- query: { documents: 'cascade' },
- })
- } catch (deletionError) {
- let deletionConfirmed = false
- try {
- await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourcesBySourceId({
- params: { id: knowledgeSpaceId, sourceId: source.id },
- })
- } catch (reconciliationError) {
- deletionConfirmed = responseStatus(reconciliationError) === 404
- }
- if (!deletionConfirmed) throw deletionError
- }
- provisionalSourcesRef.current.delete(source.id)
- deletionKeysRef.current.delete(source.id)
- setProvisionalSourceCount(provisionalSourcesRef.current.size)
- }
- setCrawlDraftDirty(false)
- setExitOpen(false)
- browserBackExitRef.current = false
- const exitDestination = exitDestinationRef.current
- exitDestinationRef.current = detailPath
- replaceAfterHistoryGuard(exitDestination)
- } catch {
- setDiscardError(true)
- } finally {
- setDiscarding(false)
- }
- }
-
- useEffect(() => {
- const handlePageHide = () => {
- for (const source of provisionalSourcesRef.current.values()) {
- void consoleClient.knowledgeFs
- .deleteKnowledgeSpacesByIdSourcesBySourceId(
- {
- body: { expectedRevision: source.version ?? 1 },
- headers: { 'idempotency-key': getDeletionKey(source.id) },
- params: { id: knowledgeSpaceId, sourceId: source.id },
- query: { documents: 'cascade' },
- },
- {
- context: {
- keepalive: true,
- silent: true,
- },
- },
- )
- .catch(() => {})
- }
- }
-
- globalThis.addEventListener('pagehide', handlePageHide)
- return () => globalThis.removeEventListener('pagehide', handlePageHide)
- }, [getDeletionKey, knowledgeSpaceId])
-
useEffect(() => {
if (
+ websiteSourceSelected &&
hasNextConnectionPage &&
!isFetchingNextConnectionPage &&
!connectionsQuery.isFetchNextPageError
@@ -981,22 +727,12 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
fetchNextConnectionPage,
hasNextConnectionPage,
isFetchingNextConnectionPage,
+ websiteSourceSelected,
])
const rememberConnection = useCallback(
(updatedConnection: Connection) => {
- if (updatedConnection.status === 'revoked') {
- setIgnoredConnectionIds((current) => new Set(current).add(updatedConnection.id))
- setConnectionOverride(undefined)
- } else {
- setIgnoredConnectionIds((current) => {
- if (!current.has(updatedConnection.id)) return current
- const next = new Set(current)
- next.delete(updatedConnection.id)
- return next
- })
- setConnectionOverride(updatedConnection)
- }
+ setConnectionOverride(updatedConnection)
void queryClient.invalidateQueries({
queryKey: consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSourceConnections.key(),
})
@@ -1006,167 +742,323 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
const reconcileConnection = useCallback(async () => {
if (connection) setConnectionOverride(connection)
- const retryDelays = connection ? [0] : CONNECTION_RECONCILIATION_RETRY_DELAYS
- let reconciliationError: Error | undefined
-
- for (const retryDelay of retryDelays) {
- if (retryDelay) await new Promise((resolve) => setTimeout(resolve, retryDelay))
- try {
- const refreshed = await refetchConnections()
- if (refreshed.error) throw refreshed.error
- reconciliationError = undefined
- const refreshedConnections = (
- refreshed.data?.pages.flatMap((page) => page.items) ?? []
- ).filter(
- (candidate) => candidate.status !== 'revoked' && !ignoredConnectionIds.has(candidate.id),
- )
- const refreshedCurrentConnection = connection
- ? findConnectionById(refreshedConnections, connection.id)
- : undefined
- const updatedConnection = connection
- ? refreshedCurrentConnection
- ? findConnectionById([connection, refreshedCurrentConnection], connection.id)
- : undefined
- : findProviderConnection(refreshedConnections, provider?.id)
- if (updatedConnection) {
- setConnectionOverride(updatedConnection)
- return updatedConnection
- }
- } catch (error) {
- reconciliationError =
- error instanceof Error ? error : new Error('Connection reconciliation failed')
- }
- }
-
- if (reconciliationError) throw new Error(reconciliationError.message)
- return undefined
- }, [connection, ignoredConnectionIds, provider?.id, refetchConnections])
-
- const reconfigureConnection = useCallback(async () => {
- if (!connection) return
- await consoleClient.knowledgeFs.deleteKnowledgeSpacesByIdSourceConnectionsByConnectionId({
- params: { connectionId: connection.id, id: knowledgeSpaceId },
- query: { expectedVersion: connection.version },
- })
- setIgnoredConnectionIds((current) => new Set(current).add(connection.id))
- setConnectionOverride(undefined)
- await queryClient.invalidateQueries({
- queryKey: consoleQuery.knowledgeFs.getKnowledgeSpacesByIdSourceConnections.key(),
- })
- }, [connection, knowledgeSpaceId, queryClient])
+ const refreshed = await refetchConnections()
+ if (refreshed.error) throw refreshed.error
+ const refreshedConnections = refreshed.data?.pages.flatMap((page) => page.items) ?? []
+ const refreshedCurrentConnection = connection
+ ? findConnectionById(refreshedConnections, connection.id)
+ : undefined
+ const updatedConnection = connection
+ ? refreshedCurrentConnection
+ ? findConnectionById([connection, refreshedCurrentConnection], connection.id)
+ : undefined
+ : findProviderConnection(refreshedConnections, provider?.id)
+ if (updatedConnection) setConnectionOverride(updatedConnection)
+ return updatedConnection
+ }, [connection, provider?.id, refetchConnections])
const loadingConnections =
connectionsQuery.isPending ||
(!connectionsQuery.isFetchNextPageError &&
(connectionsQuery.hasNextPage || connectionsQuery.isFetchingNextPage))
- if (providersQuery.isPending || loadingConnections)
+ const queryError =
+ providersQuery.error || connectionsQuery.error || connectionsQuery.isFetchNextPageError
+ const websiteReady = Boolean(
+ websiteSourceSelected &&
+ !queryError &&
+ provider?.available &&
+ supportsDirectConnection &&
+ connection?.status === 'active',
+ )
+ const websitePreviewReady =
+ websiteReady && !historyGuardArmedRef.current && !historyGuardReleaseRef.current
+ const hasUnsavedChanges =
+ sourceDraftResolved &&
+ !websiteReady &&
+ (connectionDraftDirty || JSON.stringify(sourceDraft) !== sourceDraftBaselineRef.current)
+ const armHistoryGuard = useCallback(() => {
+ globalThis.history.pushState(globalThis.history.state, '', globalThis.location.href)
+ historyGuardArmedRef.current = true
+ }, [])
+ const replaceAfterHistoryGuard = useCallback(
+ (path: string) => {
+ if (!historyGuardArmedRef.current) {
+ router.replace(path)
+ return
+ }
+ pendingNavigationRef.current = path
+ globalThis.history.back()
+ },
+ [router],
+ )
+ const requestNavigation = useCallback(
+ (path: string) => {
+ if (discarding) return
+ if (hasUnsavedChanges) {
+ exitDestinationRef.current = path
+ browserBackExitRef.current = false
+ setDiscardError(false)
+ setExitOpen(true)
+ return
+ }
+ clearStoredSourceDraft()
+ replaceAfterHistoryGuard(path)
+ },
+ [clearStoredSourceDraft, discarding, hasUnsavedChanges, replaceAfterHistoryGuard],
+ )
+
+ useEffect(() => {
+ if (
+ !hasUnsavedChanges ||
+ historyGuardArmedRef.current ||
+ browserBackExitRef.current ||
+ pendingNavigationRef.current
+ )
+ return
+ armHistoryGuard()
+ }, [armHistoryGuard, hasUnsavedChanges])
+
+ useEffect(() => {
+ const handlePopState = () => {
+ if (!historyGuardArmedRef.current) return
+ historyGuardArmedRef.current = false
+ if (historyGuardReleaseRef.current) {
+ historyGuardReleaseRef.current = false
+ setHistoryGuardReleaseVersion((version) => version + 1)
+ return
+ }
+ const pendingNavigation = pendingNavigationRef.current
+ if (pendingNavigation) {
+ pendingNavigationRef.current = undefined
+ router.replace(pendingNavigation)
+ return
+ }
+ if (!hasUnsavedChanges) {
+ clearStoredSourceDraft()
+ router.replace(detailPath)
+ return
+ }
+ browserBackExitRef.current = true
+ exitDestinationRef.current = detailPath
+ setDiscardError(false)
+ setExitOpen(true)
+ }
+
+ globalThis.addEventListener('popstate', handlePopState)
+ return () => globalThis.removeEventListener('popstate', handlePopState)
+ }, [clearStoredSourceDraft, detailPath, hasUnsavedChanges, router])
+
+ useEffect(() => {
+ if (
+ !websiteReady ||
+ !historyGuardArmedRef.current ||
+ historyGuardReleaseRef.current ||
+ pendingNavigationRef.current
+ )
+ return
+ historyGuardReleaseRef.current = true
+ globalThis.history.back()
+ }, [websiteReady])
+
+ useEffect(() => {
+ if (!hasUnsavedChanges) return
+ const handleBeforeUnload = (event: BeforeUnloadEvent) => {
+ event.preventDefault()
+ event.returnValue = ''
+ }
+ globalThis.addEventListener('beforeunload', handleBeforeUnload)
+ return () => globalThis.removeEventListener('beforeunload', handleBeforeUnload)
+ }, [hasUnsavedChanges])
+
+ useEffect(() => {
+ if (!hasUnsavedChanges) return
+ const handleDocumentClick = (event: MouseEvent) => {
+ if (
+ event.defaultPrevented ||
+ event.button !== 0 ||
+ event.metaKey ||
+ event.ctrlKey ||
+ event.shiftKey ||
+ event.altKey
+ )
+ return
+ const anchor = event
+ .composedPath()
+ .find((target): target is HTMLAnchorElement => target instanceof HTMLAnchorElement)
+ if (
+ !anchor ||
+ anchor.hasAttribute('download') ||
+ (anchor.target && anchor.target !== '_self')
+ )
+ return
+ const destination = new URL(anchor.href, globalThis.location.href)
+ if (destination.origin !== globalThis.location.origin) return
+ const current = new URL(globalThis.location.href)
+ if (destination.pathname === current.pathname && destination.search === current.search) return
+ event.preventDefault()
+ requestNavigation(`${destination.pathname}${destination.search}${destination.hash}`)
+ }
+
+ document.addEventListener('click', handleDocumentClick, true)
+ return () => document.removeEventListener('click', handleDocumentClick, true)
+ }, [hasUnsavedChanges, requestNavigation])
+
+ const requestExit = () => requestNavigation(detailPath)
+ const cancelExit = () => {
+ setExitOpen(false)
+ setDiscardError(false)
+ if (!browserBackExitRef.current) return
+ browserBackExitRef.current = false
+ armHistoryGuard()
+ }
+ const confirmExit = () => {
+ if (discarding) return
+ setDiscarding(true)
+ setDiscardError(false)
+ try {
+ clearStoredSourceDraft()
+ sourceDraftBaselineRef.current = JSON.stringify(sourceDraft)
+ setConnectionDraftDirty(false)
+ setExitOpen(false)
+ browserBackExitRef.current = false
+ const destination = exitDestinationRef.current
+ exitDestinationRef.current = detailPath
+ replaceAfterHistoryGuard(destination)
+ } catch {
+ setDiscardError(true)
+ } finally {
+ setDiscarding(false)
+ }
+ }
+
+ if (
+ !sourceDraftResolved ||
+ (websiteSourceSelected && (providersQuery.isPending || loadingConnections))
+ )
return (
)
- const queryError =
- providersQuery.error || connectionsQuery.error || connectionsQuery.isFetchNextPageError
-
return (
-
-
-
-
- {sourceType === 'websiteCrawl' ? (
- <>
-
- {queryError ? (
-
-
- {t(($) => $['newKnowledge.providerLoadFailed'])}
-
-
-
- ) : !provider ? (
-
- {t(($) => $['newKnowledge.providerUnavailable'])}
-
- ) : !provider.available || !supportsDirectConnection ? (
-
-
{provider.displayName}
-
- {provider.unavailableReason ?? t(($) => $['newKnowledge.providerUnavailable'])}
-
-
- ) : connection?.status === 'active' ? (
-
- ) : connection?.status === 'provisioning' ? (
-
- ) : connection ? (
-
- ) : (
-
- )}
-
- ) : (
-
-
- {t(($) => $[`newKnowledge.${sourceType}`])}
-
-
- {t(($) => $['newKnowledge.providerUnavailable'])}
-
-
- )}
-
-
- {sourceType === 'websiteCrawl' ? (
+ <>
+
+
+
+
{
+ sourceDraftsRef.current[sourceDraft.sourceType] = sourceDraft
+ updateSourceDraft(
+ sourceDraftsRef.current[value] ?? createNewKnowledgeSourceDraft(value),
+ )
+ setConnectedSourceBoundaryVisible(false)
+ }}
+ />
+ {sourceDraft.sourceType === 'websiteCrawl' ? (
<>
+ {
+ updateSourceDraft({ ...sourceDraft, provider })
+ setConnectedSourceBoundaryVisible(false)
+ }}
+ />
+ {!websiteSourceSelected ? (
+
+
{sourceDraft.provider}
+
+ {t(($) => $['newKnowledge.providerUnavailable'])}
+
+
+
+ {t(($) => $['newKnowledge.providerLoadFailed'])}
+
+
+
+ {t(($) => $['newKnowledge.firecrawlUnavailable'])}
+
+ ) : !provider.available || !supportsDirectConnection ? (
+
+
+ {FIRECRAWL_CONNECTION_NAME}
+
+
+ {provider.unavailableReason ?? t(($) => $['newKnowledge.providerUnavailable'])}
+
+
+
+
+ ) : connection?.status === 'provisioning' ? (
+
+ ) : connection ? (
+
+ ) : (
+
+ )}
+ {!websiteReady && (
+
+ )}
+
+ ) : (
+ {
+ updateSourceDraft(draft)
+ setConnectedSourceBoundaryVisible(false)
+ }}
+ />
+ )}
+ {sourceType === 'websiteCrawl' && !websiteReady && (
+
+
{t(($) => $['newKnowledge.addSourceRequiresSelection'])}
@@ -1177,21 +1069,39 @@ export function AddSourcePage({ knowledgeSpaceId }: { knowledgeSpaceId: string }
>
{t(($) => $['newKnowledge.addSource'])}
-
- ) : (
-
+
+ )}
+ {sourceType !== 'websiteCrawl' && (
+
+
+
+
+ )}
+ {connectedSourceBoundaryVisible && (
+
+ {t(($) => $['newKnowledge.sourceSetupBackendDependency'])}
+
)}
+
void confirmExit()}
+ onConfirm={confirmExit}
open={exitOpen}
/>
-
+
)
}
diff --git a/web/features/new-rag/add-source-placeholder.tsx b/web/features/new-rag/add-source-placeholder.tsx
new file mode 100644
index 00000000000..f877e11d529
--- /dev/null
+++ b/web/features/new-rag/add-source-placeholder.tsx
@@ -0,0 +1,269 @@
+'use client'
+
+import type {
+ NewKnowledgeOnlineDocumentsProvider,
+ NewKnowledgeOnlineDocumentsSourceDraft,
+ NewKnowledgeOnlineDriveProvider,
+ NewKnowledgeOnlineDriveSourceDraft,
+ NewKnowledgeSourceDraft,
+ NewKnowledgeWebsiteSourceDraft,
+} from './routes'
+import { Button } from '@langgenius/dify-ui/button'
+import { cn } from '@langgenius/dify-ui/cn'
+import { useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+ isValidWebsiteSourceDraft,
+ NEW_KNOWLEDGE_SOURCE_NAME_MAX_LENGTH,
+ NEW_KNOWLEDGE_SOURCE_URL_MAX_LENGTH,
+} from './routes'
+
+const connectedProviders = {
+ onlineDocuments: [
+ { icon: 'i-custom-public-common-notion', label: 'Notion' },
+ { icon: 'i-ri-file-text-line', label: 'Google Docs' },
+ { icon: 'i-ri-links-line', label: 'Confluence' },
+ ],
+ onlineDrive: [
+ { icon: 'i-custom-public-common-google-drive', label: 'Google Drive' },
+ { icon: 'i-ri-cloud-line', label: 'OneDrive' },
+ { icon: 'i-ri-box-3-line', label: 'Amazon S3' },
+ ],
+} as const
+
+export function PendingWebsiteSetup({
+ draft,
+ onDraftChange,
+}: {
+ draft: NewKnowledgeWebsiteSourceDraft
+ onDraftChange: (draft: NewKnowledgeWebsiteSourceDraft) => void
+}) {
+ const { t } = useTranslation('dataset')
+ const [optionsExpanded, setOptionsExpanded] = useState(false)
+ const [backendBoundaryVisible, setBackendBoundaryVisible] = useState(false)
+ const updateDraft = (nextDraft: NewKnowledgeWebsiteSourceDraft) => {
+ onDraftChange(nextDraft)
+ setBackendBoundaryVisible(false)
+ }
+
+ return (
+
+ )
+}
+
+export function UnavailableConnectedSourceSetup({
+ draft,
+ onDraftChange,
+}: {
+ draft: NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft
+ onDraftChange: (draft: NewKnowledgeSourceDraft) => void
+}) {
+ const { t } = useTranslation('dataset')
+ const { t: tCreation } = useTranslation('datasetCreation')
+ const sourceType = draft.sourceType
+ const providers = connectedProviders[sourceType]
+ const activeProvider = draft.provider
+ const selectProvider = (provider: string) => {
+ if (draft.sourceType === 'onlineDocuments')
+ onDraftChange({ ...draft, provider: provider as NewKnowledgeOnlineDocumentsProvider })
+ else onDraftChange({ ...draft, provider: provider as NewKnowledgeOnlineDriveProvider })
+ }
+
+ return (
+
+
+
+ {sourceType === 'onlineDrive' && (
+
$['newKnowledge.selectFilesAndFolders'])}
+ className="flex min-h-40 flex-col items-center justify-center rounded-lg border border-dashed border-divider-regular px-6 text-center"
+ >
+
+
+
+
+ {t(($) => $['newKnowledge.selectFilesAndFolders'])}
+
+
+ )}
+
+
+
+
+
+
+
+
+ {t(($) => $['newKnowledge.providerUnavailable'])}
+
+
+
-
-
-
-
-
-
- {title}
+ }
+ aria-labelledby={titleId}
+ aria-describedby={disabled ? `${descriptionId} ${unavailableId}` : descriptionId}
+ disabled={disabled}
+ className={cn(
+ 'relative flex min-h-16 w-full items-center gap-3 px-4 py-3.5 text-left outline-hidden',
+ 'hover:bg-state-base-hover focus-visible:ring-2 focus-visible:ring-state-accent-solid focus-visible:ring-inset',
+ 'data-disabled:cursor-not-allowed data-disabled:opacity-50 data-disabled:hover:bg-transparent',
+ )}
+ >
+
+
+
-
- {description}
+
+
+ {title}
+
+
+ {description}
+
-
- {value === 'source' && (
-
- )}
- {disabled && (
-
- {t(($) => $['cornerLabel.unavailable'])}
-
- )}
-
+ {value === 'source' && (
+
+ )}
+ {disabled && (
+
+ {t(($) => $['cornerLabel.unavailable'])}
+
+ )}
+
+ {selected && children}
+
)
}
diff --git a/web/features/new-rag/crawl-selection-form.tsx b/web/features/new-rag/crawl-selection-form.tsx
new file mode 100644
index 00000000000..c46c4d73a3d
--- /dev/null
+++ b/web/features/new-rag/crawl-selection-form.tsx
@@ -0,0 +1,675 @@
+'use client'
+
+import type {
+ GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse,
+ GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse,
+ PutKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyData,
+ Source,
+ SourceWorkflowRun,
+} from '@dify/contracts/knowledge-fs/types.gen'
+import type { FormEvent } from 'react'
+import { Button } from '@langgenius/dify-ui/button'
+import { Checkbox } from '@langgenius/dify-ui/checkbox'
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
+import { useId, useMemo, useRef, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { useRouter } from '@/next/navigation'
+import { consoleClient, consoleQuery } from '@/service/client'
+import { createRequestId } from './request-id'
+import { newKnowledgeDetailPath } from './routes'
+
+type PreviewPage = GetKnowledgeSpacesByIdSourceWorkflowsByRunIdPagesResponse['items'][number]
+type SyncPolicy = GetKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyResponse
+type SyncMode = SyncPolicy['mode']
+type SyncPolicyBody = PutKnowledgeSpacesByIdSourcesBySourceIdSyncPolicyData['body']
+
+const MIN_CUSTOM_INTERVAL_HOURS = 1
+const MAX_CUSTOM_INTERVAL_HOURS = 720
+const MAX_SELECTED_PAGES = 200
+const IMPORT_POLL_INTERVAL_MS = 1_000
+const IMPORT_POLL_ATTEMPTS = 120
+const SUCCESSFUL_IMPORT_STATES = new Set(['complete', 'completed', 'success', 'succeeded'])
+const TERMINAL_IMPORT_STATES = new Set([
+ ...SUCCESSFUL_IMPORT_STATES,
+ 'canceled',
+ 'cancelled',
+ 'error',
+ 'exhausted',
+ 'failed',
+ 'superseded',
+ 'timed_out',
+ 'timeout',
+ 'zero_results',
+])
+
+type PageSkipReason = 'failed' | 'off-domain'
+
+function requestStatus(error: unknown) {
+ if (error instanceof Response) return error.status
+ if (!error || typeof error !== 'object') return undefined
+ if ('status' in error && typeof error.status === 'number') return error.status
+ if ('data' in error && error.data && typeof error.data === 'object' && 'status' in error.data)
+ return typeof error.data.status === 'number' ? error.data.status : undefined
+}
+
+function isDefinitiveRequestFailure(error: unknown) {
+ const status = requestStatus(error)
+ return status !== undefined && [400, 401, 403, 404, 409, 422, 429].includes(status)
+}
+
+function normalizedWorkflowState(run: SourceWorkflowRun) {
+ return run.state.trim().toLowerCase().replaceAll('-', '_').replaceAll(' ', '_')
+}
+
+function isSuccessfulImport(run: SourceWorkflowRun) {
+ return SUCCESSFUL_IMPORT_STATES.has(normalizedWorkflowState(run))
+}
+
+function isTerminalImport(run: SourceWorkflowRun) {
+ return TERMINAL_IMPORT_STATES.has(normalizedWorkflowState(run))
+}
+
+async function waitForImportTerminal(
+ knowledgeSpaceId: string,
+ initialRun: SourceWorkflowRun,
+ onWorkflowRun: (run: SourceWorkflowRun) => void,
+ discardRequested: () => boolean,
+) {
+ let current = initialRun
+ for (let attempt = 0; attempt < IMPORT_POLL_ATTEMPTS; attempt += 1) {
+ if (discardRequested() || isTerminalImport(current)) return current
+ current = await consoleClient.knowledgeFs.getKnowledgeSpacesByIdSourceWorkflowsByRunId({
+ params: { id: knowledgeSpaceId, runId: current.id },
+ })
+ onWorkflowRun(current)
+ if (discardRequested() || isTerminalImport(current)) return current
+ await new Promise((resolve) => setTimeout(resolve, IMPORT_POLL_INTERVAL_MS))
+ }
+ throw new Error('Source import did not reach a terminal state')
+}
+
+function pageSkipReason(page: PreviewPage, rootUrl: string): PageSkipReason | undefined {
+ try {
+ const root = new URL(rootUrl)
+ const candidate = new URL(page.sourceUrl)
+ if (
+ !['http:', 'https:'].includes(candidate.protocol) ||
+ candidate.username ||
+ candidate.password
+ )
+ return 'failed'
+ if (candidate.hostname.toLocaleLowerCase() !== root.hostname.toLocaleLowerCase())
+ return 'off-domain'
+ return undefined
+ } catch {
+ return 'failed'
+ }
+}
+
+function policyConfiguration(mode: SyncMode, customIntervalHours: number) {
+ if (mode === 'manual') return { enabled: false, mode } as const
+ if (mode === 'custom') {
+ return {
+ customIntervalSeconds: customIntervalHours * 3600,
+ enabled: true,
+ mode,
+ } as const
+ }
+ return { enabled: true, mode } as const
+}
+
+function policyMatches(policy: SyncPolicy, desired: ReturnType) {
+ return (
+ policy.revision > 0 &&
+ policy.enabled === desired.enabled &&
+ policy.mode === desired.mode &&
+ (desired.mode !== 'custom' || policy.customIntervalSeconds === desired.customIntervalSeconds)
+ )
+}
+
+function initialSyncPolicy(source: Source): SyncPolicy | undefined {
+ if (!source.version) return undefined
+ return {
+ createdAt: source.createdAt,
+ enabled: true,
+ expectedSourceVersion: source.version,
+ id: source.id,
+ knowledgeSpaceId: source.knowledgeSpaceId,
+ mode: 'provider',
+ revision: 0,
+ sourceId: source.id,
+ updatedAt: source.updatedAt,
+ }
+}
+
+function PolicyLoading() {
+ const { t } = useTranslation('dataset')
+ return (
+ $['newKnowledge.loadingSyncPolicy'])}
+ className="space-y-3"
+ >
+
+
+
+
+
+
+
+
+
+
+
+
+ {t(($) => $['newKnowledge.syncPolicyLoadFailed'])}
+
+
+
+
+
+
+
+