diff --git a/web/features/new-rag/sources/__tests__/page.spec.tsx b/web/features/new-rag/sources/__tests__/page.spec.tsx index 4491142e568..a97fed52dca 100644 --- a/web/features/new-rag/sources/__tests__/page.spec.tsx +++ b/web/features/new-rag/sources/__tests__/page.spec.tsx @@ -1,9 +1,10 @@ +import type { Getter } from 'jotai' import type { Source, SourceSyncPolicy, SourceWorkflowRun } from '../source-models' import type { DataSourceItem } from '@/app/components/workflow/block-selector/types' import { act, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import datasetTranslations from '@/i18n/en-US/dataset.json' -import { renderWithNuqs as render } from '@/test/nuqs-testing' +import { renderWithNuqs } from '@/test/nuqs-testing' import { SourcesPage } from '../page' vi.mock('../../components/knowledge-model-readiness-banner', () => ({ @@ -115,6 +116,9 @@ const sourcesQuery = vi.hoisted(() => ({ isPending: false, refetch: vi.fn(), })) +const jotaiQueryMocks = vi.hoisted(() => ({ + bump: undefined as undefined | (() => void), +})) const connectionsQuery = vi.hoisted(() => ({ data: undefined as | { @@ -205,6 +209,33 @@ vi.mock('@tanstack/react-query', async (importOriginal) => { useQueryClient: () => ({ invalidateQueries: invalidateQueriesMock }), } }) +vi.mock('jotai-tanstack-query', async (importOriginal) => { + const original = await importOriginal() + const { atom, getDefaultStore } = await import('jotai/vanilla') + const revisionAtom = atom(0) + jotaiQueryMocks.bump = () => + getDefaultStore().set(revisionAtom, (revision: number) => revision + 1) + + return { + ...original, + atomWithInfiniteQuery: (getOptions: (get: Getter) => unknown) => + atom((get) => { + get(revisionAtom) + getOptions(get) + return { + ...sourcesQuery, + data: sourcesQuery.data + ? { + pages: sourcesQuery.data.pages.map((page) => ({ + data: page.items.map(sourceApiResponse), + next_cursor: page.nextCursor ?? null, + })), + } + : undefined, + } + }), + } +}) vi.mock('@/next/navigation', () => ({ usePathname: () => '/datasets/new/space-1/sources', useRouter: () => routerMock, @@ -280,6 +311,19 @@ vi.mock('@/service/client', () => ({ }, })) +function render(...args: Parameters) { + jotaiQueryMocks.bump?.() + const rendered = renderWithNuqs(...args) + const rerender = rendered.rerender + return { + ...rendered, + rerender: (ui: Parameters[0]) => { + jotaiQueryMocks.bump?.() + rerender(ui) + }, + } +} + const source = (overrides: Partial): Source => ({ createdAt: '2026-07-20T10:00:00Z', id: 'source-1', diff --git a/web/features/new-rag/sources/__tests__/state-boundary.spec.tsx b/web/features/new-rag/sources/__tests__/state-boundary.spec.tsx new file mode 100644 index 00000000000..a3ddfbf757d --- /dev/null +++ b/web/features/new-rag/sources/__tests__/state-boundary.spec.tsx @@ -0,0 +1,147 @@ +import { act, render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { atom, createStore, Provider, useAtomValue, useSetAtom } from 'jotai' +import { + removedSourceIdsAtom, + removeSourceFromListAtom, + sourcesAwaitedOperationIdAtom, + sourcesFilterAtom, + sourcesKnowledgeSpaceIdAtom, + sourcesSearchAtom, + sourcesSortAtom, +} from '../state' +import { SourcesStateBoundary } from '../state-boundary' + +const parentValueAtom = atom('missing') + +function SourcesInputs({ label }: { label: string }) { + const knowledgeSpaceId = useAtomValue(sourcesKnowledgeSpaceIdAtom) + const filter = useAtomValue(sourcesFilterAtom) + const search = useAtomValue(sourcesSearchAtom) + const sort = useAtomValue(sourcesSortAtom) + const awaitedOperationId = useAtomValue(sourcesAwaitedOperationIdAtom) + const parentValue = useAtomValue(parentValueAtom) + return ( +

{`${label}:${knowledgeSpaceId}:${filter}:${search}:${sort}:${awaitedOperationId}:${parentValue}`}

+ ) +} + +function RemovedSourcesSession() { + const removedSourceIds = useAtomValue(removedSourceIdsAtom) + const removeSource = useSetAtom(removeSourceFromListAtom) + return ( + + ) +} + +describe('SourcesStateBoundary', () => { + it('isolates route inputs between sibling instances while observing the parent store', () => { + const store = createStore() + store.set(parentValueAtom, 'parent-visible') + + render( + + + + + + + + , + ) + + expect( + screen.getByText('first:space-a:active:alpha:name-asc:operation-a:parent-visible'), + ).toBeInTheDocument() + expect( + screen.getByText('second:space-b:error:beta:name-desc:null:parent-visible'), + ).toBeInTheDocument() + }) + + it('follows authoritative URL inputs without resetting the space session', async () => { + const user = userEvent.setup() + const rendered = render( + + + + , + ) + + await user.click(screen.getByRole('button', { name: 'none' })) + expect(screen.getByRole('button', { name: 'source-1' })).toBeInTheDocument() + + act(() => + rendered.rerender( + + + + , + ), + ) + + expect( + screen.getByText('sources:space-a:syncing:docs:name-asc:operation-2:missing'), + ).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'source-1' })).toBeInTheDocument() + }) + + it('resets workflow primitives when the knowledge space identity changes', async () => { + const user = userEvent.setup() + const rendered = render( + + + , + ) + + await user.click(screen.getByRole('button', { name: 'none' })) + expect(screen.getByRole('button', { name: 'source-1' })).toBeInTheDocument() + + act(() => + rendered.rerender( + + + , + ), + ) + + expect(screen.getByRole('button', { name: 'none' })).toBeInTheDocument() + }) +}) diff --git a/web/features/new-rag/sources/page.tsx b/web/features/new-rag/sources/page.tsx index 453e2ac482a..2f9850160a4 100644 --- a/web/features/new-rag/sources/page.tsx +++ b/web/features/new-rag/sources/page.tsx @@ -2,6 +2,7 @@ import type { SourceFilter } from './source-list-query-state' import type { Source } from './source-models' +import type { SourceSort } from './state' import { Button, buttonVariants } from '@langgenius/dify-ui/button' import { Checkbox } from '@langgenius/dify-ui/checkbox' import { cn } from '@langgenius/dify-ui/cn' @@ -14,112 +15,80 @@ import { SelectLabel, SelectTrigger, } from '@langgenius/dify-ui/select' -import { useInfiniteQuery } from '@tanstack/react-query' +import { useAtomValue, useSetAtom } from 'jotai' import { useQueryState } from 'nuqs' -import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import Loading from '@/app/components/base/loading' import { SearchInput } from '@/app/components/base/search-input' import Link from '@/next/link' -import { usePathname, useRouter, useSearchParams } from '@/next/navigation' -import { consoleQuery } from '@/service/client' +import { useSearchParams } from '@/next/navigation' import { KnowledgeModelReadinessBanner } from '../components/knowledge-model-readiness-banner' import { KnowledgeModelSetupDialog } from '../components/knowledge-model-setup-dialog' import { newKnowledgeAddSourcePath } from '../routes' import { useKnowledgeSpacePermission } from '../space/context' import { useKnowledgeModelSetupGuard } from '../use-knowledge-model-setup-guard' import { SourcesEmpty } from './empty' +import { SourcesRuntimeController } from './runtime-controller' import { SourceRow } from './source-list-item' import { sourceTableGridClass } from './source-list-layout' import { sourceFilterParser, sourceSearchParser, sourceSortParser } from './source-list-query-state' import { - initialSourcePollingPhase, - isInitialSourceForOperation, - shouldHidePreviewSource, - sourceDisplayStatus, - sourceFromApi, - sourceNeedsPolling, - sourceStatusWithSyncWorkflow, - sourceWorkflowIsActive, -} from './source-models' - -const PAGE_SIZE = 200 -const MAX_AUTO_CURSOR_PAGES = 5 -const AWAIT_INITIAL_SOURCE_POLL_INTERVAL = 2000 -const SOURCE_POLL_INTERVAL = 3000 -const INITIAL_SOURCE_POLL_TIMEOUT = 10 * 60 * 1000 - -function latestSourceWorkflow( - sourceWorkflow?: Source['syncWorkflow'], - sourceOverrideWorkflow?: Source['syncWorkflow'], -) { - if (!sourceWorkflow || !sourceOverrideWorkflow) return sourceWorkflow ?? sourceOverrideWorkflow - if (sourceWorkflow.id === sourceOverrideWorkflow.id) { - if (sourceWorkflow.executionAttempts !== sourceOverrideWorkflow.executionAttempts) - return sourceWorkflow.executionAttempts > sourceOverrideWorkflow.executionAttempts - ? sourceWorkflow - : sourceOverrideWorkflow - - return sourceWorkflow.updatedAt >= sourceOverrideWorkflow.updatedAt - ? sourceWorkflow - : sourceOverrideWorkflow - } - const sourceWorkflowIsRunning = sourceWorkflowIsActive(sourceWorkflow) - const sourceOverrideWorkflowIsRunning = sourceWorkflowIsActive(sourceOverrideWorkflow) - // The server snapshot ranks active runs first, so an active server run remains authoritative - // even when it is an older run being retried. A local active override still has to be newer - // than a terminal server run, otherwise it could remain stuck after a later run completes. - if (sourceWorkflowIsRunning && !sourceOverrideWorkflowIsRunning) return sourceWorkflow - const createdAtComparison = sourceWorkflow.createdAt.localeCompare( - sourceOverrideWorkflow.createdAt, - ) - if (createdAtComparison !== 0) - return createdAtComparison > 0 ? sourceWorkflow : sourceOverrideWorkflow - const updatedAtComparison = sourceWorkflow.updatedAt.localeCompare( - sourceOverrideWorkflow.updatedAt, - ) - if (updatedAtComparison !== 0) - return updatedAtComparison > 0 ? sourceWorkflow : sourceOverrideWorkflow - return sourceWorkflow.id > sourceOverrideWorkflow.id ? sourceWorkflow : sourceOverrideWorkflow -} - -function getCurrentSource(source: Source, sourceOverride?: Source) { - if (!sourceOverride || sourceOverride.id !== source.id) return source - const sourceVersion = source.version ?? -1 - const overrideVersion = sourceOverride.version ?? -1 - if (sourceVersion > overrideVersion) return source - const overrideHasNewerSource = - sourceVersion < overrideVersion || source.updatedAt < sourceOverride.updatedAt - const sourceHasNewerSource = - sourceVersion === overrideVersion && source.updatedAt > sourceOverride.updatedAt - if (sourceHasNewerSource) return source - const syncWorkflow = overrideHasNewerSource - ? sourceOverride.syncWorkflow - : latestSourceWorkflow(source.syncWorkflow, sourceOverride.syncWorkflow) - if ( - !overrideHasNewerSource && - source.syncWorkflow && - source.syncWorkflow.id !== sourceOverride.syncWorkflow?.id && - syncWorkflow === source.syncWorkflow - ) - return source - return { - ...sourceOverride, - lastSyncedAt: source.lastSyncedAt ?? sourceOverride.lastSyncedAt, - status: sourceStatusWithSyncWorkflow(sourceOverride.status, syncWorkflow), - syncWorkflow, - syncPolicy: overrideHasNewerSource - ? (sourceOverride.syncPolicy ?? source.syncPolicy) - : (source.syncPolicy ?? sourceOverride.syncPolicy), - } -} + completingFilteredResultsAtom, + fetchNextSourcePageAtom, + filteredSourcesAtom, + refreshSourcesAtom, + sourcesFilterAtom, + sourcesKnowledgeSpaceIdAtom, + sourcesPollingPhaseAtom, + sourcesQueryErrorAtom, + sourcesQueryFetchingNextPageAtom, + sourcesQueryFetchNextPageErrorAtom, + sourcesQueryHasDataAtom, + sourcesQueryHasNextPageAtom, + sourcesQueryPendingAtom, + sourcesSearchAtom, + sourcesSortAtom, + visibleSourcesAtom, +} from './state' +import { SourcesStateBoundary } from './state-boundary' export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) { + const searchParams = useSearchParams() + const [filter, setFilter] = useQueryState('status', sourceFilterParser) + const [search, setSearch] = useQueryState('query', sourceSearchParser) + const [sort, setSort] = useQueryState('sort', sourceSortParser) + const awaitedOperationId = searchParams.get('awaitInitialSource')?.trim() || null + + return ( + + void setFilter(value)} + onSearchChange={(value) => void setSearch(value)} + onSortChange={(value) => void setSort(value)} + /> + + ) +} + +function SourcesPageContent({ + onFilterChange, + onSearchChange, + onSortChange, +}: { + onFilterChange: (value: SourceFilter) => void + onSearchChange: (value: string) => void + onSortChange: (value: SourceSort) => void +}) { const { t } = useTranslation('dataset') const { t: tCommon } = useTranslation('common') - const pathname = usePathname() - const router = useRouter() - const searchParams = useSearchParams() + const knowledgeSpaceId = useAtomValue(sourcesKnowledgeSpaceIdAtom) const { configureModelSetup, ensureModelReady, @@ -134,174 +103,25 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) [ensureModelReady], ) const canManageSources = useKnowledgeSpacePermission('knowledge_space_document_write') - const [filter, setFilter] = useQueryState('status', sourceFilterParser) - const [search, setSearch] = useQueryState('query', sourceSearchParser) - const [sort, setSort] = useQueryState('sort', sourceSortParser) - const [selectedSourceIds, setSelectedSourceIds] = useState>(() => new Set()) - const [sourceOverrides, setSourceOverrides] = useState>({}) - const [removedSourceIds, setRemovedSourceIds] = useState>(() => new Set()) - const [initialSourcePollingTimedOut, setInitialSourcePollingTimedOut] = useState(false) - const initialSourcePollingTimeoutRef = useRef | undefined>( - undefined, - ) - const normalizedAwaitedOperationId = searchParams.get('awaitInitialSource')?.trim() || null - const sourcesQuery = useInfiniteQuery( - consoleQuery.knowledgeFs.spaces.byControlSpaceId.sources.get.infiniteOptions({ - input: (pageParam) => ({ - params: { control_space_id: knowledgeSpaceId }, - query: { - limit: PAGE_SIZE, - ...(typeof pageParam === 'string' ? { cursor: pageParam } : {}), - }, - }), - getNextPageParam: (lastPage) => lastPage.next_cursor, - initialPageParam: null as string | null, - refetchInterval: (query) => { - const currentSources = - query.state.data?.pages.flatMap((page) => - page.data - .filter((source) => !removedSourceIds.has(source.id)) - .map((source) => getCurrentSource(sourceFromApi(source), sourceOverrides[source.id])), - ) ?? [] - const phase = initialSourcePollingPhase( - currentSources, - normalizedAwaitedOperationId, - initialSourcePollingTimedOut, - ) - if (phase === 'awaiting') return AWAIT_INITIAL_SOURCE_POLL_INTERVAL - - return currentSources.some( - (source) => - sourceNeedsPolling(source) && - (!initialSourcePollingTimedOut || sourceDisplayStatus(source) !== 'initializing'), - ) - ? SOURCE_POLL_INTERVAL - : false - }, - }), - ) - const remoteSources = sourcesQuery.data?.pages.flatMap((page) => - page.data.map((source) => sourceFromApi(source)), - ) - const currentSources = useMemo( - () => - (remoteSources ?? []) - .filter((source) => !removedSourceIds.has(source.id)) - .map((source) => getCurrentSource(source, sourceOverrides[source.id])), - [remoteSources, removedSourceIds, sourceOverrides], - ) - const pollingPhase = initialSourcePollingPhase( - currentSources, - normalizedAwaitedOperationId, - initialSourcePollingTimedOut, - ) - const initialSourcePollingActive = pollingPhase === 'awaiting' || pollingPhase === 'initializing' + const filter = useAtomValue(sourcesFilterAtom) + const search = useAtomValue(sourcesSearchAtom) + const sort = useAtomValue(sourcesSortAtom) + const pollingPhase = useAtomValue(sourcesPollingPhaseAtom) const waitingForInitialSource = pollingPhase === 'awaiting' - const sources = useMemo( - () => - currentSources - .filter((source) => !shouldHidePreviewSource(source)) - .sort( - (left, right) => - right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id), - ), - [currentSources], - ) - const filteredSources = useMemo(() => { - const normalizedSearch = search.trim().toLocaleLowerCase() - const nextSources = (sources ?? []).filter((source) => { - if (filter !== 'all' && sourceDisplayStatus(source) !== filter) return false - if (!normalizedSearch) return true - return `${source.name} ${source.uri}`.toLocaleLowerCase().includes(normalizedSearch) - }) - if (!sort) return nextSources - return [...nextSources].sort((left, right) => { - const result = left.name.localeCompare(right.name) - return sort === 'name-asc' ? result : -result - }) - }, [filter, search, sort, sources]) - const localTransformActive = filter !== 'all' || Boolean(search.trim()) || Boolean(sort) - const loadedSourcePageCount = sourcesQuery.data?.pages.length ?? 0 - const canAutoLoadNextPage = loadedSourcePageCount < MAX_AUTO_CURSOR_PAGES - const canAutoCompleteFilteredResults = localTransformActive && canAutoLoadNextPage - const latestSourcePage = sourcesQuery.data?.pages.at(-1) - const needsVisibleSource = - latestSourcePage !== undefined && - latestSourcePage.data.some((source) => - shouldHidePreviewSource(getCurrentSource(sourceFromApi(source), sourceOverrides[source.id])), - ) && - !latestSourcePage.data.some((source) => { - if (removedSourceIds.has(source.id)) return false - return !shouldHidePreviewSource( - getCurrentSource(sourceFromApi(source), sourceOverrides[source.id]), - ) - }) - const completingFilteredResults = - (canAutoCompleteFilteredResults || (needsVisibleSource && canAutoLoadNextPage)) && - !sourcesQuery.isFetchNextPageError && - (sourcesQuery.hasNextPage || sourcesQuery.isFetchingNextPage) - const allFilteredSourcesSelected = - filteredSources.length > 0 && - filteredSources.every((source) => selectedSourceIds.has(source.id)) - const someFilteredSourcesSelected = filteredSources.some((source) => - selectedSourceIds.has(source.id), - ) - const { - fetchNextPage: fetchNextSourcePage, - hasNextPage: hasNextSourcePage, - isFetchingNextPage: isFetchingNextSourcePage, - } = sourcesQuery - - useEffect(() => { - if ( - normalizedAwaitedOperationId && - currentSources.some((source) => - isInitialSourceForOperation(source, normalizedAwaitedOperationId), - ) - ) { - const nextSearchParams = new URLSearchParams(searchParams) - nextSearchParams.delete('awaitInitialSource') - const queryString = nextSearchParams.toString() - router.replace(queryString ? `${pathname}?${queryString}` : pathname, { scroll: false }) - } - }, [currentSources, normalizedAwaitedOperationId, pathname, router, searchParams]) - - useEffect(() => { - if (!initialSourcePollingActive) { - globalThis.clearTimeout(initialSourcePollingTimeoutRef.current) - initialSourcePollingTimeoutRef.current = undefined - return - } - if (initialSourcePollingTimeoutRef.current) return - - initialSourcePollingTimeoutRef.current = globalThis.setTimeout(() => { - initialSourcePollingTimeoutRef.current = undefined - setInitialSourcePollingTimedOut(true) - }, INITIAL_SOURCE_POLL_TIMEOUT) - }, [initialSourcePollingActive]) - - useEffect(() => () => globalThis.clearTimeout(initialSourcePollingTimeoutRef.current), []) - - useEffect(() => { - if ( - (canAutoCompleteFilteredResults || (needsVisibleSource && canAutoLoadNextPage)) && - hasNextSourcePage && - !isFetchingNextSourcePage && - !sourcesQuery.isFetchNextPageError - ) - void fetchNextSourcePage() - }, [ - canAutoCompleteFilteredResults, - canAutoLoadNextPage, - fetchNextSourcePage, - hasNextSourcePage, - isFetchingNextSourcePage, - needsVisibleSource, - sourcesQuery.isFetchNextPageError, - ]) - + const sources = useAtomValue(visibleSourcesAtom) + const filteredSources = useAtomValue(filteredSourcesAtom) + const completingFilteredResults = useAtomValue(completingFilteredResultsAtom) + const sourcesQueryPending = useAtomValue(sourcesQueryPendingAtom) + const sourcesQueryError = useAtomValue(sourcesQueryErrorAtom) + const sourcesQueryHasData = useAtomValue(sourcesQueryHasDataAtom) + const sourcesQueryHasNextPage = useAtomValue(sourcesQueryHasNextPageAtom) + const sourcesQueryFetchNextPageError = useAtomValue(sourcesQueryFetchNextPageErrorAtom) + const sourcesQueryFetchingNextPage = useAtomValue(sourcesQueryFetchingNextPageAtom) + const refreshSources = useSetAtom(refreshSourcesAtom) + const fetchNextSourcePage = useSetAtom(fetchNextSourcePageAtom) return (
+

@@ -312,7 +132,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })

{pollingPhase === 'timed-out' && ( - @@ -335,11 +155,11 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) {t(($) => $['newKnowledge.awaitingInitialSource'])}
)} - {sourcesQuery.isPending ? ( + {sourcesQueryPending ? (
- ) : sourcesQuery.error && !sourcesQuery.data ? ( + ) : sourcesQueryError && !sourcesQueryHasData ? (

@@ -348,7 +168,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })

{t(($) => $['newKnowledge.sourcesErrorDescription'])}

-

@@ -359,7 +179,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) {t(($) => $['newKnowledge.awaitingInitialSource'])}

- ) : !sources?.length && !sourcesQuery.hasNextPage ? ( + ) : !sources?.length && !sourcesQueryHasNextPage ? ( ) : ( <> @@ -367,7 +187,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) value={filter} onValueChange={(value) => { - if (value) void setFilter(value) + if (value) onFilterChange(value) }} > @@ -399,7 +219,7 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) aria-label={t(($) => $['newKnowledge.searchSources'])} className="@min-[768px]/knowledge-content:w-60" value={search} - onValueChange={(value) => void setSearch(value)} + onValueChange={onSearchChange} placeholder={t(($) => $['newKnowledge.searchSources'])} /> {canManageSources && ( @@ -415,138 +235,29 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string }) )} -
- - - - - - - - - - - - - {filteredSources.map((source) => ( - { - setRemovedSourceIds((current) => new Set(current).add(source.id)) - setSelectedSourceIds((current) => { - if (!current.has(source.id)) return current - const next = new Set(current) - next.delete(source.id) - return next - }) - }} - onSourceChange={(updatedSource) => - setSourceOverrides((current) => ({ - ...current, - [updatedSource.id]: updatedSource, - })) - } - onCheckedChange={(checked) => { - setSelectedSourceIds((current) => { - const next = new Set(current) - if (checked) next.add(source.id) - else next.delete(source.id) - return next - }) - }} - /> - ))} - -
- $['operation.selectAll'])} - checked={allFilteredSourcesSelected} - indeterminate={someFilteredSourcesSelected && !allFilteredSourcesSelected} - onCheckedChange={(checked) => { - setSelectedSourceIds((current) => { - const next = new Set(current) - for (const source of filteredSources) { - if (checked) next.add(source.id) - else next.delete(source.id) - } - return next - }) - }} - /> - - - - {t(($) => $['metadata.createMetadata.type'])} - - {t(($) => $['newKnowledge.statusColumn'])} - - {t(($) => $['newKnowledge.syncPolicyColumn'])} - - {t(($) => $['newKnowledge.lastSyncColumn'])} - $['newKnowledge.actionsColumn'])} - /> -
- {!filteredSources.length && - !sourcesQuery.hasNextPage && - !completingFilteredResults && - !sourcesQuery.isFetchNextPageError && ( -

- {t(($) => $['newKnowledge.noMatchingSources'])} -

- )} - {!filteredSources.length && completingFilteredResults && ( -
- -
- )} -
- {sourcesQuery.isFetchNextPageError ? ( + + {sourcesQueryFetchNextPageError ? (
{t(($) => $['newKnowledge.sourcesErrorDescription'])} -
- ) : sourcesQuery.hasNextPage && !completingFilteredResults ? ( + ) : sourcesQueryHasNextPage && !completingFilteredResults ? (
@@ -563,3 +274,130 @@ export function SourcesPage({ knowledgeSpaceId }: { knowledgeSpaceId: string })
) } + +function SourcesTable({ + completingFilteredResults, + ensureModelSetupReady, + filteredSources, + hasNextPage, + isFetchNextPageError, + onSortChange, + sort, +}: { + completingFilteredResults: boolean + ensureModelSetupReady: () => Promise + filteredSources: Source[] + hasNextPage: boolean + isFetchNextPageError: boolean + onSortChange: (value: SourceSort) => void + sort: SourceSort +}) { + const { t } = useTranslation('dataset') + const { t: tCommon } = useTranslation('common') + const [selectedSourceIds, setSelectedSourceIds] = useState>(() => new Set()) + const allFilteredSourcesSelected = + filteredSources.length > 0 && + filteredSources.every((source) => selectedSourceIds.has(source.id)) + const someFilteredSourcesSelected = filteredSources.some((source) => + selectedSourceIds.has(source.id), + ) + + return ( +
+ + + + + + + + + + + + + {filteredSources.map((source) => ( + { + setSelectedSourceIds((current) => { + const next = new Set(current) + if (checked) next.add(source.id) + else next.delete(source.id) + return next + }) + }} + /> + ))} + +
+ $['operation.selectAll'])} + checked={allFilteredSourcesSelected} + indeterminate={someFilteredSourcesSelected && !allFilteredSourcesSelected} + onCheckedChange={(checked) => { + setSelectedSourceIds((current) => { + const next = new Set(current) + for (const source of filteredSources) { + if (checked) next.add(source.id) + else next.delete(source.id) + } + return next + }) + }} + /> + + + + {t(($) => $['metadata.createMetadata.type'])} + + {t(($) => $['newKnowledge.statusColumn'])} + + {t(($) => $['newKnowledge.syncPolicyColumn'])} + + {t(($) => $['newKnowledge.lastSyncColumn'])} + $['newKnowledge.actionsColumn'])} + /> +
+ {!filteredSources.length && + !hasNextPage && + !completingFilteredResults && + !isFetchNextPageError && ( +

+ {t(($) => $['newKnowledge.noMatchingSources'])} +

+ )} + {!filteredSources.length && completingFilteredResults && ( +
+ +
+ )} +
+ ) +} diff --git a/web/features/new-rag/sources/runtime-controller.tsx b/web/features/new-rag/sources/runtime-controller.tsx new file mode 100644 index 00000000000..e7d81a93e66 --- /dev/null +++ b/web/features/new-rag/sources/runtime-controller.tsx @@ -0,0 +1,52 @@ +'use client' + +import { useAtomValue, useSetAtom } from 'jotai' +import { useEffect } from 'react' +import { usePathname, useRouter, useSearchParams } from '@/next/navigation' +import { isInitialSourceForOperation } from './source-models' +import { + currentSourcesAtom, + fetchNextSourcePageAtom, + markSourcePollingTimedOutAtom, + shouldAutoLoadNextSourcePageAtom, + sourcesAwaitedOperationIdAtom, + sourcesPollingPhaseAtom, +} from './state' + +const INITIAL_SOURCE_POLL_TIMEOUT = 10 * 60 * 1000 + +export function SourcesRuntimeController() { + const pathname = usePathname() + const router = useRouter() + const searchParams = useSearchParams() + const awaitedOperationId = useAtomValue(sourcesAwaitedOperationIdAtom) + const currentSources = useAtomValue(currentSourcesAtom) + const pollingPhase = useAtomValue(sourcesPollingPhaseAtom) + const shouldAutoLoadNextSourcePage = useAtomValue(shouldAutoLoadNextSourcePageAtom) + const fetchNextSourcePage = useSetAtom(fetchNextSourcePageAtom) + const markPollingTimedOut = useSetAtom(markSourcePollingTimedOutAtom) + + useEffect(() => { + if ( + awaitedOperationId && + currentSources.some((source) => isInitialSourceForOperation(source, awaitedOperationId)) + ) { + const nextSearchParams = new URLSearchParams(searchParams) + nextSearchParams.delete('awaitInitialSource') + const queryString = nextSearchParams.toString() + router.replace(queryString ? `${pathname}?${queryString}` : pathname, { scroll: false }) + } + }, [awaitedOperationId, currentSources, pathname, router, searchParams]) + + useEffect(() => { + if (pollingPhase !== 'awaiting' && pollingPhase !== 'initializing') return + const timeout = globalThis.setTimeout(markPollingTimedOut, INITIAL_SOURCE_POLL_TIMEOUT) + return () => globalThis.clearTimeout(timeout) + }, [awaitedOperationId, markPollingTimedOut, pollingPhase]) + + useEffect(() => { + if (shouldAutoLoadNextSourcePage) void fetchNextSourcePage() + }, [fetchNextSourcePage, shouldAutoLoadNextSourcePage]) + + return null +} diff --git a/web/features/new-rag/sources/setup/connected-source-configuration.tsx b/web/features/new-rag/sources/setup/connected-source-configuration.tsx index 407f9019bba..de9244d1395 100644 --- a/web/features/new-rag/sources/setup/connected-source-configuration.tsx +++ b/web/features/new-rag/sources/setup/connected-source-configuration.tsx @@ -5,6 +5,7 @@ import type { KnowledgeFsInitialSourcePreviewFileResponse, KnowledgeFsSpaceCreatePayload, } from '@dify/contracts/api/console/knowledge-fs/types.gen' +import type { ReactNode } from 'react' import type { InstalledSourceProviderOption } from './provider-options' import type { NewKnowledgeOnlineDocumentsSourceDraft, @@ -28,6 +29,7 @@ import { SourceNameField, SourceSyncPolicyField } from './fields' type ConnectedDraft = NewKnowledgeOnlineDocumentsSourceDraft | NewKnowledgeOnlineDriveSourceDraft type InitialSource = NonNullable +type ConnectedInitialSource = Extract type PreviewDocument = KnowledgeFsInitialSourcePreviewDocumentResponse type PreviewFile = KnowledgeFsInitialSourcePreviewFileResponse type PreviewResource = @@ -75,11 +77,75 @@ function resourceIcon(resource: PreviewResource) { : 'i-ri-file-3-line text-text-tertiary' } -export function ConnectedSourceConfiguration({ +type ConnectedSourceConfigurationProps = { + disabled: boolean + draft: ConnectedDraft + previewBinding: ConnectedSourceConfigurationBinding + providerOption: InstalledSourceProviderOption + onDraftChange: (draft: NewKnowledgeSourceDraft) => void + onInitialSourceChange: (source?: InitialSource) => void +} + +export function ConnectedSourceConfiguration(props: ConnectedSourceConfigurationProps) { + return +} + +export function ConnectedSourceEditForm({ + disabled, + initialDraft, + previewBinding, + providerOption, + onCancel, + onSubmit, +}: { + disabled: boolean + initialDraft: ConnectedDraft + previewBinding: ConnectedSourceConfigurationBinding + providerOption: InstalledSourceProviderOption + onCancel: () => void + onSubmit: (source: ConnectedInitialSource) => Promise +}) { + const { t: tCommon } = useTranslation('common') + const [draft, setDraft] = useState(initialDraft) + + return ( + ( +
+ + +
+ )} + onDraftChange={(nextDraft) => { + if (nextDraft.sourceType === 'onlineDocuments' || nextDraft.sourceType === 'onlineDrive') + setDraft(nextDraft) + }} + /> + ) +} + +function ConnectedSourceConfigurationFields({ disabled, draft, previewBinding, providerOption, + renderActions, onDraftChange, onInitialSourceChange, }: { @@ -87,8 +153,9 @@ export function ConnectedSourceConfiguration({ draft: ConnectedDraft previewBinding: ConnectedSourceConfigurationBinding providerOption: InstalledSourceProviderOption + renderActions?: (source?: ConnectedInitialSource) => ReactNode onDraftChange: (draft: NewKnowledgeSourceDraft) => void - onInitialSourceChange: (source?: InitialSource) => void + onInitialSourceChange?: (source?: InitialSource) => void }) { const { t } = useTranslation('dataset') const { credentialId, datasource, pluginId, provider, providerDisplayName } = previewBinding @@ -134,13 +201,10 @@ export function ConnectedSourceConfiguration({ }) }, [expanded, resources]) - useEffect(() => { + const initialSource = useMemo(() => { const selectedResources = selectableResources.filter((resource) => selected.has(resource.key)) const name = draft.sourceName.trim() - if (!name || !selectedResources.length) { - onInitialSourceChange(undefined) - return - } + if (!name || !selectedResources.length) return undefined const binding = { credentialId, datasource, @@ -150,7 +214,7 @@ export function ConnectedSourceConfiguration({ providerDisplayName, } if (!driveTransport) { - onInitialSourceChange({ + return { ...binding, kind: 'online_document', name, @@ -172,10 +236,9 @@ export function ConnectedSourceConfiguration({ ? { custom_interval_seconds: draft.customIntervalSeconds } : {}), sync_policy: draft.syncPolicy, - }) - return + } } - onInitialSourceChange({ + return { ...binding, kind: 'online_drive', name, @@ -196,11 +259,10 @@ export function ConnectedSourceConfiguration({ ? { custom_interval_seconds: draft.customIntervalSeconds } : {}), sync_policy: draft.syncPolicy, - }) + } }, [ draft, driveTransport, - onInitialSourceChange, parameters, credentialId, datasource, @@ -211,6 +273,10 @@ export function ConnectedSourceConfiguration({ selected, ]) + useEffect(() => { + onInitialSourceChange?.(initialSource) + }, [initialSource, onInitialSourceChange]) + const requestPreview = useCallback( async ({ append = false, @@ -338,168 +404,178 @@ export function ConnectedSourceConfiguration({ const visibleNextPageRequests = [...nextPageRequests.entries()].filter( ([scope]) => scope === ROOT_PAGE_SCOPE || expanded.has(scope), ) + const handleDraftChange = (nextDraft: NewKnowledgeSourceDraft) => { + if (nextDraft.sourceType !== draft.sourceType) return + if (nextDraft.sourceType === 'onlineDocuments' || nextDraft.sourceType === 'onlineDrive') + onDraftChange(nextDraft) + } return ( -
- - { - setResources([]) - setSelected(new Set()) - setExpanded(new Set()) - setNextPageRequests(new Map()) - setPreviewed(false) - onDraftChange({ ...draft, parameters: nextParameters }) - }} - /> - {!previewed && ( - - )} - {loading && ( -
- -
- )} - {error && ( -
-

- {t(($) => $['newKnowledge.providerLoadFailed'])} -

- -
- )} - {previewed && !loading && ( -
-
- $['newKnowledge.selectAll'])} - aria-describedby={selectionAtLimit ? SELECTION_LIMIT_ID : undefined} - checked={ - selectableResources.length > 0 && - selectableResources.every((resource) => selected.has(resource.key)) - } - disabled={ - disabled || - !selectableResources.length || - (selectionAtLimit && - !selectableResources.every((resource) => selected.has(resource.key))) - } - indeterminate={ - selected.size > 0 && - !selectableResources.every((resource) => selected.has(resource.key)) - } - onCheckedChange={toggleAll} - /> - - {draft.sourceType === 'onlineDocuments' - ? t(($) => $['newKnowledge.selectPagesToSync']) - : t(($) => $['newKnowledge.selectFilesAndFolders'])} - - - {t(($) => $['newKnowledge.pagesSelected'], { count: selected.size })} - {selectionAtLimit && ( - - {t(($) => $['newKnowledge.maxPages'])}: {MAX_SELECTION} - - )} - + )} + {loading && ( +
+
-
    - {visibleResources.map((resource) => { - const container = resource.kind === 'file' && isDriveContainer(resource.file) - return ( -
  • - {container ? ( - - ) : ( - toggle(resource.key)} - /> - )} - - - {resourceName(resource)} + )} + {error && ( +
    +

    + {t(($) => $['newKnowledge.providerLoadFailed'])} +

    + +
    + )} + {previewed && !loading && ( +
    +
    + $['newKnowledge.selectAll'])} + aria-describedby={selectionAtLimit ? SELECTION_LIMIT_ID : undefined} + checked={ + selectableResources.length > 0 && + selectableResources.every((resource) => selected.has(resource.key)) + } + disabled={ + disabled || + !selectableResources.length || + (selectionAtLimit && + !selectableResources.every((resource) => selected.has(resource.key))) + } + indeterminate={ + selected.size > 0 && + !selectableResources.every((resource) => selected.has(resource.key)) + } + onCheckedChange={toggleAll} + /> + + {draft.sourceType === 'onlineDocuments' + ? t(($) => $['newKnowledge.selectPagesToSync']) + : t(($) => $['newKnowledge.selectFilesAndFolders'])} + + + {t(($) => $['newKnowledge.pagesSelected'], { count: selected.size })} + {selectionAtLimit && ( + + {t(($) => $['newKnowledge.maxPages'])}: {MAX_SELECTION} -
    • - ) - })} -
- {visibleNextPageRequests.length > 0 && ( -
- {visibleNextPageRequests.map(([scope, request]) => { - const parent = - scope === ROOT_PAGE_SCOPE ? undefined : resources.find(({ key }) => key === scope) + )} + +
+
    + {visibleResources.map((resource) => { + const container = resource.kind === 'file' && isDriveContainer(resource.file) return ( - + {container ? ( + + ) : ( + toggle(resource.key)} + /> + )} + + + {resourceName(resource)} + + ) })} -
- )} -
- )} - -
+ + {visibleNextPageRequests.length > 0 && ( +
+ {visibleNextPageRequests.map(([scope, request]) => { + const parent = + scope === ROOT_PAGE_SCOPE + ? undefined + : resources.find(({ key }) => key === scope) + return ( + + ) + })} +
+ )} + + )} + + + {renderActions?.(initialSource)} + ) } diff --git a/web/features/new-rag/sources/source-actions.tsx b/web/features/new-rag/sources/source-actions.tsx index 61b83276b20..aaaf6f1ec77 100644 --- a/web/features/new-rag/sources/source-actions.tsx +++ b/web/features/new-rag/sources/source-actions.tsx @@ -11,6 +11,7 @@ import { AlertDialogDescription, AlertDialogTitle, } from '@langgenius/dify-ui/alert-dialog' +import { Button } from '@langgenius/dify-ui/button' import { cn } from '@langgenius/dify-ui/cn' import { DropdownMenu, @@ -20,43 +21,195 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from '@langgenius/dify-ui/dropdown-menu' +import { toast } from '@langgenius/dify-ui/toast' +import { useQueryClient } from '@tanstack/react-query' +import { useAtomValue, useSetAtom } from 'jotai' import { useState } from 'react' import { useTranslation } from 'react-i18next' +import { consoleClient, consoleQuery } from '@/service/client' +import { useKnowledgeSpacePermission } from '../space/context' import { SourceEditDialog } from './source-edit-dialog' -import { getOpenableSourceUri } from './source-list-model' +import { createIdempotencyKey, getOpenableSourceUri } from './source-list-model' +import { + initialSourceWorkflowId, + sourceAsyncImportWorkflowId, + sourceDisplayStatus, + sourceFromApi, + sourceStatusWithSyncWorkflow, + sourceWorkflowFromApi, + sourceWorkflowIsActive, +} from './source-models' +import { + acceptSourceSnapshotAtom, + removeSourceFromListAtom, + sourcesKnowledgeSpaceIdAtom, +} from './state' export function SourceActions({ - canEdit, - canRemove, - canSync, - canToggle, - onEdit, - onRemove, - onSync, - onToggle, - pendingAction, + ensureModelSetupReady, source, - syncAction, }: { - canEdit: boolean - canRemove: boolean - canSync: boolean - canToggle: boolean - onEdit: (values: SourceEditValues) => Promise - onRemove: () => Promise - onSync: () => Promise - onToggle: () => Promise - pendingAction?: SourceAction + ensureModelSetupReady: () => Promise source: Source - syncAction: 'retry' | 'sync' }) { const { t } = useTranslation('dataset') const { t: tCommon } = useTranslation('common') + const queryClient = useQueryClient() + const knowledgeSpaceId = useAtomValue(sourcesKnowledgeSpaceIdAtom) + const acceptSourceSnapshot = useSetAtom(acceptSourceSnapshotAtom) + const removeSourceFromList = useSetAtom(removeSourceFromListAtom) + const canManageSources = useKnowledgeSpacePermission('knowledge_space_document_write') + const [pendingAction, setPendingAction] = useState() const [menuOpen, setMenuOpen] = useState(false) const [editDialogOpen, setEditDialogOpen] = useState(false) const [removeDialogOpen, setRemoveDialogOpen] = useState(false) + const syncWorkflow = source.syncWorkflow + const displayStatus = sourceDisplayStatus(source) + const initializing = displayStatus === 'initializing' + const initialWorkflowId = initialSourceWorkflowId(source) + const initialImportRetrying = Boolean(initialWorkflowId) && displayStatus === 'syncing' + const canEdit = canManageSources && !initializing && !initialWorkflowId + const canRemove = canManageSources && !initializing && !initialImportRetrying + const canSync = canManageSources && !initializing && displayStatus !== 'syncing' + const canToggle = canManageSources && !initializing && !initialWorkflowId + const syncAction = displayStatus === 'error' ? 'retry' : 'sync' const sourceUri = getOpenableSourceUri(source.uri) + const runAction = async ( + action: SourceAction, + mutation: () => Promise, + onAccepted?: (result: Result) => void, + beforeAction?: () => Promise, + ) => { + if (pendingAction) return false + setPendingAction(action) + try { + if (beforeAction && !(await beforeAction())) return false + let result: Result + try { + result = await mutation() + } catch { + toast.error(t(($) => $['newKnowledge.sourcesErrorDescription'])) + try { + await queryClient.invalidateQueries({ + queryKey: consoleQuery.knowledgeFs.spaces.byControlSpaceId.sources.get.key(), + }) + } catch { + return false + } + return false + } + onAccepted?.(result) + + try { + await queryClient.invalidateQueries( + { + queryKey: consoleQuery.knowledgeFs.spaces.byControlSpaceId.sources.get.key(), + }, + { throwOnError: true }, + ) + } catch { + // The accepted mutation is already reflected by the feature graph. + } + return true + } finally { + setPendingAction(undefined) + } + } + + const applyAcceptedWorkflow = (workflow: Parameters[0]) => { + const run = sourceWorkflowFromApi(workflow) + acceptSourceSnapshot({ + ...source, + syncWorkflow: run, + status: sourceStatusWithSyncWorkflow(source.status, run), + }) + } + + const syncSource = () => + runAction( + 'sync', + () => + consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.sync.post({ + headers: { 'Idempotency-Key': createIdempotencyKey() }, + params: { control_space_id: knowledgeSpaceId, source_id: source.id }, + }), + applyAcceptedWorkflow, + ensureModelSetupReady, + ) + + const retrySource = () => { + const retryWorkflowId = + initialWorkflowId ?? sourceAsyncImportWorkflowId(source) ?? syncWorkflow?.id + if (!retryWorkflowId) return syncSource() + + return runAction( + 'sync', + () => + consoleClient.knowledgeFs.spaces.byControlSpaceId.sourceWorkflows.byRunId.retry.post({ + params: { control_space_id: knowledgeSpaceId, run_id: retryWorkflowId }, + }), + applyAcceptedWorkflow, + ensureModelSetupReady, + ) + } + + const toggleSource = () => + runAction( + 'toggle', + async () => + sourceFromApi( + await consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.patch({ + body: { + ...(source.version === undefined ? {} : { expectedVersion: source.version }), + status: source.status === 'disabled' ? 'active' : 'disabled', + }, + params: { control_space_id: knowledgeSpaceId, source_id: source.id }, + }), + ), + (updatedSource) => { + const nextSyncWorkflow = + updatedSource.syncWorkflow ?? + (sourceWorkflowIsActive(source.syncWorkflow) ? source.syncWorkflow : undefined) + acceptSourceSnapshot({ + ...updatedSource, + lastSyncedAt: updatedSource.lastSyncedAt ?? source.lastSyncedAt, + status: sourceStatusWithSyncWorkflow(updatedSource.status, nextSyncWorkflow), + syncWorkflow: nextSyncWorkflow, + syncPolicy: updatedSource.syncPolicy ?? source.syncPolicy, + }) + }, + ) + + const editSource = (values: SourceEditValues) => + runAction( + 'edit', + async () => + sourceFromApi( + await consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.patch({ + body: values, + params: { control_space_id: knowledgeSpaceId, source_id: source.id }, + }), + { useResponseStatus: true }, + ), + acceptSourceSnapshot, + ) + + const removeSource = () => + runAction( + 'remove', + async () => { + if (source.version === undefined) throw new Error('Source version is required') + return consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.delete({ + body: { expectedRevision: source.version }, + headers: { 'Idempotency-Key': createIdempotencyKey() }, + params: { control_space_id: knowledgeSpaceId, source_id: source.id }, + query: { documents: 'keep' }, + }) + }, + () => removeSourceFromList(source.id), + ) + const openEditDialog = () => { setMenuOpen(false) setEditDialogOpen(true) @@ -66,6 +219,18 @@ export function SourceActions({ return ( <> + {canSync && displayStatus === 'error' && ( + + )} $['newKnowledge.sourceActions'], { name: source.name })} @@ -83,7 +248,7 @@ export function SourceActions({ {canSync && ( void onSync()} + onClick={() => void (syncAction === 'retry' ? retrySource() : syncSource())} className="mb-px h-7 gap-2 px-2 system-sm-medium" > @@ -119,7 +284,7 @@ export function SourceActions({ )} {canToggle && ( void onToggle()} + onClick={() => void toggleSource()} className="h-7 gap-2 px-2 system-sm-medium" > - void onRemove().then((removed) => { + void removeSource().then((removed) => { if (removed) setRemoveDialogOpen(false) }) } diff --git a/web/features/new-rag/sources/source-edit-dialog.tsx b/web/features/new-rag/sources/source-edit-dialog.tsx index 5d6e0cc7139..8e5ce83c02f 100644 --- a/web/features/new-rag/sources/source-edit-dialog.tsx +++ b/web/features/new-rag/sources/source-edit-dialog.tsx @@ -8,7 +8,6 @@ import type { import type { NewKnowledgeOnlineDocumentsSourceDraft, NewKnowledgeOnlineDriveSourceDraft, - NewKnowledgeSourceDraft, } from './setup/source-draft' import type { SourceEditValues } from './source-list-model' import type { CrawlPreviewPage, Source, SourceSyncPolicy } from './source-models' @@ -17,11 +16,11 @@ import { Dialog, DialogContent, DialogTitle } from '@langgenius/dify-ui/dialog' import { Field, FieldLabel } from '@langgenius/dify-ui/field' import { Input } from '@langgenius/dify-ui/input' import { useInfiniteQuery } from '@tanstack/react-query' -import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { consoleClient, consoleQuery } from '@/service/client' import { useDataSourceList } from '@/service/use-pipeline' -import { ConnectedSourceConfiguration } from './setup/connected-source-configuration' +import { ConnectedSourceEditForm } from './setup/connected-source-configuration' import { CrawlPreviewPageSelection } from './setup/crawl-selection' import { WebsiteDatasourceParameterForm } from './setup/datasource-parameter-form' import { @@ -137,29 +136,39 @@ export function SourceEditDialog({ pending: boolean source: Source }) { + const connectedDraft = connectedDraftFromSource(source) return ( - {open && ( + - )} + ) } function SourceEditDialogContent(props: { + connectedDraft?: ConnectedSourceDraft onEdit: (values: SourceEditValues) => Promise onOpenChange: (open: boolean) => void pending: boolean source: Source }) { - const draft = connectedDraftFromSource(props.source) - if (draft) return - return + if (props.connectedDraft) + return + if (props.source.type === 'web') return + return } function ConnectedSourceEditDialogContent({ @@ -177,8 +186,6 @@ function ConnectedSourceEditDialogContent({ }) { const { t: tCommon } = useTranslation('common') const { t } = useTranslation('dataset') - const [draft, setDraft] = useState(initialDraft) - const [submission, setSubmission] = useState() const datasourcePluginsQuery = useDataSourceList(true) const { data: connectionsData, @@ -204,10 +211,10 @@ function ConnectedSourceEditDialogContent({ }), ) const providerOptions = useMemo( - () => discoverSourceProviderOptions(draft.sourceType, datasourcePluginsQuery.data ?? []), - [datasourcePluginsQuery.data, draft.sourceType], + () => discoverSourceProviderOptions(initialDraft.sourceType, datasourcePluginsQuery.data ?? []), + [datasourcePluginsQuery.data, initialDraft.sourceType], ) - const normalizedProviderName = normalizeSourceProviderName(draft.provider) + const normalizedProviderName = normalizeSourceProviderName(initialDraft.provider) const providerOption = providerOptions.find( (option) => normalizeSourceProviderName(option.label) === normalizedProviderName, ) @@ -238,32 +245,19 @@ function ConnectedSourceEditDialogContent({ : undefined, [bindingReady, credentialId, datasource, installedProviderOption, pluginId, provider], ) - const handleDraftChange = useCallback( - (nextDraft: NewKnowledgeSourceDraft) => { - if (nextDraft.sourceType !== initialDraft.sourceType) return - setDraft(nextDraft) - setSubmission(undefined) - }, - [initialDraft.sourceType], - ) - const handleInitialSourceChange = useCallback((value?: InitialSource) => { - if (value?.kind === 'online_document' || value?.kind === 'online_drive') setSubmission(value) - else setSubmission(undefined) - }, []) - useEffect(() => { if (source.connectionId && !connection && hasNextPage && !isFetchingNextPage) void fetchNextPage() }, [connection, fetchNextPage, hasNextPage, isFetchingNextPage, source.connectionId]) - const submitEdit = async () => { - if (!submission || pending) return + const submitEdit = async (submission: ConnectedInitialSource) => { + if (pending) return false const syncPolicy = - draft.syncPolicy === 'manual' + submission.sync_policy === 'manual' ? ({ enabled: false, mode: 'manual' } as const) - : draft.syncPolicy === 'custom' + : submission.sync_policy === 'custom' ? ({ - customIntervalSeconds: draft.customIntervalSeconds, + customIntervalSeconds: submission.custom_interval_seconds, enabled: true, mode: 'custom', } as const) @@ -279,6 +273,7 @@ function ConnectedSourceEditDialogContent({ syncPolicy, }) if (accepted) onOpenChange(false) + return accepted } const loading = datasourcePluginsQuery.isPending || connectionsPending @@ -289,60 +284,148 @@ function ConnectedSourceEditDialogContent({ (!loading && !bindingReady) return ( - -
{ - event.preventDefault() - void submitEdit() - }} - > - - {tCommon(($) => $['operation.edit'])} {source.name} - -
- {loading ? ( -
$.loading)} className="py-16 text-center"> - -
- ) : unavailable || !installedProviderOption || !previewBinding ? ( -

- {datasourcePluginsQuery.isError || connectionsError - ? t(($) => $['newKnowledge.providerLoadFailed']) - : t(($) => $['newKnowledge.providerUnavailable'])} -

- ) : ( - + + {tCommon(($) => $['operation.edit'])} {source.name} + +
+ {loading ? ( +
$.loading)} className="py-16 text-center"> + - )} -
+
+ ) : unavailable || !installedProviderOption || !previewBinding ? ( +

+ {datasourcePluginsQuery.isError || connectionsError + ? t(($) => $['newKnowledge.providerLoadFailed']) + : t(($) => $['newKnowledge.providerUnavailable'])} +

+ ) : ( + onOpenChange(false)} + onSubmit={submitEdit} + /> + )} +
+ {(loading || unavailable) && (
-
-
- + )} + ) } -function StandardSourceEditDialogContent({ +function BasicSourceEditDialogContent({ + onEdit, + onOpenChange, + pending, + source, +}: { + onEdit: (values: SourceEditValues) => Promise + onOpenChange: (open: boolean) => void + pending: boolean + source: Source +}) { + const { t } = useTranslation('dataset') + const { t: tCommon } = useTranslation('common') + const [initialSource] = useState(source) + const [nextName, setNextName] = useState(initialSource.name) + const [nextSyncMode, setNextSyncMode] = useState(() => + sourceSyncMode(initialSource), + ) + const [nextCustomIntervalHours, setNextCustomIntervalHours] = useState(() => + sourceCustomIntervalHours(initialSource), + ) + const customIntervalValid = + typeof nextCustomIntervalHours === 'number' && + Number.isInteger(nextCustomIntervalHours) && + nextCustomIntervalHours >= MIN_CUSTOM_INTERVAL_HOURS && + nextCustomIntervalHours <= MAX_CUSTOM_INTERVAL_HOURS + + const submitEdit = async () => { + const name = nextName.trim() + if (!name || !customIntervalValid || pending) return + if ( + await onEdit({ + expectedVersion: initialSource.version, + name, + syncPolicy: syncPolicyConfiguration(nextSyncMode, nextCustomIntervalHours as number), + }) + ) + onOpenChange(false) + } + + return ( +
{ + event.preventDefault() + void submitEdit() + }} + > + + {tCommon(($) => $['operation.edit'])} {initialSource.name} + +
+ + + {t(($) => $['newKnowledge.sourceName'])} + + setNextName(event.target.value)} + /> + +
+
+ { + setNextSyncMode(value.mode) + if (value.customIntervalSeconds) + setNextCustomIntervalHours(value.customIntervalSeconds / 3600) + }} + /> +
+
+ + +
+
+ ) +} + +function WebsiteSourceEditDialogContent({ onEdit, onOpenChange, pending, @@ -587,7 +670,7 @@ function StandardSourceEditDialogContent({ ) return ( - + <>
{ event.preventDefault() @@ -712,6 +795,6 @@ function StandardSourceEditDialogContent({
- + ) } diff --git a/web/features/new-rag/sources/source-list-item.tsx b/web/features/new-rag/sources/source-list-item.tsx index 72ae5486a61..6135cc0c608 100644 --- a/web/features/new-rag/sources/source-list-item.tsx +++ b/web/features/new-rag/sources/source-list-item.tsx @@ -1,40 +1,25 @@ 'use client' import type { StatusDotStatus } from '@langgenius/dify-ui/status-dot' -import type { SourceAction, SourceEditValues } from './source-list-model' import type { Source, SourceDisplayStatus } from './source-models' -import { Button } from '@langgenius/dify-ui/button' import { Checkbox } from '@langgenius/dify-ui/checkbox' import { cn } from '@langgenius/dify-ui/cn' import { StatusDot } from '@langgenius/dify-ui/status-dot' -import { toast } from '@langgenius/dify-ui/toast' import { Tooltip, TooltipContent, TooltipTrigger } from '@langgenius/dify-ui/tooltip' -import { useQueryClient } from '@tanstack/react-query' -import { useState } from 'react' import { useTranslation } from 'react-i18next' import { Infotip } from '@/app/components/base/infotip' import { useFormatTimeFromNow } from '@/hooks/use-format-time-from-now' -import { consoleClient, consoleQuery } from '@/service/client' import { knowledgeFsTaskFailureMessageKey } from '../knowledge-fs-task-error' import { SourceProviderIcon } from './setup/fields' import { SourceActions } from './source-actions' import { sourceTableGridClass } from './source-list-layout' import { - createIdempotencyKey, metadataString, sourceLastSyncAt, sourceProviderDetails, sourceSyncPolicyTranslationKey, } from './source-list-model' -import { - initialSourceWorkflowId, - sourceAsyncImportWorkflowId, - sourceDisplayStatus, - sourceFromApi, - sourceStatusWithSyncWorkflow, - sourceWorkflowFromApi, - sourceWorkflowIsActive, -} from './source-models' +import { sourceDisplayStatus } from './source-models' const statusDotStatus: Record = { active: 'success', @@ -56,36 +41,21 @@ function TruncatedSourceValue({ children, className }: { children: string; class } export function SourceRow({ - canEdit, - canSync, checked, ensureModelSetupReady, - knowledgeSpaceId, onCheckedChange, - onRemoved, - onSourceChange, source, }: { - canEdit: boolean - canSync: boolean checked: boolean ensureModelSetupReady: () => Promise - knowledgeSpaceId: string onCheckedChange: (checked: boolean) => void - onRemoved: () => void - onSourceChange: (source: Source) => void source: Source }) { const { t } = useTranslation('dataset') - const { t: tCommon } = useTranslation('common') const { formatTimeFromNow } = useFormatTimeFromNow() - const queryClient = useQueryClient() - const [pendingAction, setPendingAction] = useState() const syncWorkflow = source.syncWorkflow const displayStatus = sourceDisplayStatus(source) const initializing = displayStatus === 'initializing' - const initialWorkflowId = initialSourceWorkflowId(source) - const initialImportRetrying = Boolean(initialWorkflowId) && displayStatus === 'syncing' const provider = sourceProviderDetails(source) const providerName = provider.name @@ -120,143 +90,6 @@ export function SourceRow({ const sourceIcon = provider.iconClass ?? (source.type === 'web' ? 'i-ri-global-line' : 'i-ri-links-line') - const runAction = async ( - action: SourceAction, - mutation: () => Promise, - onAccepted?: (result: Result) => void, - beforeAction?: () => Promise, - ) => { - if (pendingAction) return false - setPendingAction(action) - try { - if (beforeAction && !(await beforeAction())) return false - let result: Result - try { - result = await mutation() - } catch { - toast.error(t(($) => $['newKnowledge.sourcesErrorDescription'])) - try { - await queryClient.invalidateQueries({ - queryKey: consoleQuery.knowledgeFs.spaces.byControlSpaceId.sources.get.key(), - }) - } catch { - return false - } - return false - } - onAccepted?.(result) - - try { - await queryClient.invalidateQueries( - { - queryKey: consoleQuery.knowledgeFs.spaces.byControlSpaceId.sources.get.key(), - }, - { - throwOnError: true, - }, - ) - } catch { - // The accepted mutation is already reflected by the list-owner state. - } - return true - } finally { - setPendingAction(undefined) - } - } - - const applyAcceptedWorkflow = (workflow: Parameters[0]) => { - const run = sourceWorkflowFromApi(workflow) - onSourceChange({ - ...source, - syncWorkflow: run, - status: sourceStatusWithSyncWorkflow(source.status, run), - }) - } - - const syncSource = () => - runAction( - 'sync', - () => - consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.sync.post({ - headers: { 'Idempotency-Key': createIdempotencyKey() }, - params: { control_space_id: knowledgeSpaceId, source_id: source.id }, - }), - applyAcceptedWorkflow, - ensureModelSetupReady, - ) - - const retrySource = () => { - const retryWorkflowId = - initialWorkflowId ?? sourceAsyncImportWorkflowId(source) ?? syncWorkflow?.id - if (!retryWorkflowId) return syncSource() - - return runAction( - 'sync', - () => - consoleClient.knowledgeFs.spaces.byControlSpaceId.sourceWorkflows.byRunId.retry.post({ - params: { control_space_id: knowledgeSpaceId, run_id: retryWorkflowId }, - }), - applyAcceptedWorkflow, - ensureModelSetupReady, - ) - } - - const toggleSource = () => - runAction( - 'toggle', - async () => - sourceFromApi( - await consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.patch({ - body: { - ...(source.version === undefined ? {} : { expectedVersion: source.version }), - status: source.status === 'disabled' ? 'active' : 'disabled', - }, - params: { control_space_id: knowledgeSpaceId, source_id: source.id }, - }), - ), - (updatedSource) => { - const syncWorkflow = - updatedSource.syncWorkflow ?? - (sourceWorkflowIsActive(source.syncWorkflow) ? source.syncWorkflow : undefined) - onSourceChange({ - ...updatedSource, - lastSyncedAt: updatedSource.lastSyncedAt ?? source.lastSyncedAt, - status: sourceStatusWithSyncWorkflow(updatedSource.status, syncWorkflow), - syncWorkflow, - syncPolicy: updatedSource.syncPolicy ?? source.syncPolicy, - }) - }, - ) - - const editSource = (values: SourceEditValues) => - runAction( - 'edit', - async () => - sourceFromApi( - await consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.patch({ - body: values, - params: { control_space_id: knowledgeSpaceId, source_id: source.id }, - }), - { useResponseStatus: true }, - ), - onSourceChange, - ) - - const removeSource = () => - runAction( - 'remove', - async () => { - if (source.version === undefined) throw new Error('Source version is required') - return consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.delete({ - body: { expectedRevision: source.version }, - headers: { 'Idempotency-Key': createIdempotencyKey() }, - params: { control_space_id: knowledgeSpaceId, source_id: source.id }, - query: { documents: 'keep' }, - }) - }, - onRemoved, - ) - return (
- {canSync && displayStatus === 'error' && ( - - )} - +
diff --git a/web/features/new-rag/sources/state-boundary.tsx b/web/features/new-rag/sources/state-boundary.tsx new file mode 100644 index 00000000000..9edbab7887a --- /dev/null +++ b/web/features/new-rag/sources/state-boundary.tsx @@ -0,0 +1,81 @@ +'use client' + +import type { ReactNode } from 'react' +import type { SourceFilter } from './source-list-query-state' +import type { SourceSort } from './state' +import { ScopeProvider } from 'jotai-scope' +import { useHydrateAtoms } from 'jotai/utils' +import { + sourcesAwaitedOperationIdAtom, + sourcesFilterAtom, + sourcesKnowledgeSpaceIdAtom, + sourcesSearchAtom, + sourcesSessionAtoms, + sourcesSortAtom, +} from './state' + +function SourcesExternalInputsBridge({ + awaitedOperationId, + children, + filter, + search, + sort, +}: { + awaitedOperationId: string | null + children: ReactNode + filter: SourceFilter + search: string + sort: SourceSort +}) { + useHydrateAtoms( + [ + [sourcesFilterAtom, filter], + [sourcesSearchAtom, search], + [sourcesSortAtom, sort], + [sourcesAwaitedOperationIdAtom, awaitedOperationId], + ], + { dangerouslyForceHydrate: true }, + ) + + return children +} + +export function SourcesStateBoundary({ + awaitedOperationId, + children, + filter, + knowledgeSpaceId, + search, + sort, +}: { + awaitedOperationId: string | null + children: ReactNode + filter: SourceFilter + knowledgeSpaceId: string + search: string + sort: SourceSort +}) { + return ( + + + {children} + + + ) +} diff --git a/web/features/new-rag/sources/state.ts b/web/features/new-rag/sources/state.ts new file mode 100644 index 00000000000..12b9146b09d --- /dev/null +++ b/web/features/new-rag/sources/state.ts @@ -0,0 +1,276 @@ +import type { SourceFilter } from './source-list-query-state' +import type { Source } from './source-models' +import { atom } from 'jotai' +import { atomWithInfiniteQuery } from 'jotai-tanstack-query' +import { atomWithLazy, selectAtom } from 'jotai/utils' +import { consoleQuery } from '@/service/client' +import { + initialSourcePollingPhase, + shouldHidePreviewSource, + sourceDisplayStatus, + sourceFromApi, + sourceNeedsPolling, + sourceStatusWithSyncWorkflow, + sourceWorkflowIsActive, +} from './source-models' + +export type SourceSort = 'name-asc' | 'name-desc' | null + +const PAGE_SIZE = 200 +export const MAX_AUTO_CURSOR_PAGES = 5 +const AWAIT_INITIAL_SOURCE_POLL_INTERVAL = 2000 +const SOURCE_POLL_INTERVAL = 3000 + +export const sourcesKnowledgeSpaceIdAtom = atomWithLazy(() => { + throw new Error('Missing Sources knowledge space id') +}) +export const sourcesFilterAtom = atom('all') +export const sourcesSearchAtom = atom('') +export const sourcesSortAtom = atom(null) +export const sourcesAwaitedOperationIdAtom = atom(null) + +export const sourceOverridesAtom = atom>({}) +export const removedSourceIdsAtom = atom>(new Set()) +export const sourcePollingTimeoutAtom = atom<{ + awaitedOperationId: string | null + timedOut: boolean +}>({ awaitedOperationId: null, timedOut: false }) + +function latestSourceWorkflow( + sourceWorkflow?: Source['syncWorkflow'], + sourceOverrideWorkflow?: Source['syncWorkflow'], +) { + if (!sourceWorkflow || !sourceOverrideWorkflow) return sourceWorkflow ?? sourceOverrideWorkflow + if (sourceWorkflow.id === sourceOverrideWorkflow.id) { + if (sourceWorkflow.executionAttempts !== sourceOverrideWorkflow.executionAttempts) + return sourceWorkflow.executionAttempts > sourceOverrideWorkflow.executionAttempts + ? sourceWorkflow + : sourceOverrideWorkflow + + return sourceWorkflow.updatedAt >= sourceOverrideWorkflow.updatedAt + ? sourceWorkflow + : sourceOverrideWorkflow + } + const sourceWorkflowIsRunning = sourceWorkflowIsActive(sourceWorkflow) + const sourceOverrideWorkflowIsRunning = sourceWorkflowIsActive(sourceOverrideWorkflow) + if (sourceWorkflowIsRunning && !sourceOverrideWorkflowIsRunning) return sourceWorkflow + const createdAtComparison = sourceWorkflow.createdAt.localeCompare( + sourceOverrideWorkflow.createdAt, + ) + if (createdAtComparison !== 0) + return createdAtComparison > 0 ? sourceWorkflow : sourceOverrideWorkflow + const updatedAtComparison = sourceWorkflow.updatedAt.localeCompare( + sourceOverrideWorkflow.updatedAt, + ) + if (updatedAtComparison !== 0) + return updatedAtComparison > 0 ? sourceWorkflow : sourceOverrideWorkflow + return sourceWorkflow.id > sourceOverrideWorkflow.id ? sourceWorkflow : sourceOverrideWorkflow +} + +function getCurrentSource(source: Source, sourceOverride?: Source) { + if (!sourceOverride || sourceOverride.id !== source.id) return source + const sourceVersion = source.version ?? -1 + const overrideVersion = sourceOverride.version ?? -1 + if (sourceVersion > overrideVersion) return source + const overrideHasNewerSource = + sourceVersion < overrideVersion || source.updatedAt < sourceOverride.updatedAt + const sourceHasNewerSource = + sourceVersion === overrideVersion && source.updatedAt > sourceOverride.updatedAt + if (sourceHasNewerSource) return source + const syncWorkflow = overrideHasNewerSource + ? sourceOverride.syncWorkflow + : latestSourceWorkflow(source.syncWorkflow, sourceOverride.syncWorkflow) + if ( + !overrideHasNewerSource && + source.syncWorkflow && + source.syncWorkflow.id !== sourceOverride.syncWorkflow?.id && + syncWorkflow === source.syncWorkflow + ) + return source + return { + ...sourceOverride, + lastSyncedAt: source.lastSyncedAt ?? sourceOverride.lastSyncedAt, + status: sourceStatusWithSyncWorkflow(sourceOverride.status, syncWorkflow), + syncWorkflow, + syncPolicy: overrideHasNewerSource + ? (sourceOverride.syncPolicy ?? source.syncPolicy) + : (source.syncPolicy ?? sourceOverride.syncPolicy), + } +} + +const sourcePollingTimedOutAtom = atom((get) => { + const timeout = get(sourcePollingTimeoutAtom) + return timeout.timedOut && timeout.awaitedOperationId === get(sourcesAwaitedOperationIdAtom) +}) + +const sourcesQueryAtom = atomWithInfiniteQuery((get) => { + const knowledgeSpaceId = get(sourcesKnowledgeSpaceIdAtom) + const removedSourceIds = get(removedSourceIdsAtom) + const sourceOverrides = get(sourceOverridesAtom) + const awaitedOperationId = get(sourcesAwaitedOperationIdAtom) + const pollingTimedOut = get(sourcePollingTimedOutAtom) + + return consoleQuery.knowledgeFs.spaces.byControlSpaceId.sources.get.infiniteOptions({ + input: (pageParam) => ({ + params: { control_space_id: knowledgeSpaceId }, + query: { + limit: PAGE_SIZE, + ...(typeof pageParam === 'string' ? { cursor: pageParam } : {}), + }, + }), + getNextPageParam: (lastPage) => lastPage.next_cursor, + initialPageParam: null as string | null, + refetchInterval: (query) => { + const currentSources = + query.state.data?.pages.flatMap((page) => + page.data + .filter((source) => !removedSourceIds.has(source.id)) + .map((source) => getCurrentSource(sourceFromApi(source), sourceOverrides[source.id])), + ) ?? [] + const phase = initialSourcePollingPhase(currentSources, awaitedOperationId, pollingTimedOut) + if (phase === 'awaiting') return AWAIT_INITIAL_SOURCE_POLL_INTERVAL + + return currentSources.some( + (source) => + sourceNeedsPolling(source) && + (!pollingTimedOut || sourceDisplayStatus(source) !== 'initializing'), + ) + ? SOURCE_POLL_INTERVAL + : false + }, + }) +}) + +const sourcesQueryDataAtom = selectAtom(sourcesQueryAtom, (query) => query.data) +export const sourcesQueryHasDataAtom = atom((get) => Boolean(get(sourcesQueryDataAtom))) +export const sourcesQueryErrorAtom = selectAtom(sourcesQueryAtom, (query) => query.error) +export const sourcesQueryPendingAtom = selectAtom(sourcesQueryAtom, (query) => query.isPending) +export const sourcesQueryHasNextPageAtom = selectAtom( + sourcesQueryAtom, + (query) => query.hasNextPage, +) +export const sourcesQueryFetchNextPageErrorAtom = selectAtom( + sourcesQueryAtom, + (query) => query.isFetchNextPageError, +) +export const sourcesQueryFetchingNextPageAtom = selectAtom( + sourcesQueryAtom, + (query) => query.isFetchingNextPage, +) + +export const currentSourcesAtom = atom((get) => { + const removedSourceIds = get(removedSourceIdsAtom) + const sourceOverrides = get(sourceOverridesAtom) + return ( + get(sourcesQueryDataAtom) + ?.pages.flatMap((page) => page.data.map((source) => sourceFromApi(source))) + .filter((source) => !removedSourceIds.has(source.id)) + .map((source) => getCurrentSource(source, sourceOverrides[source.id])) ?? [] + ) +}) + +export const sourcesPollingPhaseAtom = atom((get) => + initialSourcePollingPhase( + get(currentSourcesAtom), + get(sourcesAwaitedOperationIdAtom), + get(sourcePollingTimedOutAtom), + ), +) + +export const visibleSourcesAtom = atom((get) => + get(currentSourcesAtom) + .filter((source) => !shouldHidePreviewSource(source)) + .sort( + (left, right) => + right.createdAt.localeCompare(left.createdAt) || right.id.localeCompare(left.id), + ), +) + +export const filteredSourcesAtom = atom((get) => { + const filter = get(sourcesFilterAtom) + const search = get(sourcesSearchAtom) + const sort = get(sourcesSortAtom) + const normalizedSearch = search.trim().toLocaleLowerCase() + const nextSources = get(visibleSourcesAtom).filter((source) => { + if (filter !== 'all' && sourceDisplayStatus(source) !== filter) return false + if (!normalizedSearch) return true + return `${source.name} ${source.uri}`.toLocaleLowerCase().includes(normalizedSearch) + }) + if (!sort) return nextSources + return [...nextSources].sort((left, right) => { + const result = left.name.localeCompare(right.name) + return sort === 'name-asc' ? result : -result + }) +}) + +const loadedSourcePageCountAtom = atom((get) => get(sourcesQueryDataAtom)?.pages.length ?? 0) +const localTransformActiveAtom = atom((get) => + Boolean( + get(sourcesFilterAtom) !== 'all' || get(sourcesSearchAtom).trim() || get(sourcesSortAtom), + ), +) +const canAutoLoadNextPageAtom = atom( + (get) => get(loadedSourcePageCountAtom) < MAX_AUTO_CURSOR_PAGES, +) +const needsVisibleSourceAtom = atom((get) => { + const latestSourcePage = get(sourcesQueryDataAtom)?.pages.at(-1) + if (!latestSourcePage) return false + const removedSourceIds = get(removedSourceIdsAtom) + const sourceOverrides = get(sourceOverridesAtom) + return ( + latestSourcePage.data.some((source) => + shouldHidePreviewSource(getCurrentSource(sourceFromApi(source), sourceOverrides[source.id])), + ) && + !latestSourcePage.data.some((source) => { + if (removedSourceIds.has(source.id)) return false + return !shouldHidePreviewSource( + getCurrentSource(sourceFromApi(source), sourceOverrides[source.id]), + ) + }) + ) +}) + +export const completingFilteredResultsAtom = atom((get) => { + const canAutoLoadNextPage = get(canAutoLoadNextPageAtom) + const shouldComplete = + (get(localTransformActiveAtom) && canAutoLoadNextPage) || + (get(needsVisibleSourceAtom) && canAutoLoadNextPage) + return ( + shouldComplete && + !get(sourcesQueryFetchNextPageErrorAtom) && + (get(sourcesQueryHasNextPageAtom) || get(sourcesQueryFetchingNextPageAtom)) + ) +}) + +export const shouldAutoLoadNextSourcePageAtom = atom( + (get) => + ((get(localTransformActiveAtom) && get(canAutoLoadNextPageAtom)) || + (get(needsVisibleSourceAtom) && get(canAutoLoadNextPageAtom))) && + get(sourcesQueryHasNextPageAtom) && + !get(sourcesQueryFetchingNextPageAtom) && + !get(sourcesQueryFetchNextPageErrorAtom), +) + +export const acceptSourceSnapshotAtom = atom(null, (_get, set, source: Source) => { + set(sourceOverridesAtom, (current) => ({ ...current, [source.id]: source })) +}) + +export const removeSourceFromListAtom = atom(null, (_get, set, sourceId: string) => { + set(removedSourceIdsAtom, (current) => new Set(current).add(sourceId)) +}) + +export const markSourcePollingTimedOutAtom = atom(null, (get, set) => { + set(sourcePollingTimeoutAtom, { + awaitedOperationId: get(sourcesAwaitedOperationIdAtom), + timedOut: true, + }) +}) + +export const refreshSourcesAtom = atom(null, (get) => get(sourcesQueryAtom).refetch()) +export const fetchNextSourcePageAtom = atom(null, (get) => get(sourcesQueryAtom).fetchNextPage()) + +export const sourcesSessionAtoms = [ + sourceOverridesAtom, + removedSourceIdsAtom, + sourcePollingTimeoutAtom, +] as const