diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx index 357fb57f64..944752fc04 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx @@ -20,12 +20,14 @@ type OnlineDocumentsProps = { isInPipeline?: boolean nodeId: string nodeData: DataSourceNodeType + onCredentialChange: (credentialId: string) => void } const OnlineDocuments = ({ nodeId, nodeData, isInPipeline = false, + onCredentialChange, }: OnlineDocumentsProps) => { const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal) @@ -33,13 +35,11 @@ const OnlineDocuments = ({ documentsData, searchValue, selectedPagesId, - currentWorkspaceId, currentCredentialId, } = useDataSourceStoreWithSelector(useShallow(state => ({ documentsData: state.documentsData, searchValue: state.searchValue, selectedPagesId: state.selectedPagesId, - currentWorkspaceId: state.currentWorkspaceId, currentCredentialId: state.currentCredentialId, }))) @@ -69,19 +69,21 @@ const OnlineDocuments = ({ : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` const getOnlineDocuments = useCallback(async () => { + const { currentCredentialId } = dataSourceStore.getState() + if (!currentCredentialId) return ssePost( datasourceNodeRunURL, { body: { inputs: {}, + credential_id: currentCredentialId, datasource_type: DatasourceType.onlineDocument, }, }, { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { - const { setDocumentsData, setCurrentWorkspaceId } = dataSourceStore.getState() + const { setDocumentsData } = dataSourceStore.getState() setDocumentsData(documentsData.data as DataSourceNotionWorkspace[]) - setCurrentWorkspaceId(documentsData.data[0].workspace_id) }, onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => { Toast.notify({ @@ -94,33 +96,8 @@ const OnlineDocuments = ({ }, [dataSourceStore, datasourceNodeRunURL]) useEffect(() => { - const { - setDocumentsData, - setCurrentWorkspaceId, - setSearchValue, - setSelectedPagesId, - setOnlineDocuments, - setCurrentDocument, - currentNodeIdRef, - } = dataSourceStore.getState() - if (nodeId !== currentNodeIdRef.current) { - setDocumentsData([]) - setCurrentWorkspaceId('') - setSearchValue('') - setSelectedPagesId(new Set()) - setOnlineDocuments([]) - setCurrentDocument(undefined) - currentNodeIdRef.current = nodeId - getOnlineDocuments() - } - else { - // Avoid fetching documents when come back from next step - if (!documentsData.length) - getOnlineDocuments() - } - }, [nodeId]) - - const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId) + getOnlineDocuments() + }, [currentCredentialId]) const handleSearchValueChange = useCallback((value: string) => { const { setSearchValue } = dataSourceStore.getState() @@ -153,7 +130,7 @@ const OnlineDocuments = ({ onClickConfiguration={handleSetting} pluginName={nodeData.datasource_label} currentCredentialId={currentCredentialId} - onCredentialChange={dataSourceStore.getState().setCurrentCredentialId} + onCredentialChange={onCredentialChange} credentials={dataSourceAuth?.result || []} />
@@ -172,13 +149,13 @@ const OnlineDocuments = ({ checkedIds={selectedPagesId} disabledValue={new Set()} searchValue={searchValue} - list={currentWorkspace?.pages || []} + list={documentsData[0].pages || []} pagesMap={PagesMapAndSelectedPagesId} onSelect={handleSelectPages} canPreview={!isInPipeline} onPreview={handlePreviewPage} isMultipleChoice={!isInPipeline} - currentWorkspaceId={currentWorkspaceId} + currentCredentialId={currentCredentialId} /> ) : (
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx index a096d13329..36be578a87 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx @@ -15,7 +15,7 @@ type PageSelectorProps = { canPreview?: boolean onPreview?: (selectedPageId: string) => void isMultipleChoice?: boolean - currentWorkspaceId: string + currentCredentialId: string } export type NotionPageTreeItem = { @@ -42,7 +42,7 @@ const PageSelector = ({ canPreview = true, onPreview, isMultipleChoice = true, - currentWorkspaceId, + currentCredentialId, }: PageSelectorProps) => { const { t } = useTranslation() const [dataList, setDataList] = useState([]) @@ -56,8 +56,7 @@ const PageSelector = ({ depth: 0, } })) - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [currentWorkspaceId]) + }, [currentCredentialId]) const searchDataList = list.filter((item) => { return item.page_name.includes(searchValue) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx index 6e01f2123b..6969539adb 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx @@ -4,7 +4,6 @@ import { useDataSourceStore } from '../../../../store' import Bucket from './bucket' import BreadcrumbItem from './item' import Dropdown from './dropdown' -import type { OnlineDriveFile } from '@/models/pipeline' type BreadcrumbsProps = { prefix: string[] @@ -12,12 +11,6 @@ type BreadcrumbsProps = { bucket: string searchResultsLength: number isInPipeline: boolean - getOnlineDriveFiles: (params: { - prefix?: string[] - bucket?: string - startAfter?: string - fileList?: OnlineDriveFile[] - }) => void } const Breadcrumbs = ({ @@ -26,7 +19,6 @@ const Breadcrumbs = ({ bucket, searchResultsLength, isInPipeline, - getOnlineDriveFiles, }: BreadcrumbsProps) => { const { t } = useTranslation() const dataSourceStore = useDataSourceStore() @@ -56,23 +48,14 @@ const Breadcrumbs = ({ setSelectedFileKeys([]) setBucket('') setPrefix([]) - getOnlineDriveFiles({ - prefix: [], - bucket: '', - fileList: [], - }) - }, [dataSourceStore, getOnlineDriveFiles]) + }, [dataSourceStore]) const handleClickBucketName = useCallback(() => { const { setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState() setFileList([]) setSelectedFileKeys([]) setPrefix([]) - getOnlineDriveFiles({ - prefix: [], - fileList: [], - }) - }, [dataSourceStore, getOnlineDriveFiles]) + }, [dataSourceStore]) const handleClickBreadcrumb = useCallback((index: number) => { const { prefix, setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState() @@ -80,11 +63,7 @@ const Breadcrumbs = ({ setFileList([]) setSelectedFileKeys([]) setPrefix(newPrefix) - getOnlineDriveFiles({ - prefix: newPrefix, - fileList: [], - }) - }, [dataSourceStore, getOnlineDriveFiles]) + }, [dataSourceStore]) return (
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/index.tsx index 174e44ea78..5cdeb5faaa 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/index.tsx @@ -2,7 +2,6 @@ import React from 'react' import Breadcrumbs from './breadcrumbs' import Input from '@/app/components/base/input' import { useTranslation } from 'react-i18next' -import type { OnlineDriveFile } from '@/models/pipeline' type HeaderProps = { prefix: string[] @@ -13,12 +12,6 @@ type HeaderProps = { handleInputChange: React.ChangeEventHandler handleResetKeywords: () => void isInPipeline: boolean - getOnlineDriveFiles: (params: { - prefix?: string[] - bucket?: string - startAfter?: string - fileList?: OnlineDriveFile[] - }) => void } const Header = ({ @@ -30,7 +23,6 @@ const Header = ({ searchResultsLength, handleInputChange, handleResetKeywords, - getOnlineDriveFiles, }: HeaderProps) => { const { t } = useTranslation() @@ -42,7 +34,6 @@ const Header = ({ bucket={bucket} searchResultsLength={searchResultsLength} isInPipeline={isInPipeline} - getOnlineDriveFiles={getOnlineDriveFiles} /> void handleOpenFolder: (file: OnlineDriveFile) => void isLoading: boolean - getOnlineDriveFiles: (params: { - prefix?: string[] - bucket?: string - startAfter?: string - fileList?: OnlineDriveFile[] - }) => void } const FileList = ({ @@ -38,7 +32,6 @@ const FileList = ({ handleOpenFolder, isInPipeline, isLoading, - getOnlineDriveFiles, }: FileListProps) => { const [inputValue, setInputValue] = useState(keywords) @@ -71,7 +64,6 @@ const FileList = ({ handleInputChange={handleInputChange} searchResultsLength={searchResultsLength} handleResetKeywords={handleResetKeywords} - getOnlineDriveFiles={getOnlineDriveFiles} />
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx index 69dc460bc6..e843d4abdb 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx @@ -16,12 +16,6 @@ type FileListProps = { handleResetKeywords: () => void handleSelectFile: (file: OnlineDriveFile) => void handleOpenFolder: (file: OnlineDriveFile) => void - getOnlineDriveFiles: (params: { - prefix?: string[] - bucket?: string - startAfter?: string - fileList?: OnlineDriveFile[] - }) => void } const List = ({ @@ -33,27 +27,23 @@ const List = ({ handleOpenFolder, isInPipeline, isLoading, - getOnlineDriveFiles, }: FileListProps) => { const anchorRef = useRef(null) - const observerRef = useRef() + const observerRef = useRef(null) const dataSourceStore = useDataSourceStore() useEffect(() => { if (anchorRef.current) { observerRef.current = new IntersectionObserver((entries) => { - const { startAfter, isTruncated } = dataSourceStore.getState() - if (entries[0].isIntersecting && isTruncated.current && !isLoading) { - startAfter.current = fileList[fileList.length - 1].key - getOnlineDriveFiles({ startAfter: fileList[fileList.length - 1].key }) - } + const { setStartAfter, isTruncated } = dataSourceStore.getState() + if (entries[0].isIntersecting && isTruncated.current && !isLoading) + setStartAfter(fileList[fileList.length - 1].key) }, { rootMargin: '100px', }) observerRef.current.observe(anchorRef.current) } return () => observerRef.current?.disconnect() - // eslint-disable-next-line react-hooks/exhaustive-deps }, [anchorRef]) const isAllLoading = isLoading && fileList.length === 0 && keywords.length === 0 diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx index 7fe5f01471..e4ddad68ca 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx @@ -13,54 +13,55 @@ import { convertOnlineDriveData } from './utils' import produce from 'immer' import { useShallow } from 'zustand/react/shallow' import { useModalContextSelector } from '@/context/modal-context' -import { noop } from 'lodash-es' -import { CredentialTypeEnum } from '@/app/components/plugins/plugin-auth' +import { useGetDataSourceAuth } from '@/service/use-datasource' type OnlineDriveProps = { nodeId: string nodeData: DataSourceNodeType isInPipeline?: boolean + onCredentialChange: (credentialId: string) => void } const OnlineDrive = ({ nodeId, nodeData, isInPipeline = false, + onCredentialChange, }: OnlineDriveProps) => { const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal) const { + startAfter, prefix, keywords, bucket, selectedFileKeys, fileList, + currentCredentialId, } = useDataSourceStoreWithSelector(useShallow(state => ({ + startAfter: state.startAfter, prefix: state.prefix, keywords: state.keywords, bucket: state.bucket, selectedFileKeys: state.selectedFileKeys, fileList: state.fileList, + currentCredentialId: state.currentCredentialId, }))) const dataSourceStore = useDataSourceStore() const [isLoading, setIsLoading] = useState(false) + const { data: dataSourceAuth } = useGetDataSourceAuth({ + pluginId: nodeData.plugin_id, + provider: nodeData.provider_name, + }) + const datasourceNodeRunURL = !isInPipeline ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` - const getOnlineDriveFiles = useCallback(async (params: { - prefix?: string[] - bucket?: string - startAfter?: string - fileList?: OnlineDriveFile[] - }) => { - const { startAfter, prefix, bucket, fileList } = dataSourceStore.getState() - const _prefix = params.prefix ?? prefix - const _bucket = params.bucket ?? bucket - const _startAfter = params.startAfter ?? startAfter.current - const _fileList = params.fileList ?? fileList - const prefixString = _prefix.length > 0 ? `${_prefix.join('/')}/` : '' + const getOnlineDriveFiles = useCallback(async () => { + const { startAfter, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState() + const prefixString = prefix.length > 0 ? `${prefix.join('/')}/` : '' setIsLoading(true) ssePost( datasourceNodeRunURL, @@ -68,18 +69,19 @@ const OnlineDrive = ({ body: { inputs: { prefix: prefixString, - bucket: _bucket, - start_after: _startAfter, + bucket, + start_after: startAfter, max_keys: 30, // Adjust as needed }, datasource_type: DatasourceType.onlineDrive, + credential_id: currentCredentialId, }, }, { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { const { setFileList, isTruncated } = dataSourceStore.getState() - const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, _prefix, _bucket) - setFileList([..._fileList, ...newFileList]) + const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, prefix, bucket) + setFileList([...fileList, ...newFileList]) isTruncated.current = newIsTruncated setIsLoading(false) }, @@ -95,34 +97,8 @@ const OnlineDrive = ({ }, [datasourceNodeRunURL, dataSourceStore]) useEffect(() => { - const { - setFileList, - setBucket, - setPrefix, - setKeywords, - setSelectedFileKeys, - currentNodeIdRef, - } = dataSourceStore.getState() - if (nodeId !== currentNodeIdRef.current) { - setFileList([]) - setBucket('') - setPrefix([]) - setKeywords('') - setSelectedFileKeys([]) - currentNodeIdRef.current = nodeId - getOnlineDriveFiles({ - prefix: [], - bucket: '', - fileList: [], - startAfter: '', - }) - } - else { - // Avoid fetching files when come back from next step - if (fileList.length > 0) return - getOnlineDriveFiles({}) - } - }, [nodeId]) + getOnlineDriveFiles() + }, [startAfter, prefix, bucket, currentCredentialId]) const onlineDriveFileList = useMemo(() => { if (keywords) @@ -163,7 +139,6 @@ const OnlineDrive = ({ setFileList([]) if (file.type === OnlineDriveFileType.bucket) { setBucket(file.displayName) - getOnlineDriveFiles({ bucket: file.displayName, fileList: [] }) } else { setSelectedFileKeys([]) @@ -172,7 +147,6 @@ const OnlineDrive = ({ draft.push(displayName) }) setPrefix(newPrefix) - getOnlineDriveFiles({ prefix: newPrefix, fileList: [] }) } }, [dataSourceStore, getOnlineDriveFiles]) @@ -185,23 +159,13 @@ const OnlineDrive = ({ return (
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts index 2b5eb1271b..a68acadde1 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts @@ -6,8 +6,6 @@ export type OnlineDocumentSliceShape = { setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void searchValue: string setSearchValue: (searchValue: string) => void - currentWorkspaceId: string - setCurrentWorkspaceId: (workspaceId: string) => void onlineDocuments: NotionPage[] setOnlineDocuments: (documents: NotionPage[]) => void currentDocument: NotionPage | undefined @@ -27,10 +25,6 @@ export const createOnlineDocumentSlice: StateCreator = setSearchValue: (searchValue: string) => set(() => ({ searchValue, })), - currentWorkspaceId: '', - setCurrentWorkspaceId: (workspaceId: string) => set(() => ({ - currentWorkspaceId: workspaceId, - })), onlineDocuments: [], setOnlineDocuments: (documents: NotionPage[]) => { set(() => ({ diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts index 2c9834e941..1093d98c54 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts @@ -12,7 +12,8 @@ export type OnlineDriveSliceShape = { setFileList: (fileList: OnlineDriveFile[]) => void bucket: string setBucket: (bucket: string) => void - startAfter: React.RefObject + startAfter: string + setStartAfter: (startAfter: string) => void isTruncated: React.RefObject previewOnlineDriveFileRef: React.RefObject } @@ -44,7 +45,10 @@ export const createOnlineDriveSlice: StateCreator = (set, setBucket: (bucket: string) => set(() => ({ bucket, })), - startAfter: { current: '' }, + startAfter: '', + setStartAfter: (startAfter: string) => set(() => ({ + startAfter, + })), isTruncated: { current: false }, previewOnlineDriveFileRef: { current: undefined }, }) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/options/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/options/index.tsx index b6960fc1fe..56dab28815 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/options/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/base/options/index.tsx @@ -11,22 +11,21 @@ import Toast from '@/app/components/base/toast' import type { RAGPipelineVariables } from '@/models/pipeline' import { useConfigurations, useInitialData } from '@/app/components/rag-pipeline/hooks/use-input-fields' import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils' +import { CrawlStep } from '@/models/datasets' const I18N_PREFIX = 'datasetCreation.stepOne.website' type OptionsProps = { variables: RAGPipelineVariables - isRunning: boolean + step: CrawlStep runDisabled?: boolean - controlFoldOptions?: number onSubmit: (data: Record) => void } const Options = ({ variables, - isRunning, + step, runDisabled, - controlFoldOptions, onSubmit, }: OptionsProps) => { const { t } = useTranslation() @@ -62,12 +61,18 @@ const Options = ({ const [fold, { toggle: foldToggle, setTrue: foldHide, + setFalse: foldShow, }] = useBoolean(false) useEffect(() => { - if (controlFoldOptions !== 0) + // When the step change + if (step !== CrawlStep.init) foldHide() - }, [controlFoldOptions]) + else + foldShow() + }, [step]) + + const isRunning = useMemo(() => step === CrawlStep.running, [step]) return (
void } const WebsiteCrawl = ({ nodeId, nodeData, isInPipeline = false, + onCredentialChange, }: WebsiteCrawlProps) => { const { t } = useTranslation() - const [controlFoldOptions, setControlFoldOptions] = useState(0) const [totalNum, setTotalNum] = useState(0) const [crawledNum, setCrawledNum] = useState(0) const [crawlErrorMessage, setCrawlErrorMessage] = useState('') @@ -52,12 +52,20 @@ const WebsiteCrawl = ({ step, checkedCrawlResult, previewIndex, + currentCredentialId, } = useDataSourceStoreWithSelector(useShallow(state => ({ crawlResult: state.crawlResult, step: state.step, checkedCrawlResult: state.websitePages, previewIndex: state.previewIndex, + currentCredentialId: state.currentCredentialId, }))) + + const { data: dataSourceAuth } = useGetDataSourceAuth({ + pluginId: nodeData.plugin_id, + provider: nodeData.provider_name, + }) + const dataSourceStore = useDataSourceStore() const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams) @@ -66,33 +74,6 @@ const WebsiteCrawl = ({ node_id: nodeId, }, !!pipelineId && !!nodeId) - useEffect(() => { - if (step !== CrawlStep.init) - setControlFoldOptions(Date.now()) - }, [step]) - - useEffect(() => { - const { - setStep, - setCrawlResult, - setWebsitePages, - setPreviewIndex, - setCurrentWebsite, - currentNodeIdRef, - } = dataSourceStore.getState() - if (nodeId !== currentNodeIdRef.current) { - setStep(CrawlStep.init) - setCrawlResult(undefined) - setCurrentWebsite(undefined) - setWebsitePages([]) - setPreviewIndex(-1) - setCrawledNum(0) - setTotalNum(0) - setCrawlErrorMessage('') - currentNodeIdRef.current = nodeId - } - }, [nodeId]) - const isInit = step === CrawlStep.init const isCrawlFinished = step === CrawlStep.finished const isRunning = step === CrawlStep.running @@ -113,7 +94,7 @@ const WebsiteCrawl = ({ }, [dataSourceStore]) const handleRun = useCallback(async (value: Record) => { - const { setStep, setCrawlResult } = dataSourceStore.getState() + const { setStep, setCrawlResult, currentCredentialId } = dataSourceStore.getState() setStep(CrawlStep.running) ssePost( @@ -122,6 +103,7 @@ const WebsiteCrawl = ({ body: { inputs: value, datasource_type: DatasourceType.websiteCrawl, + credential_id: currentCredentialId, response_mode: 'streaming', }, }, @@ -165,33 +147,29 @@ const WebsiteCrawl = ({ }) }, [setShowAccountSettingModal]) + const handleCredentialChange = useCallback((credentialId: string) => { + setCrawledNum(0) + setTotalNum(0) + setCrawlErrorMessage('') + onCredentialChange(credentialId) + }, [dataSourceStore, onCredentialChange]) + return (
diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 498bb8acdd..6879de1982 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -87,21 +87,19 @@ export const useLocalFile = () => { } } -export const useOnlineDocuments = () => { +export const useOnlineDocument = () => { const { documentsData, - currentWorkspaceId, onlineDocuments, currentDocument, } = useDataSourceStoreWithSelector(useShallow(state => ({ documentsData: state.documentsData, - currentWorkspaceId: state.currentWorkspaceId, onlineDocuments: state.onlineDocuments, currentDocument: state.currentDocument, }))) const dataSourceStore = useDataSourceStore() - const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId) + const currentWorkspace = documentsData[0] const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => { const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 758aeb1908..8384c2d7c1 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -24,10 +24,15 @@ import WebsitePreview from './preview/web-preview' import ProcessDocuments from './process-documents' import ChunkPreview from './preview/chunk-preview' import Processing from './processing' -import type { InitialDocumentDetail, OnlineDriveFile, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline' +import type { + InitialDocumentDetail, + OnlineDriveFile, + PublishedPipelineRunPreviewResponse, + PublishedPipelineRunResponse, +} from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' -import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks' +import { useAddDocumentsSteps, useLocalFile, useOnlineDocument, useOnlineDrive, useWebsiteCrawl } from './hooks' import DataSourceProvider from './data-source/store/provider' import { useDataSourceStore } from './data-source/store' import { useFileUploadConfig } from '@/service/use-common' @@ -67,7 +72,7 @@ const CreateFormPipeline = () => { currentDocument, PagesMapAndSelectedPagesId, hidePreviewOnlineDocument, - } = useOnlineDocuments() + } = useOnlineDocument() const { websitePages, currentWebsite, diff --git a/web/app/components/datasets/list/datasets.tsx b/web/app/components/datasets/list/datasets.tsx index c47d08d75e..5602730296 100644 --- a/web/app/components/datasets/list/datasets.tsx +++ b/web/app/components/datasets/list/datasets.tsx @@ -34,7 +34,7 @@ const Datasets = ({ }) const resetDatasetList = useResetDatasetList() const anchorRef = useRef(null) - const observerRef = useRef() + const observerRef = useRef(null) useEffect(() => { document.title = `${t('dataset.knowledge')} - Dify` @@ -51,7 +51,6 @@ const Datasets = ({ observerRef.current.observe(anchorRef.current) } return () => observerRef.current?.disconnect() - // eslint-disable-next-line react-hooks/exhaustive-deps }, [anchorRef]) return ( diff --git a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts index 59de068304..bc5f4819bc 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts @@ -5,6 +5,8 @@ import { useNodes } from 'reactflow' import { BlockEnum } from '@/app/components/workflow/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' import { useCallback, useMemo, useState } from 'react' +import { useDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store' +import { CrawlStep } from '@/models/datasets' export const useTestRunSteps = () => { const { t } = useTranslation() @@ -56,3 +58,72 @@ export const useDatasourceOptions = () => { return options } + +export const useOnlineDocument = () => { + const dataSourceStore = useDataSourceStore() + + const clearOnlineDocumentData = useCallback(() => { + const { + setDocumentsData, + setSearchValue, + setSelectedPagesId, + setOnlineDocuments, + setCurrentDocument, + } = dataSourceStore.getState() + setDocumentsData([]) + setSearchValue('') + setSelectedPagesId(new Set()) + setOnlineDocuments([]) + setCurrentDocument(undefined) + }, [dataSourceStore]) + + return { + clearOnlineDocumentData, + } +} + +export const useWebsiteCrawl = () => { + const dataSourceStore = useDataSourceStore() + + const clearWebsiteCrawlData = useCallback(() => { + const { + setStep, + setCrawlResult, + setWebsitePages, + setPreviewIndex, + setCurrentWebsite, + } = dataSourceStore.getState() + setStep(CrawlStep.init) + setCrawlResult(undefined) + setCurrentWebsite(undefined) + setWebsitePages([]) + setPreviewIndex(-1) + }, [dataSourceStore]) + + return { + clearWebsiteCrawlData, + } +} + +export const useOnlineDrive = () => { + const dataSourceStore = useDataSourceStore() + + const clearOnlineDriveData = useCallback(() => { + const { + setFileList, + setBucket, + setPrefix, + setKeywords, + setSelectedFileKeys, + } = dataSourceStore.getState() + setFileList([]) + setBucket('') + setPrefix([]) + setKeywords('') + setSelectedFileKeys([]) + }, [dataSourceStore]) + + return { + clearOnlineDriveData, + } +} diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index be333b2f48..8a06781271 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -1,6 +1,11 @@ import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store' import { useCallback, useMemo, useState } from 'react' -import { useTestRunSteps } from './hooks' +import { + useOnlineDocument, + useOnlineDrive, + useTestRunSteps, + useWebsiteCrawl, +} from './hooks' import DataSourceOptions from './data-source-options' import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file' import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents' @@ -42,6 +47,10 @@ const TestRunPanel = () => { handleBackStep, } = useTestRunSteps() + const { clearOnlineDocumentData } = useOnlineDocument() + const { clearWebsiteCrawlData } = useWebsiteCrawl() + const { clearOnlineDriveData } = useOnlineDrive() + const datasourceType = datasource?.nodeData.provider_type const nextBtnDisabled = useMemo(() => { @@ -67,6 +76,7 @@ const TestRunPanel = () => { if (!datasource) return const datasourceInfoList: Record[] = [] + const credentialId = dataSourceStore.getState().currentCredentialId if (datasourceType === DatasourceType.localFile) { const { id, name, type, size, extension, mime_type } = fileList[0].file const documentInfo = { @@ -86,16 +96,22 @@ const TestRunPanel = () => { const documentInfo = { workspace_id, page: rest, + credential_id: credentialId, } datasourceInfoList.push(documentInfo) } - if (datasourceType === DatasourceType.websiteCrawl) - datasourceInfoList.push(websitePages[0]) + if (datasourceType === DatasourceType.websiteCrawl) { + datasourceInfoList.push({ + ...websitePages[0], + credential_id: credentialId, + }) + } if (datasourceType === DatasourceType.onlineDrive) { const { bucket } = dataSourceStore.getState() datasourceInfoList.push({ bucket, key: selectedFileKeys[0], + credential_id: credentialId, }) } handleRun({ @@ -106,6 +122,32 @@ const TestRunPanel = () => { }) }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileKeys, websitePages]) + const clearDataSourceData = useCallback((dataSource: Datasource) => { + if (dataSource.nodeData.provider_type === DatasourceType.onlineDocument) + clearOnlineDocumentData() + else if (dataSource.nodeData.provider_type === DatasourceType.websiteCrawl) + clearWebsiteCrawlData() + else if (dataSource.nodeData.provider_type === DatasourceType.onlineDrive) + clearOnlineDriveData() + }, []) + + const handleSwitchDataSource = useCallback((dataSource: Datasource) => { + const { + setCurrentCredentialId, + currentNodeIdRef, + } = dataSourceStore.getState() + clearDataSourceData(dataSource) + setCurrentCredentialId('') + currentNodeIdRef.current = dataSource.nodeId + setDatasource(dataSource) + }, [dataSourceStore]) + + const handleCredentialChange = useCallback((credentialId: string) => { + const { setCurrentCredentialId } = dataSourceStore.getState() + clearDataSourceData(datasource!) + setCurrentCredentialId(credentialId) + }, [dataSourceStore, datasource]) + return (
{
{datasourceType === DatasourceType.localFile && ( { nodeId={datasource!.nodeId} nodeData={datasource!.nodeData} isInPipeline + onCredentialChange={handleCredentialChange} /> )} {datasourceType === DatasourceType.websiteCrawl && ( @@ -139,6 +182,7 @@ const TestRunPanel = () => { nodeId={datasource!.nodeId} nodeData={datasource!.nodeData} isInPipeline + onCredentialChange={handleCredentialChange} /> )} {datasourceType === DatasourceType.onlineDrive && ( @@ -146,6 +190,7 @@ const TestRunPanel = () => { nodeId={datasource!.nodeId} nodeData={datasource!.nodeData} isInPipeline + onCredentialChange={handleCredentialChange} /> )}
diff --git a/web/app/components/workflow/hooks/use-shortcuts.ts b/web/app/components/workflow/hooks/use-shortcuts.ts index def4eef9ce..b2d71555d7 100644 --- a/web/app/components/workflow/hooks/use-shortcuts.ts +++ b/web/app/components/workflow/hooks/use-shortcuts.ts @@ -218,7 +218,6 @@ export const useShortcuts = (): void => { useKeyPress( 'shift', (e) => { - console.log('Shift down', e) if (shouldHandleShortcut(e)) dimOtherNodes() },