diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx index cb88aec1da..f99b0bcf32 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx @@ -32,7 +32,6 @@ const DataSourceOptions = ({ useEffect(() => { if (options.length > 0 && !datasourceNodeId) handelSelect(options[0].value) - // eslint-disable-next-line react-hooks/exhaustive-deps }, []) return ( diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx index 09ea38078c..144decada5 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx @@ -32,7 +32,7 @@ const LocalFile = ({ const { t } = useTranslation() const { notify } = useContext(ToastContext) const { locale } = useContext(I18n) - const fileList = useDataSourceStoreWithSelector(state => state.localFileList) + const localFileList = useDataSourceStoreWithSelector(state => state.localFileList) const dataSourceStore = useDataSourceStore() const [dragging, setDragging] = useState(false) @@ -41,7 +41,7 @@ const LocalFile = ({ const fileUploader = useRef(null) const fileListRef = useRef([]) - const hideUpload = notSupportBatchUpload && fileList.length > 0 + const hideUpload = notSupportBatchUpload && localFileList.length > 0 const { data: fileUploadConfigResponse } = useFileUploadConfig() const supportTypesShowNames = useMemo(() => { @@ -179,7 +179,7 @@ const LocalFile = ({ if (!files.length) return false - if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) { + if (files.length + localFileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) { notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) }) return false } @@ -193,7 +193,7 @@ const LocalFile = ({ updateFileList(newFiles) fileListRef.current = newFiles uploadMultipleFiles(preparedFiles) - }, [updateFileList, uploadMultipleFiles, notify, t, fileList]) + }, [updateFileList, uploadMultipleFiles, notify, t, localFileList]) const handleDragEnter = (e: DragEvent) => { e.preventDefault() @@ -297,9 +297,9 @@ const LocalFile = ({ {dragging &&
}
)} - {fileList.length > 0 && ( + {localFileList.length > 0 && (
- {fileList.map((fileItem, index) => { + {localFileList.map((fileItem, index) => { const isUploading = fileItem.progress >= 0 && fileItem.progress < 100 const isError = fileItem.progress === -2 return ( diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx index faf8e09c12..b0d8e8d70b 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx @@ -17,6 +17,7 @@ const Menu = ({ {breadcrumbs.map((breadcrumb, index) => { return ( { - const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } = dataSourceStore.getState() - setFileList([]) + const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } = dataSourceStore.getState() + setOnlineDriveFileList([]) setSelectedFileIds([]) setBucket('') setBreadcrumbs([]) @@ -54,26 +54,26 @@ const Breadcrumbs = ({ }, [dataSourceStore]) const handleClickBucketName = useCallback(() => { - const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState() - setFileList([]) + const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState() + setOnlineDriveFileList([]) setSelectedFileIds([]) setBreadcrumbs([]) setPrefix([]) }, [dataSourceStore]) const handleBackToRoot = useCallback(() => { - const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState() - setFileList([]) + const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState() + setOnlineDriveFileList([]) setSelectedFileIds([]) setBreadcrumbs([]) setPrefix([]) }, [dataSourceStore]) const handleClickBreadcrumb = useCallback((index: number) => { - const { breadcrumbs, prefix, setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState() + const { breadcrumbs, prefix, setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState() const newBreadcrumbs = breadcrumbs.slice(0, index + 1) const newPrefix = prefix.slice(0, index + 1) - setFileList([]) + setOnlineDriveFileList([]) setSelectedFileIds([]) setBreadcrumbs(newBreadcrumbs) setPrefix(newPrefix) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx index e048d87dc0..75656162df 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx @@ -1,6 +1,6 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' import Header from '../base/header' -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import FileList from './file-list' import type { OnlineDriveFile } from '@/models/pipeline' import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline' @@ -38,7 +38,7 @@ const OnlineDrive = ({ keywords, bucket, selectedFileIds, - fileList, + onlineDriveFileList, currentCredentialId, } = useDataSourceStoreWithSelector(useShallow(state => ({ nextPageParameters: state.nextPageParameters, @@ -47,11 +47,12 @@ const OnlineDrive = ({ keywords: state.keywords, bucket: state.bucket, selectedFileIds: state.selectedFileIds, - fileList: state.fileList, + onlineDriveFileList: state.onlineDriveFileList, currentCredentialId: state.currentCredentialId, }))) const dataSourceStore = useDataSourceStore() const [isLoading, setIsLoading] = useState(false) + const isLoadingRef = useRef(false) const { data: dataSourceAuth } = useGetDataSourceAuth({ pluginId: nodeData.plugin_id, @@ -63,8 +64,10 @@ const OnlineDrive = ({ : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` const getOnlineDriveFiles = useCallback(async () => { - const { nextPageParameters, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState() + if (isLoadingRef.current) return + const { nextPageParameters, prefix, bucket, onlineDriveFileList, currentCredentialId } = dataSourceStore.getState() setIsLoading(true) + isLoadingRef.current = true ssePost( datasourceNodeRunURL, { @@ -81,18 +84,19 @@ const OnlineDrive = ({ }, { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { - const { setFileList, isTruncated, currentNextPageParametersRef, setHasBucket } = dataSourceStore.getState() + const { setOnlineDriveFileList, isTruncated, currentNextPageParametersRef, setHasBucket } = dataSourceStore.getState() const { fileList: newFileList, isTruncated: newIsTruncated, nextPageParameters: newNextPageParameters, hasBucket: newHasBucket, } = convertOnlineDriveData(documentsData.data, breadcrumbs, bucket) - setFileList([...fileList, ...newFileList]) + setOnlineDriveFileList([...onlineDriveFileList, ...newFileList]) isTruncated.current = newIsTruncated currentNextPageParametersRef.current = newNextPageParameters setHasBucket(newHasBucket) setIsLoading(false) + isLoadingRef.current = false }, onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => { Toast.notify({ @@ -100,6 +104,7 @@ const OnlineDrive = ({ message: error.error, }) setIsLoading(false) + isLoadingRef.current = false }, }, ) @@ -109,7 +114,7 @@ const OnlineDrive = ({ if (!currentCredentialId) return if (isInitialMount) { // Only fetch files on initial mount if fileList is empty - if (fileList.length === 0) + if (onlineDriveFileList.length === 0) getOnlineDriveFiles() setIsInitialMount(false) } @@ -118,11 +123,11 @@ const OnlineDrive = ({ } }, [nextPageParameters, prefix, bucket, currentCredentialId]) - const onlineDriveFileList = useMemo(() => { + const filteredOnlineDriveFileList = useMemo(() => { if (keywords) - return fileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase())) - return fileList - }, [fileList, keywords]) + return onlineDriveFileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase())) + return onlineDriveFileList + }, [onlineDriveFileList, keywords]) const updateKeywords = useCallback((keywords: string) => { const { setKeywords } = dataSourceStore.getState() @@ -152,9 +157,9 @@ const OnlineDrive = ({ }, [dataSourceStore, isInPipeline]) const handleOpenFolder = useCallback((file: OnlineDriveFile) => { - const { breadcrumbs, setBreadcrumbs, setPrefix, setBucket, setFileList, setSelectedFileIds } = dataSourceStore.getState() + const { breadcrumbs, prefix, setBreadcrumbs, setPrefix, setBucket, setOnlineDriveFileList, setSelectedFileIds } = dataSourceStore.getState() if (file.type === OnlineDriveFileType.file) return - setFileList([]) + setOnlineDriveFileList([]) if (file.type === OnlineDriveFileType.bucket) { setBucket(file.name) } @@ -189,14 +194,14 @@ const OnlineDrive = ({ credentials={dataSourceAuth?.result || []} /> void selectedFileIds: string[] setSelectedFileIds: (selectedFileIds: string[]) => void - fileList: OnlineDriveFile[] - setFileList: (fileList: OnlineDriveFile[]) => void + onlineDriveFileList: OnlineDriveFile[] + setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => void bucket: string setBucket: (bucket: string) => void nextPageParameters: Record @@ -43,12 +43,12 @@ export const createOnlineDriveSlice: StateCreator = (set, selectedFileIds, })) const id = selectedFileIds[0] - const { fileList, previewOnlineDriveFileRef } = get() - previewOnlineDriveFileRef.current = fileList.find(file => file.id === id) + const { onlineDriveFileList, previewOnlineDriveFileRef } = get() + previewOnlineDriveFileRef.current = onlineDriveFileList.find(file => file.id === id) }, - fileList: [], - setFileList: (fileList: OnlineDriveFile[]) => set(() => ({ - fileList, + onlineDriveFileList: [], + setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => set(() => ({ + onlineDriveFileList, })), bucket: '', setBucket: (bucket: string) => set(() => ({ diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index fb24ce5925..45dcaa6111 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -65,7 +65,7 @@ export const useDatasourceOptions = (pipelineNodes: Node[]) export const useLocalFile = () => { const { - localFileList: fileList, + localFileList, currentLocalFile, } = useDataSourceStoreWithSelector(useShallow(state => ({ localFileList: state.localFileList, @@ -73,7 +73,7 @@ export const useLocalFile = () => { }))) const dataSourceStore = useDataSourceStore() - const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList]) + const allFileLoaded = useMemo(() => (localFileList.length > 0 && localFileList.every(file => file.file.id)), [localFileList]) const hidePreviewLocalFile = useCallback(() => { const { setCurrentLocalFile } = dataSourceStore.getState() @@ -81,7 +81,7 @@ export const useLocalFile = () => { }, [dataSourceStore]) return { - fileList, + localFileList, allFileLoaded, currentLocalFile, hidePreviewLocalFile, @@ -187,27 +187,27 @@ export const useWebsiteCrawl = () => { export const useOnlineDrive = () => { const { - fileList, + onlineDriveFileList, selectedFileIds, } = useDataSourceStoreWithSelector(useShallow(state => ({ - fileList: state.fileList, + onlineDriveFileList: state.onlineDriveFileList, selectedFileIds: state.selectedFileIds, }))) const dataSourceStore = useDataSourceStore() const selectedOnlineDriveFileList = useMemo(() => { - return selectedFileIds.map(key => fileList.find(item => item.id === key)!) - }, [fileList, selectedFileIds]) + return selectedFileIds.map(key => onlineDriveFileList.find(item => item.id === key)!) + }, [onlineDriveFileList, selectedFileIds]) const clearOnlineDriveData = useCallback(() => { const { - setFileList, + setOnlineDriveFileList, setBucket, setPrefix, setKeywords, setSelectedFileIds, } = dataSourceStore.getState() - setFileList([]) + setOnlineDriveFileList([]) setBucket('') setPrefix([]) setKeywords('') @@ -215,7 +215,7 @@ export const useOnlineDrive = () => { }, [dataSourceStore]) return { - fileList, + onlineDriveFileList, selectedFileIds, selectedOnlineDriveFileList, clearOnlineDriveData, diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index b679051cf3..c9bb641287 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -61,7 +61,7 @@ const CreateFormPipeline = () => { handleBackStep, } = useAddDocumentsSteps() const { - fileList, + localFileList, allFileLoaded, currentLocalFile, hidePreviewLocalFile, @@ -81,7 +81,7 @@ const CreateFormPipeline = () => { clearWebsiteCrawlData, } = useWebsiteCrawl() const { - fileList: onlineDriveFileList, + onlineDriveFileList, selectedFileIds, selectedOnlineDriveFileList, clearOnlineDriveData, @@ -107,7 +107,7 @@ const CreateFormPipeline = () => { const nextBtnDisabled = useMemo(() => { if (!datasource) return true if (datasourceType === DatasourceType.localFile) - return isShowVectorSpaceFull || !fileList.length || !allFileLoaded + return isShowVectorSpaceFull || !localFileList.length || !allFileLoaded if (datasourceType === DatasourceType.onlineDocument) return isShowVectorSpaceFull || !onlineDocuments.length if (datasourceType === DatasourceType.websiteCrawl) @@ -115,7 +115,7 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.onlineDrive) return isShowVectorSpaceFull || !selectedFileIds.length return false - }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length]) + }, [datasource, datasourceType, isShowVectorSpaceFull, localFileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length]) const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { file_size_limit: 15, @@ -285,7 +285,7 @@ const CreateFormPipeline = () => { const { bucket, selectedFileIds, - fileList: onlineDriveFileList, + onlineDriveFileList, } = dataSourceStore.getState() selectedFileIds.forEach((id) => { const file = onlineDriveFileList.find(file => file.id === id) @@ -354,7 +354,7 @@ const CreateFormPipeline = () => { const handleSelectAll = useCallback(() => { const { onlineDocuments, - fileList: onlineDriveFileList, + onlineDriveFileList, selectedFileIds, setOnlineDocuments, setSelectedFileIds, @@ -535,7 +535,7 @@ const CreateFormPipeline = () => {
file.file)} + localFiles={localFileList.map(file => file.file)} onlineDocuments={onlineDocuments} websitePages={websitePages} onlineDriveFiles={selectedOnlineDriveFileList} diff --git a/web/app/components/rag-pipeline/components/panel/test-run/preparation/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/preparation/hooks.ts index 64f05c0cd5..3f2f49f455 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/preparation/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/preparation/hooks.ts @@ -110,13 +110,13 @@ export const useOnlineDrive = () => { const clearOnlineDriveData = useCallback(() => { const { - setFileList, + setOnlineDriveFileList, setBucket, setPrefix, setKeywords, setSelectedFileIds, } = dataSourceStore.getState() - setFileList([]) + setOnlineDriveFileList([]) setBucket('') setPrefix([]) setKeywords('') diff --git a/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx index b4dc32c122..0f869c2621 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx @@ -104,8 +104,8 @@ const Preparation = () => { }) } if (datasourceType === DatasourceType.onlineDrive) { - const { bucket, fileList, selectedFileIds } = dataSourceStore.getState() - const file = fileList.find(file => file.id === selectedFileIds[0]) + const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState() + const file = onlineDriveFileList.find(file => file.id === selectedFileIds[0]) datasourceInfoList.push({ bucket, id: file?.id, diff --git a/web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts b/web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts index 36540d6217..7b0a225e45 100644 --- a/web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts +++ b/web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts @@ -110,8 +110,8 @@ const useBeforeRunForm = ({ } } if (datasourceType === DatasourceType.onlineDrive) { - const { bucket, fileList, selectedFileIds } = dataSourceStore.getState() - const file = fileList.find(file => file.id === selectedFileIds[0]) + const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState() + const file = onlineDriveFileList.find(file => file.id === selectedFileIds[0]) datasourceInfo = { bucket, id: file?.id,