refactor: update local file and online drive state management in create-from-pipeline components

web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx

@@ -32,7 +32,6 @@ const DataSourceOptions = ({
   useEffect(() => {
     if (options.length > 0 && !datasourceNodeId)
       handelSelect(options[0].value)
-    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
   return (

web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx

@@ -32,7 +32,7 @@ const LocalFile = ({
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
   const { locale } = useContext(I18n)
-  const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
+  const localFileList = useDataSourceStoreWithSelector(state => state.localFileList)
   const dataSourceStore = useDataSourceStore()
   const [dragging, setDragging] = useState(false)
 
@@ -41,7 +41,7 @@ const LocalFile = ({
   const fileUploader = useRef<HTMLInputElement>(null)
   const fileListRef = useRef<FileItem[]>([])
 
-  const hideUpload = notSupportBatchUpload && fileList.length > 0
+  const hideUpload = notSupportBatchUpload && localFileList.length > 0
 
   const { data: fileUploadConfigResponse } = useFileUploadConfig()
   const supportTypesShowNames = useMemo(() => {
@@ -179,7 +179,7 @@ const LocalFile = ({
     if (!files.length)
       return false
 
-    if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
+    if (files.length + localFileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
       notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
       return false
     }
@@ -193,7 +193,7 @@ const LocalFile = ({
     updateFileList(newFiles)
     fileListRef.current = newFiles
     uploadMultipleFiles(preparedFiles)
-  }, [updateFileList, uploadMultipleFiles, notify, t, fileList])
+  }, [updateFileList, uploadMultipleFiles, notify, t, localFileList])
 
   const handleDragEnter = (e: DragEvent) => {
     e.preventDefault()
@@ -297,9 +297,9 @@ const LocalFile = ({
           {dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
         </div>
       )}
-      {fileList.length > 0 && (
+      {localFileList.length > 0 && (
         <div className='mt-1 flex flex-col gap-y-1'>
-          {fileList.map((fileItem, index) => {
+          {localFileList.map((fileItem, index) => {
             const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
             const isError = fileItem.progress === -2
             return (

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx

@@ -17,6 +17,7 @@ const Menu = ({
       {breadcrumbs.map((breadcrumb, index) => {
         return (
           <Item
+            key={`${breadcrumb}-${index}`}
             name={breadcrumb}
             index={startIndex + index}
             onBreadcrumbClick={onBreadcrumbClick}

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx

@@ -45,8 +45,8 @@ const Breadcrumbs = ({
   }, [displayBreadcrumbNum, breadcrumbs])
 
   const handleBackToBucketList = useCallback(() => {
-    const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } = dataSourceStore.getState()
+    const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } = dataSourceStore.getState()
-    setFileList([])
+    setOnlineDriveFileList([])
     setSelectedFileIds([])
     setBucket('')
     setBreadcrumbs([])
@@ -54,26 +54,26 @@ const Breadcrumbs = ({
   }, [dataSourceStore])
 
   const handleClickBucketName = useCallback(() => {
-    const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
+    const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
-    setFileList([])
+    setOnlineDriveFileList([])
     setSelectedFileIds([])
     setBreadcrumbs([])
     setPrefix([])
   }, [dataSourceStore])
 
   const handleBackToRoot = useCallback(() => {
-    const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
+    const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
-    setFileList([])
+    setOnlineDriveFileList([])
     setSelectedFileIds([])
     setBreadcrumbs([])
     setPrefix([])
   }, [dataSourceStore])
 
   const handleClickBreadcrumb = useCallback((index: number) => {
-    const { breadcrumbs, prefix, setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
+    const { breadcrumbs, prefix, setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
     const newBreadcrumbs = breadcrumbs.slice(0, index + 1)
     const newPrefix = prefix.slice(0, index + 1)
-    setFileList([])
+    setOnlineDriveFileList([])
     setSelectedFileIds([])
     setBreadcrumbs(newBreadcrumbs)
     setPrefix(newPrefix)

web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx

@@ -1,6 +1,6 @@
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 import Header from '../base/header'
-import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import FileList from './file-list'
 import type { OnlineDriveFile } from '@/models/pipeline'
 import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
@@ -38,7 +38,7 @@ const OnlineDrive = ({
     keywords,
     bucket,
     selectedFileIds,
-    fileList,
+    onlineDriveFileList,
     currentCredentialId,
   } = useDataSourceStoreWithSelector(useShallow(state => ({
     nextPageParameters: state.nextPageParameters,
@@ -47,11 +47,12 @@ const OnlineDrive = ({
     keywords: state.keywords,
     bucket: state.bucket,
     selectedFileIds: state.selectedFileIds,
-    fileList: state.fileList,
+    onlineDriveFileList: state.onlineDriveFileList,
     currentCredentialId: state.currentCredentialId,
   })))
   const dataSourceStore = useDataSourceStore()
   const [isLoading, setIsLoading] = useState(false)
+  const isLoadingRef = useRef(false)
 
   const { data: dataSourceAuth } = useGetDataSourceAuth({
     pluginId: nodeData.plugin_id,
@@ -63,8 +64,10 @@ const OnlineDrive = ({
     : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
 
   const getOnlineDriveFiles = useCallback(async () => {
-    const { nextPageParameters, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState()
+    if (isLoadingRef.current) return
+    const { nextPageParameters, prefix, bucket, onlineDriveFileList, currentCredentialId } = dataSourceStore.getState()
     setIsLoading(true)
+    isLoadingRef.current = true
     ssePost(
       datasourceNodeRunURL,
       {
@@ -81,18 +84,19 @@ const OnlineDrive = ({
       },
       {
         onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
-          const { setFileList, isTruncated, currentNextPageParametersRef, setHasBucket } = dataSourceStore.getState()
+          const { setOnlineDriveFileList, isTruncated, currentNextPageParametersRef, setHasBucket } = dataSourceStore.getState()
           const {
             fileList: newFileList,
             isTruncated: newIsTruncated,
             nextPageParameters: newNextPageParameters,
             hasBucket: newHasBucket,
           } = convertOnlineDriveData(documentsData.data, breadcrumbs, bucket)
-          setFileList([...fileList, ...newFileList])
+          setOnlineDriveFileList([...onlineDriveFileList, ...newFileList])
           isTruncated.current = newIsTruncated
           currentNextPageParametersRef.current = newNextPageParameters
           setHasBucket(newHasBucket)
           setIsLoading(false)
+          isLoadingRef.current = false
         },
         onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
           Toast.notify({
@@ -100,6 +104,7 @@ const OnlineDrive = ({
             message: error.error,
           })
           setIsLoading(false)
+          isLoadingRef.current = false
         },
       },
     )
@@ -109,7 +114,7 @@ const OnlineDrive = ({
     if (!currentCredentialId) return
     if (isInitialMount) {
       // Only fetch files on initial mount if fileList is empty
-      if (fileList.length === 0)
+      if (onlineDriveFileList.length === 0)
         getOnlineDriveFiles()
       setIsInitialMount(false)
     }
@@ -118,11 +123,11 @@ const OnlineDrive = ({
     }
   }, [nextPageParameters, prefix, bucket, currentCredentialId])
 
-  const onlineDriveFileList = useMemo(() => {
+  const filteredOnlineDriveFileList = useMemo(() => {
     if (keywords)
-      return fileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase()))
+      return onlineDriveFileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase()))
-    return fileList
+    return onlineDriveFileList
-  }, [fileList, keywords])
+  }, [onlineDriveFileList, keywords])
 
   const updateKeywords = useCallback((keywords: string) => {
     const { setKeywords } = dataSourceStore.getState()
@@ -152,9 +157,9 @@ const OnlineDrive = ({
   }, [dataSourceStore, isInPipeline])
 
   const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
-    const { breadcrumbs, setBreadcrumbs, setPrefix, setBucket, setFileList, setSelectedFileIds } = dataSourceStore.getState()
+    const { breadcrumbs, prefix, setBreadcrumbs, setPrefix, setBucket, setOnlineDriveFileList, setSelectedFileIds } = dataSourceStore.getState()
     if (file.type === OnlineDriveFileType.file) return
-    setFileList([])
+    setOnlineDriveFileList([])
     if (file.type === OnlineDriveFileType.bucket) {
       setBucket(file.name)
     }
@@ -189,14 +194,14 @@ const OnlineDrive = ({
         credentials={dataSourceAuth?.result || []}
       />
       <FileList
-        fileList={onlineDriveFileList}
+        fileList={filteredOnlineDriveFileList}
         selectedFileIds={selectedFileIds}
         breadcrumbs={breadcrumbs}
         keywords={keywords}
         bucket={bucket}
         resetKeywords={resetKeywords}
         updateKeywords={updateKeywords}
-        searchResultsLength={onlineDriveFileList.length}
+        searchResultsLength={filteredOnlineDriveFileList.length}
         handleSelectFile={handleSelectFile}
         handleOpenFolder={handleOpenFolder}
         isInPipeline={isInPipeline}

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts

@@ -10,8 +10,8 @@ export type OnlineDriveSliceShape = {
   setKeywords: (keywords: string) => void
   selectedFileIds: string[]
   setSelectedFileIds: (selectedFileIds: string[]) => void
-  fileList: OnlineDriveFile[]
+  onlineDriveFileList: OnlineDriveFile[]
-  setFileList: (fileList: OnlineDriveFile[]) => void
+  setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => void
   bucket: string
   setBucket: (bucket: string) => void
   nextPageParameters: Record<string, any>
@@ -43,12 +43,12 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set,
         selectedFileIds,
       }))
       const id = selectedFileIds[0]
-      const { fileList, previewOnlineDriveFileRef } = get()
+      const { onlineDriveFileList, previewOnlineDriveFileRef } = get()
-      previewOnlineDriveFileRef.current = fileList.find(file => file.id === id)
+      previewOnlineDriveFileRef.current = onlineDriveFileList.find(file => file.id === id)
     },
-    fileList: [],
+    onlineDriveFileList: [],
-    setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
+    setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => set(() => ({
-      fileList,
+      onlineDriveFileList,
     })),
     bucket: '',
     setBucket: (bucket: string) => set(() => ({

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -65,7 +65,7 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
 
 export const useLocalFile = () => {
   const {
-    localFileList: fileList,
+    localFileList,
     currentLocalFile,
   } = useDataSourceStoreWithSelector(useShallow(state => ({
     localFileList: state.localFileList,
@@ -73,7 +73,7 @@ export const useLocalFile = () => {
   })))
   const dataSourceStore = useDataSourceStore()
 
-  const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
+  const allFileLoaded = useMemo(() => (localFileList.length > 0 && localFileList.every(file => file.file.id)), [localFileList])
 
   const hidePreviewLocalFile = useCallback(() => {
     const { setCurrentLocalFile } = dataSourceStore.getState()
@@ -81,7 +81,7 @@ export const useLocalFile = () => {
   }, [dataSourceStore])
 
   return {
-    fileList,
+    localFileList,
     allFileLoaded,
     currentLocalFile,
     hidePreviewLocalFile,
@@ -187,27 +187,27 @@ export const useWebsiteCrawl = () => {
 
 export const useOnlineDrive = () => {
   const {
-    fileList,
+    onlineDriveFileList,
     selectedFileIds,
   } = useDataSourceStoreWithSelector(useShallow(state => ({
-    fileList: state.fileList,
+    onlineDriveFileList: state.onlineDriveFileList,
     selectedFileIds: state.selectedFileIds,
   })))
   const dataSourceStore = useDataSourceStore()
 
   const selectedOnlineDriveFileList = useMemo(() => {
-    return selectedFileIds.map(key => fileList.find(item => item.id === key)!)
+    return selectedFileIds.map(key => onlineDriveFileList.find(item => item.id === key)!)
-  }, [fileList, selectedFileIds])
+  }, [onlineDriveFileList, selectedFileIds])
 
   const clearOnlineDriveData = useCallback(() => {
     const {
-      setFileList,
+      setOnlineDriveFileList,
       setBucket,
       setPrefix,
       setKeywords,
       setSelectedFileIds,
     } = dataSourceStore.getState()
-    setFileList([])
+    setOnlineDriveFileList([])
     setBucket('')
     setPrefix([])
     setKeywords('')
@@ -215,7 +215,7 @@ export const useOnlineDrive = () => {
   }, [dataSourceStore])
 
   return {
-    fileList,
+    onlineDriveFileList,
     selectedFileIds,
     selectedOnlineDriveFileList,
     clearOnlineDriveData,

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -61,7 +61,7 @@ const CreateFormPipeline = () => {
     handleBackStep,
   } = useAddDocumentsSteps()
   const {
-    fileList,
+    localFileList,
     allFileLoaded,
     currentLocalFile,
     hidePreviewLocalFile,
@@ -81,7 +81,7 @@ const CreateFormPipeline = () => {
     clearWebsiteCrawlData,
   } = useWebsiteCrawl()
   const {
-    fileList: onlineDriveFileList,
+    onlineDriveFileList,
     selectedFileIds,
     selectedOnlineDriveFileList,
     clearOnlineDriveData,
@@ -107,7 +107,7 @@ const CreateFormPipeline = () => {
   const nextBtnDisabled = useMemo(() => {
     if (!datasource) return true
     if (datasourceType === DatasourceType.localFile)
-      return isShowVectorSpaceFull || !fileList.length || !allFileLoaded
+      return isShowVectorSpaceFull || !localFileList.length || !allFileLoaded
     if (datasourceType === DatasourceType.onlineDocument)
       return isShowVectorSpaceFull || !onlineDocuments.length
     if (datasourceType === DatasourceType.websiteCrawl)
@@ -115,7 +115,7 @@ const CreateFormPipeline = () => {
     if (datasourceType === DatasourceType.onlineDrive)
       return isShowVectorSpaceFull || !selectedFileIds.length
     return false
-  }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length])
+  }, [datasource, datasourceType, isShowVectorSpaceFull, localFileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length])
 
   const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
     file_size_limit: 15,
@@ -285,7 +285,7 @@ const CreateFormPipeline = () => {
       const {
         bucket,
         selectedFileIds,
-        fileList: onlineDriveFileList,
+        onlineDriveFileList,
       } = dataSourceStore.getState()
       selectedFileIds.forEach((id) => {
         const file = onlineDriveFileList.find(file => file.id === id)
@@ -354,7 +354,7 @@ const CreateFormPipeline = () => {
   const handleSelectAll = useCallback(() => {
     const {
       onlineDocuments,
-      fileList: onlineDriveFileList,
+      onlineDriveFileList,
       selectedFileIds,
       setOnlineDocuments,
       setSelectedFileIds,
@@ -535,7 +535,7 @@ const CreateFormPipeline = () => {
             <div className='flex h-full flex-col pl-2 pt-2'>
               <ChunkPreview
                 dataSourceType={datasourceType as DatasourceType}
-                localFiles={fileList.map(file => file.file)}
+                localFiles={localFileList.map(file => file.file)}
                 onlineDocuments={onlineDocuments}
                 websitePages={websitePages}
                 onlineDriveFiles={selectedOnlineDriveFileList}

web/app/components/rag-pipeline/components/panel/test-run/preparation/hooks.ts

@@ -110,13 +110,13 @@ export const useOnlineDrive = () => {
 
   const clearOnlineDriveData = useCallback(() => {
     const {
-      setFileList,
+      setOnlineDriveFileList,
       setBucket,
       setPrefix,
       setKeywords,
       setSelectedFileIds,
     } = dataSourceStore.getState()
-    setFileList([])
+    setOnlineDriveFileList([])
     setBucket('')
     setPrefix([])
     setKeywords('')

web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx

@@ -104,8 +104,8 @@ const Preparation = () => {
       })
     }
     if (datasourceType === DatasourceType.onlineDrive) {
-      const { bucket, fileList, selectedFileIds } = dataSourceStore.getState()
+      const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState()
-      const file = fileList.find(file => file.id === selectedFileIds[0])
+      const file = onlineDriveFileList.find(file => file.id === selectedFileIds[0])
       datasourceInfoList.push({
         bucket,
         id: file?.id,

web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts

@@ -110,8 +110,8 @@ const useBeforeRunForm = ({
       }
     }
     if (datasourceType === DatasourceType.onlineDrive) {
-      const { bucket, fileList, selectedFileIds } = dataSourceStore.getState()
+      const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState()
-      const file = fileList.find(file => file.id === selectedFileIds[0])
+      const file = onlineDriveFileList.find(file => file.id === selectedFileIds[0])
       datasourceInfo = {
         bucket,
         id: file?.id,