diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx index 7db046e13f..cbe565e56d 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx @@ -17,7 +17,6 @@ type FileListProps = { handleSelectFile: (file: OnlineDriveFile) => void handleOpenFolder: (file: OnlineDriveFile) => void isLoading: boolean - isTruncated: boolean getOnlineDriveFiles: (params: { prefix?: string[] bucket?: string @@ -39,7 +38,6 @@ const FileList = ({ handleOpenFolder, isInPipeline, isLoading, - isTruncated, getOnlineDriveFiles, }: FileListProps) => { const [inputValue, setInputValue] = useState(keywords) @@ -84,7 +82,6 @@ const FileList = ({ handleSelectFile={handleSelectFile} isInPipeline={isInPipeline} isLoading={isLoading} - isTruncated={isTruncated} getOnlineDriveFiles={getOnlineDriveFiles} /> diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx index 9060862d8e..a91f8b1dcc 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx @@ -12,7 +12,6 @@ type FileListProps = { selectedFileList: string[] keywords: string isInPipeline: boolean - isTruncated: boolean isLoading: boolean handleResetKeywords: () => void handleSelectFile: (file: OnlineDriveFile) => void @@ -34,7 +33,6 @@ const List = ({ handleOpenFolder, isInPipeline, isLoading, - isTruncated, getOnlineDriveFiles, }: FileListProps) => { const anchorRef = useRef(null) @@ -44,9 +42,9 @@ const List = ({ useEffect(() => { if (anchorRef.current) { observerRef.current = new IntersectionObserver((entries) => { - const { setStartAfter } = dataSourceStore.getState() - if (entries[0].isIntersecting && isTruncated && !isLoading) { - setStartAfter(fileList[fileList.length - 1].key) + const { startAfter, isTruncated } = dataSourceStore.getState() + if (entries[0].isIntersecting && isTruncated.current && !isLoading) { + startAfter.current = fileList[fileList.length - 1].key getOnlineDriveFiles({ startAfter: fileList[fileList.length - 1].key }) } }, { diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx index f33d03d348..9aabd13ab5 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx @@ -27,10 +27,8 @@ const OnlineDrive = ({ const prefix = useDataSourceStoreWithSelector(state => state.prefix) const keywords = useDataSourceStoreWithSelector(state => state.keywords) const bucket = useDataSourceStoreWithSelector(state => state.bucket) - const startAfter = useDataSourceStoreWithSelector(state => state.startAfter) const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) const fileList = useDataSourceStoreWithSelector(state => state.fileList) - const isTruncated = useDataSourceStoreWithSelector(state => state.isTruncated) const dataSourceStore = useDataSourceStore() const [isLoading, setIsLoading] = useState(false) @@ -44,9 +42,10 @@ const OnlineDrive = ({ startAfter?: string fileList?: OnlineDriveFile[] }) => { + const { startAfter } = dataSourceStore.getState() const _prefix = params.prefix ?? prefix const _bucket = params.bucket ?? bucket - const _startAfter = params.startAfter ?? startAfter + const _startAfter = params.startAfter ?? startAfter.current const _fileList = params.fileList ?? fileList const prefixString = _prefix.length > 0 ? `${_prefix.join('/')}/` : '' setIsLoading(true) @@ -65,10 +64,10 @@ const OnlineDrive = ({ }, { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { - const { setFileList, setIsTruncated } = dataSourceStore.getState() - const { fileList: newFileList, isTruncated } = convertOnlineDriveData(documentsData.data, _prefix) + const { setFileList, isTruncated } = dataSourceStore.getState() + const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, _prefix, _bucket) setFileList([..._fileList, ...newFileList]) - setIsTruncated(isTruncated) + isTruncated.current = newIsTruncated setIsLoading(false) }, onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => { @@ -80,7 +79,7 @@ const OnlineDrive = ({ }, }, ) - }, [prefix, bucket, startAfter, datasourceNodeRunURL, dataSourceStore, fileList]) + }, [prefix, bucket, datasourceNodeRunURL, dataSourceStore, fileList]) useEffect(() => { if (fileList.length > 0) return @@ -99,7 +98,7 @@ const OnlineDrive = ({ setKeywords(keywords) }, [dataSourceStore]) - const resetPrefix = useCallback(() => { + const resetKeywords = useCallback(() => { const { setKeywords } = dataSourceStore.getState() setKeywords('') @@ -152,14 +151,13 @@ const OnlineDrive = ({ prefix={prefix} keywords={keywords} bucket={bucket} - resetKeywords={resetPrefix} + resetKeywords={resetKeywords} updateKeywords={updateKeywords} searchResultsLength={onlineDriveFileList.length} handleSelectFile={handleSelectFile} handleOpenFolder={handleOpenFolder} isInPipeline={isInPipeline} isLoading={isLoading} - isTruncated={isTruncated} getOnlineDriveFiles={getOnlineDriveFiles} /> diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts index 105d8af632..e3bbac22a0 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts @@ -7,19 +7,19 @@ export const isFile = (path: string): boolean => { return filePathRegex.test(path) } -export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[]): boolean => { - if (prefix.length > 0) return false +export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[], bucket: string): boolean => { + if (bucket || prefix.length > 0) return false return data.length > 1 || (data.length === 1 && data[0].files.length === 0) } -export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[]): { fileList: OnlineDriveFile[], isTruncated: boolean } => { +export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[], bucket: string): { fileList: OnlineDriveFile[], isTruncated: boolean } => { const fileList: OnlineDriveFile[] = [] let isTruncated = false if (data.length === 0) return { fileList, isTruncated } - if (isBucketListInitiation(data, prefix)) { + if (isBucketListInitiation(data, prefix, bucket)) { data.forEach((item) => { fileList.push({ key: item.bucket, diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts index aef30447d2..186f23fcc4 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts @@ -6,16 +6,14 @@ export type OnlineDriveSliceShape = { setPrefix: (prefix: string[]) => void keywords: string setKeywords: (keywords: string) => void - startAfter: string - setStartAfter: (startAfter: string) => void selectedFileList: string[] setSelectedFileList: (selectedFileList: string[]) => void fileList: OnlineDriveFile[] setFileList: (fileList: OnlineDriveFile[]) => void bucket: string setBucket: (bucket: string) => void - isTruncated: boolean - setIsTruncated: (isTruncated: boolean) => void + startAfter: React.MutableRefObject + isTruncated: React.MutableRefObject } export const createOnlineDriveSlice: StateCreator = (set) => { @@ -28,10 +26,7 @@ export const createOnlineDriveSlice: StateCreator = (set) setKeywords: (keywords: string) => set(() => ({ keywords, })), - startAfter: '', - setStartAfter: (startAfter: string) => set(() => ({ - startAfter, - })), + startAfter: { current: '' }, selectedFileList: [], setSelectedFileList: (selectedFileList: string[]) => set(() => ({ selectedFileList, @@ -44,9 +39,6 @@ export const createOnlineDriveSlice: StateCreator = (set) setBucket: (bucket: string) => set(() => ({ bucket, })), - isTruncated: false, - setIsTruncated: (isTruncated: boolean) => set(() => ({ - isTruncated, - })), + isTruncated: { current: false }, }) }