diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx index cb42df02ff..d540c6373c 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx @@ -12,6 +12,8 @@ type FileListProps = { resetKeywords: () => void updateKeywords: (keywords: string) => void searchResultsLength: number + handleSelectFile: (file: OnlineDriveFile) => void + handleOpenFolder: (file: OnlineDriveFile) => void } const FileList = ({ @@ -22,6 +24,8 @@ const FileList = ({ resetKeywords, updateKeywords, searchResultsLength, + handleSelectFile, + handleOpenFolder, }: FileListProps) => { const [inputValue, setInputValue] = useState(keywords) @@ -58,6 +62,8 @@ const FileList = ({ selectedFileList={selectedFileList} keywords={keywords} handleResetKeywords={handleResetKeywords} + handleOpenFolder={handleOpenFolder} + handleSelectFile={handleSelectFile} /> ) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx index 3968e673e1..b142bba72c 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx @@ -8,6 +8,8 @@ type FileListProps = { selectedFileList: string[] keywords: string handleResetKeywords: () => void + handleSelectFile: (file: OnlineDriveFile) => void + handleOpenFolder: (file: OnlineDriveFile) => void } const List = ({ @@ -15,6 +17,8 @@ const List = ({ selectedFileList, keywords, handleResetKeywords, + handleSelectFile, + handleOpenFolder, }: FileListProps) => { const isEmptyFolder = fileList.length === 0 && keywords.length === 0 const isSearchResultEmpty = fileList.length === 0 && keywords.length > 0 @@ -41,9 +45,8 @@ const List = ({ key={file.key} file={file} isSelected={isSelected} - onSelect={(file) => { console.log(file) }} - onOpen={(file) => { console.log(file) }} - disabled + onSelect={handleSelectFile} + onOpen={handleOpenFolder} /> ) }) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx index bf7892bf0f..d8df0d24c6 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx @@ -40,6 +40,7 @@ const Item = ({ if (disabled) return if (isBucket || isFolder) onOpen(file) + if (isBucket) return // Do not select bucket onSelect(file) }, [disabled, file, isBucket, isFolder, onOpen, onSelect]) @@ -61,11 +62,16 @@ const Item = ({ >
- {file.key} + + {file.key} + {!isFolder && file.size && ( {formatFileSize(file.size)} )} diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx index c453498529..d1a8c04bd3 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx @@ -2,12 +2,15 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so import Header from './header' import { useCallback, useEffect } from 'react' import FileList from './file-list' -import { DatasourceType } from '@/models/pipeline' +import type { OnlineDriveFile } from '@/models/pipeline' +import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { ssePost } from '@/service/base' import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline' import Toast from '@/app/components/base/toast' import { useDataSourceStore } from '../store' +import { convertOnlineDriveDataToFileList } from './utils' +import produce from 'immer' type OnlineDriveProps = { nodeId: string @@ -54,7 +57,8 @@ const OnlineDrive = ({ }, { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { - console.log('Online Drive documents data:', documentsData) + const newFileList = convertOnlineDriveDataToFileList(documentsData.data) + setFileList([...fileList, ...newFileList]) }, onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => { Toast.notify({ @@ -64,13 +68,12 @@ const OnlineDrive = ({ }, }, ) - }, [bucket, datasourceNodeRunURL, prefix, startAfter]) + }, [bucket, datasourceNodeRunURL, prefix, fileList, setFileList, startAfter]) useEffect(() => { - if (!fileList.length) - getOnlineDrive() + getOnlineDrive() // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) + }, [bucket, prefix, startAfter]) const updateKeywords = useCallback((keywords: string) => { setKeywords(keywords) @@ -80,6 +83,35 @@ const OnlineDrive = ({ setKeywords('') }, [setKeywords]) + const handleSelectFile = useCallback((file: OnlineDriveFile) => { + if (file.type === OnlineDriveFileType.bucket) return + const newSelectedFileList = produce(selectedFileList, (draft) => { + if (draft.includes(file.key)) { + const index = draft.indexOf(file.key) + draft.splice(index, 1) + } + else { + draft.push(file.key) + } + }) + setSelectedFileList(newSelectedFileList) + }, [selectedFileList, setSelectedFileList]) + + const handleOpenFolder = useCallback((file: OnlineDriveFile) => { + if (file.type === OnlineDriveFileType.file) return + setFileList([]) + if (file.type === OnlineDriveFileType.bucket) { + setBucket(file.key) + } + else { + const newPrefix = produce(prefix, (draft) => { + const newList = file.key.split('/') + draft.push(...newList) + }) + setPrefix(newPrefix) + } + }, [prefix, setBucket, setFileList, setPrefix]) + return (
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts new file mode 100644 index 0000000000..e608b50b64 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts @@ -0,0 +1,35 @@ +import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline' +import type { OnlineDriveData } from '@/types/pipeline' + +const filePathRegex = /^(?:.*\/)?[^\/]+\.[^\/\.]+$/ + +export const isFile = (path: string): boolean => { + return filePathRegex.test(path) +} + +export const hasBuckets = (data: OnlineDriveData[]): boolean => { + return data.length > 1 || (data.length === 1 && data[0].files.length === 0) +} + +export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[]): OnlineDriveFile[] => { + const fileList: OnlineDriveFile[] = [] + + if (hasBuckets(data)) { + data.forEach((item) => { + fileList.push({ + key: item.bucket, + type: OnlineDriveFileType.bucket, + }) + }) + } + else { + data[0].files.forEach((file) => { + fileList.push({ + key: file.key, + size: isFile(file.key) ? file.size : undefined, + type: isFile(file.key) ? OnlineDriveFileType.file : OnlineDriveFileType.folder, + }) + }) + } + return fileList +} diff --git a/web/types/pipeline.tsx b/web/types/pipeline.tsx index 229912cf81..080c98b47e 100644 --- a/web/types/pipeline.tsx +++ b/web/types/pipeline.tsx @@ -10,10 +10,21 @@ export type DataSourceNodeError = { code?: string } +export type OnlineDriveFile = { + key: string + size: number +} + +export type OnlineDriveData = { + bucket: string + files: OnlineDriveFile[] + is_truncated: boolean +} + export type DataSourceNodeCompletedResponse = { event: 'datasource_completed' - data: any - time_consuming?: number + data: OnlineDriveData[] + time_consuming: number } export type DataSourceNodeErrorResponse = {