diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
index cb42df02ff..d540c6373c 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
@@ -12,6 +12,8 @@ type FileListProps = {
resetKeywords: () => void
updateKeywords: (keywords: string) => void
searchResultsLength: number
+ handleSelectFile: (file: OnlineDriveFile) => void
+ handleOpenFolder: (file: OnlineDriveFile) => void
}
const FileList = ({
@@ -22,6 +24,8 @@ const FileList = ({
resetKeywords,
updateKeywords,
searchResultsLength,
+ handleSelectFile,
+ handleOpenFolder,
}: FileListProps) => {
const [inputValue, setInputValue] = useState(keywords)
@@ -58,6 +62,8 @@ const FileList = ({
selectedFileList={selectedFileList}
keywords={keywords}
handleResetKeywords={handleResetKeywords}
+ handleOpenFolder={handleOpenFolder}
+ handleSelectFile={handleSelectFile}
/>
)
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
index 3968e673e1..b142bba72c 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
@@ -8,6 +8,8 @@ type FileListProps = {
selectedFileList: string[]
keywords: string
handleResetKeywords: () => void
+ handleSelectFile: (file: OnlineDriveFile) => void
+ handleOpenFolder: (file: OnlineDriveFile) => void
}
const List = ({
@@ -15,6 +17,8 @@ const List = ({
selectedFileList,
keywords,
handleResetKeywords,
+ handleSelectFile,
+ handleOpenFolder,
}: FileListProps) => {
const isEmptyFolder = fileList.length === 0 && keywords.length === 0
const isSearchResultEmpty = fileList.length === 0 && keywords.length > 0
@@ -41,9 +45,8 @@ const List = ({
key={file.key}
file={file}
isSelected={isSelected}
- onSelect={(file) => { console.log(file) }}
- onOpen={(file) => { console.log(file) }}
- disabled
+ onSelect={handleSelectFile}
+ onOpen={handleOpenFolder}
/>
)
})
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx
index bf7892bf0f..d8df0d24c6 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/item.tsx
@@ -40,6 +40,7 @@ const Item = ({
if (disabled) return
if (isBucket || isFolder)
onOpen(file)
+ if (isBucket) return // Do not select bucket
onSelect(file)
}, [disabled, file, isBucket, isFolder, onOpen, onSelect])
@@ -61,11 +62,16 @@ const Item = ({
>
-
{file.key}
+
+ {file.key}
+
{!isFolder && file.size && (
{formatFileSize(file.size)}
)}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
index c453498529..d1a8c04bd3 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
@@ -2,12 +2,15 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so
import Header from './header'
import { useCallback, useEffect } from 'react'
import FileList from './file-list'
-import { DatasourceType } from '@/models/pipeline'
+import type { OnlineDriveFile } from '@/models/pipeline'
+import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { ssePost } from '@/service/base'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import Toast from '@/app/components/base/toast'
import { useDataSourceStore } from '../store'
+import { convertOnlineDriveDataToFileList } from './utils'
+import produce from 'immer'
type OnlineDriveProps = {
nodeId: string
@@ -54,7 +57,8 @@ const OnlineDrive = ({
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
- console.log('Online Drive documents data:', documentsData)
+ const newFileList = convertOnlineDriveDataToFileList(documentsData.data)
+ setFileList([...fileList, ...newFileList])
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
Toast.notify({
@@ -64,13 +68,12 @@ const OnlineDrive = ({
},
},
)
- }, [bucket, datasourceNodeRunURL, prefix, startAfter])
+ }, [bucket, datasourceNodeRunURL, prefix, fileList, setFileList, startAfter])
useEffect(() => {
- if (!fileList.length)
- getOnlineDrive()
+ getOnlineDrive()
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [])
+ }, [bucket, prefix, startAfter])
const updateKeywords = useCallback((keywords: string) => {
setKeywords(keywords)
@@ -80,6 +83,35 @@ const OnlineDrive = ({
setKeywords('')
}, [setKeywords])
+ const handleSelectFile = useCallback((file: OnlineDriveFile) => {
+ if (file.type === OnlineDriveFileType.bucket) return
+ const newSelectedFileList = produce(selectedFileList, (draft) => {
+ if (draft.includes(file.key)) {
+ const index = draft.indexOf(file.key)
+ draft.splice(index, 1)
+ }
+ else {
+ draft.push(file.key)
+ }
+ })
+ setSelectedFileList(newSelectedFileList)
+ }, [selectedFileList, setSelectedFileList])
+
+ const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
+ if (file.type === OnlineDriveFileType.file) return
+ setFileList([])
+ if (file.type === OnlineDriveFileType.bucket) {
+ setBucket(file.key)
+ }
+ else {
+ const newPrefix = produce(prefix, (draft) => {
+ const newList = file.key.split('/')
+ draft.push(...newList)
+ })
+ setPrefix(newPrefix)
+ }
+ }, [prefix, setBucket, setFileList, setPrefix])
+
return (
)
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
new file mode 100644
index 0000000000..e608b50b64
--- /dev/null
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
@@ -0,0 +1,35 @@
+import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline'
+import type { OnlineDriveData } from '@/types/pipeline'
+
+const filePathRegex = /^(?:.*\/)?[^\/]+\.[^\/\.]+$/
+
+export const isFile = (path: string): boolean => {
+ return filePathRegex.test(path)
+}
+
+export const hasBuckets = (data: OnlineDriveData[]): boolean => {
+ return data.length > 1 || (data.length === 1 && data[0].files.length === 0)
+}
+
+export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[]): OnlineDriveFile[] => {
+ const fileList: OnlineDriveFile[] = []
+
+ if (hasBuckets(data)) {
+ data.forEach((item) => {
+ fileList.push({
+ key: item.bucket,
+ type: OnlineDriveFileType.bucket,
+ })
+ })
+ }
+ else {
+ data[0].files.forEach((file) => {
+ fileList.push({
+ key: file.key,
+ size: isFile(file.key) ? file.size : undefined,
+ type: isFile(file.key) ? OnlineDriveFileType.file : OnlineDriveFileType.folder,
+ })
+ })
+ }
+ return fileList
+}
diff --git a/web/types/pipeline.tsx b/web/types/pipeline.tsx
index 229912cf81..080c98b47e 100644
--- a/web/types/pipeline.tsx
+++ b/web/types/pipeline.tsx
@@ -10,10 +10,21 @@ export type DataSourceNodeError = {
code?: string
}
+export type OnlineDriveFile = {
+ key: string
+ size: number
+}
+
+export type OnlineDriveData = {
+ bucket: string
+ files: OnlineDriveFile[]
+ is_truncated: boolean
+}
+
export type DataSourceNodeCompletedResponse = {
event: 'datasource_completed'
- data: any
- time_consuming?: number
+ data: OnlineDriveData[]
+ time_consuming: number
}
export type DataSourceNodeErrorResponse = {