diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx index 6969539adb..da379d8eac 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx @@ -43,25 +43,25 @@ const Breadcrumbs = ({ }, [displayBreadcrumbNum, prefix]) const handleBackToBucketList = useCallback(() => { - const { setFileList, setSelectedFileKeys, setPrefix, setBucket } = dataSourceStore.getState() + const { setFileList, setSelectedFileIds, setPrefix, setBucket } = dataSourceStore.getState() setFileList([]) - setSelectedFileKeys([]) + setSelectedFileIds([]) setBucket('') setPrefix([]) }, [dataSourceStore]) const handleClickBucketName = useCallback(() => { - const { setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState() + const { setFileList, setSelectedFileIds, setPrefix } = dataSourceStore.getState() setFileList([]) - setSelectedFileKeys([]) + setSelectedFileIds([]) setPrefix([]) }, [dataSourceStore]) const handleClickBreadcrumb = useCallback((index: number) => { - const { prefix, setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState() + const { prefix, setFileList, setSelectedFileIds, setPrefix } = dataSourceStore.getState() const newPrefix = prefix.slice(0, index + 1) setFileList([]) - setSelectedFileKeys([]) + setSelectedFileIds([]) setPrefix(newPrefix) }, [dataSourceStore]) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx index 97c3fecbcf..def7cfa2c5 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx @@ -6,7 +6,7 @@ import { useDebounceFn } from 'ahooks' type FileListProps = { fileList: OnlineDriveFile[] - selectedFileKeys: string[] + selectedFileIds: string[] prefix: string[] keywords: string bucket: string @@ -21,7 +21,7 @@ type FileListProps = { const FileList = ({ fileList, - selectedFileKeys, + selectedFileIds, prefix, keywords, bucket, @@ -67,7 +67,7 @@ const FileList = ({ /> { if (anchorRef.current) { observerRef.current = new IntersectionObserver((entries) => { - const { setStartAfter, isTruncated } = dataSourceStore.getState() + const { setNextPageParameters, currentNextPageParametersRef, isTruncated } = dataSourceStore.getState() if (entries[0].isIntersecting && isTruncated.current && !isLoading) - setStartAfter(fileList[fileList.length - 1].key) + setNextPageParameters(currentNextPageParametersRef.current) }, { rootMargin: '100px', }) @@ -72,10 +72,10 @@ const List = ({
{ fileList.map((file) => { - const isSelected = selectedFileKeys.includes(file.key) + const isSelected = selectedFileIds.includes(file.id) return ( { const { t } = useTranslation() - const isBucket = file.type === 'bucket' - const isFolder = file.type === 'folder' + const { id, name, type, size } = file + const isBucket = useMemo(() => type === 'bucket', [type]) + const isFolder = useMemo(() => type === 'folder', [type]) const Wrapper = disabled ? Tooltip : React.Fragment @@ -55,7 +56,7 @@ const Item = ({ @@ -78,15 +79,15 @@ const Item = ({ 'flex grow items-center gap-x-1 overflow-hidden py-0.5', disabled && 'opacity-30', )}> - + - {file.displayName} + {name} - {!isFolder && typeof file.size === 'number' && ( - {formatFileSize(file.size)} + {!isFolder && typeof size === 'number' && ( + {formatFileSize(size)} )}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx index 97429704d9..db9977fc60 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx @@ -32,19 +32,19 @@ const OnlineDrive = ({ const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) const setShowAccountSettingModal = useModalContextSelector(s => s.setShowAccountSettingModal) const { - startAfter, + nextPageParameters, prefix, keywords, bucket, - selectedFileKeys, + selectedFileIds, fileList, currentCredentialId, } = useDataSourceStoreWithSelector(useShallow(state => ({ - startAfter: state.startAfter, + nextPageParameters: state.nextPageParameters, prefix: state.prefix, keywords: state.keywords, bucket: state.bucket, - selectedFileKeys: state.selectedFileKeys, + selectedFileIds: state.selectedFileIds, fileList: state.fileList, currentCredentialId: state.currentCredentialId, }))) @@ -61,7 +61,7 @@ const OnlineDrive = ({ : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` const getOnlineDriveFiles = useCallback(async () => { - const { startAfter, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState() + const { nextPageParameters, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState() const prefixString = prefix.length > 0 ? `${prefix.join('/')}/` : '' setIsLoading(true) ssePost( @@ -71,7 +71,7 @@ const OnlineDrive = ({ inputs: { prefix: prefixString, bucket, - start_after: startAfter, + next_page_parameters: nextPageParameters, max_keys: 30, // Adjust as needed }, datasource_type: DatasourceType.onlineDrive, @@ -80,10 +80,15 @@ const OnlineDrive = ({ }, { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { - const { setFileList, isTruncated } = dataSourceStore.getState() - const { fileList: newFileList, isTruncated: newIsTruncated } = convertOnlineDriveData(documentsData.data, prefix, bucket) + const { setFileList, isTruncated, currentNextPageParametersRef } = dataSourceStore.getState() + const { + fileList: newFileList, + isTruncated: newIsTruncated, + nextPageParameters: newNextPageParameters, + } = convertOnlineDriveData(documentsData.data, prefix, bucket) setFileList([...fileList, ...newFileList]) isTruncated.current = newIsTruncated + currentNextPageParametersRef.current = newNextPageParameters setIsLoading(false) }, onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => { @@ -107,11 +112,11 @@ const OnlineDrive = ({ else { getOnlineDriveFiles() } - }, [startAfter, prefix, bucket, currentCredentialId]) + }, [nextPageParameters, prefix, bucket, currentCredentialId]) const onlineDriveFileList = useMemo(() => { if (keywords) - return fileList.filter(file => file.key.toLowerCase().includes(keywords.toLowerCase())) + return fileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase())) return fileList }, [fileList, keywords]) @@ -127,33 +132,32 @@ const OnlineDrive = ({ }, [dataSourceStore]) const handleSelectFile = useCallback((file: OnlineDriveFile) => { - const { selectedFileKeys, setSelectedFileKeys } = dataSourceStore.getState() + const { selectedFileIds, setSelectedFileIds } = dataSourceStore.getState() if (file.type === OnlineDriveFileType.bucket) return - const newSelectedFileList = produce(selectedFileKeys, (draft) => { - if (draft.includes(file.key)) { - const index = draft.indexOf(file.key) + const newSelectedFileList = produce(selectedFileIds, (draft) => { + if (draft.includes(file.id)) { + const index = draft.indexOf(file.id) draft.splice(index, 1) } else { if (isInPipeline && draft.length >= 1) return - draft.push(file.key) + draft.push(file.id) } }) - setSelectedFileKeys(newSelectedFileList) + setSelectedFileIds(newSelectedFileList) }, [dataSourceStore, isInPipeline]) const handleOpenFolder = useCallback((file: OnlineDriveFile) => { - const { prefix, setPrefix, setBucket, setFileList, setSelectedFileKeys } = dataSourceStore.getState() + const { prefix, setPrefix, setBucket, setFileList, setSelectedFileIds } = dataSourceStore.getState() if (file.type === OnlineDriveFileType.file) return setFileList([]) if (file.type === OnlineDriveFileType.bucket) { - setBucket(file.displayName) + setBucket(file.name) } else { - setSelectedFileKeys([]) - const displayName = file.displayName.endsWith('/') ? file.displayName.slice(0, -1) : file.displayName + setSelectedFileIds([]) const newPrefix = produce(prefix, (draft) => { - draft.push(displayName) + draft.push(file.name) }) setPrefix(newPrefix) } @@ -178,7 +182,7 @@ const OnlineDrive = ({ /> { - return filePathRegex.test(path) +export const isFile = (type: 'file' | 'folder'): boolean => { + return type === 'file' } export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[], bucket: string): boolean => { @@ -12,34 +10,40 @@ export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[] return data.length > 1 || (data.length === 1 && !!data[0].bucket && data[0].files.length === 0) } -export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[], bucket: string): { fileList: OnlineDriveFile[], isTruncated: boolean } => { +export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[], bucket: string): { + fileList: OnlineDriveFile[], + isTruncated: boolean, + nextPageParameters: Record +} => { const fileList: OnlineDriveFile[] = [] let isTruncated = false + let nextPageParameters: Record = {} if (data.length === 0) - return { fileList, isTruncated } + return { fileList, isTruncated, nextPageParameters } if (isBucketListInitiation(data, prefix, bucket)) { data.forEach((item) => { fileList.push({ - key: item.bucket, - displayName: item.bucket, + id: item.bucket, + name: item.bucket, type: OnlineDriveFileType.bucket, }) }) } else { data[0].files.forEach((file) => { - const isFileType = isFile(file.key) - const filePathList = file.key.split('/') + const { id, name, size, type } = file + const isFileType = isFile(type) fileList.push({ - key: file.key, - displayName: `${isFileType ? filePathList.pop() : filePathList[filePathList.length - 2]}${isFileType ? '' : '/'}`, - size: isFileType ? file.size : undefined, + id, + name, + size: isFileType ? size : undefined, type: isFileType ? OnlineDriveFileType.file : OnlineDriveFileType.folder, }) }) isTruncated = data[0].is_truncated ?? false + nextPageParameters = data[0].next_page_parameters ?? {} } - return { fileList, isTruncated } + return { fileList, isTruncated, nextPageParameters } } diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts index 1093d98c54..ab7b641a01 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts @@ -6,14 +6,15 @@ export type OnlineDriveSliceShape = { setPrefix: (prefix: string[]) => void keywords: string setKeywords: (keywords: string) => void - selectedFileKeys: string[] - setSelectedFileKeys: (selectedFileKeys: string[]) => void + selectedFileIds: string[] + setSelectedFileIds: (selectedFileIds: string[]) => void fileList: OnlineDriveFile[] setFileList: (fileList: OnlineDriveFile[]) => void bucket: string setBucket: (bucket: string) => void - startAfter: string - setStartAfter: (startAfter: string) => void + nextPageParameters: Record + currentNextPageParametersRef: React.RefObject> + setNextPageParameters: (nextPageParameters: Record) => void isTruncated: React.RefObject previewOnlineDriveFileRef: React.RefObject } @@ -28,14 +29,14 @@ export const createOnlineDriveSlice: StateCreator = (set, setKeywords: (keywords: string) => set(() => ({ keywords, })), - selectedFileKeys: [], - setSelectedFileKeys: (selectedFileKeys: string[]) => { + selectedFileIds: [], + setSelectedFileIds: (selectedFileIds: string[]) => { set(() => ({ - selectedFileKeys, + selectedFileIds, })) - const key = selectedFileKeys[0] + const id = selectedFileIds[0] const { fileList, previewOnlineDriveFileRef } = get() - previewOnlineDriveFileRef.current = fileList.find(file => file.key === key) + previewOnlineDriveFileRef.current = fileList.find(file => file.id === id) }, fileList: [], setFileList: (fileList: OnlineDriveFile[]) => set(() => ({ @@ -45,9 +46,10 @@ export const createOnlineDriveSlice: StateCreator = (set, setBucket: (bucket: string) => set(() => ({ bucket, })), - startAfter: '', - setStartAfter: (startAfter: string) => set(() => ({ - startAfter, + nextPageParameters: {}, + currentNextPageParametersRef: { current: {} }, + setNextPageParameters: (nextPageParameters: Record) => set(() => ({ + nextPageParameters, })), isTruncated: { current: false }, previewOnlineDriveFileRef: { current: undefined }, diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 910b8243d4..c05bceb7e2 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -188,16 +188,16 @@ export const useWebsiteCrawl = () => { export const useOnlineDrive = () => { const { fileList, - selectedFileKeys, + selectedFileIds, } = useDataSourceStoreWithSelector(useShallow(state => ({ fileList: state.fileList, - selectedFileKeys: state.selectedFileKeys, + selectedFileIds: state.selectedFileIds, }))) const dataSourceStore = useDataSourceStore() const selectedOnlineDriveFileList = useMemo(() => { - return selectedFileKeys.map(key => fileList.find(item => item.key === key)!) - }, [fileList, selectedFileKeys]) + return selectedFileIds.map(key => fileList.find(item => item.key === key)!) + }, [fileList, selectedFileIds]) const clearOnlineDriveData = useCallback(() => { const { @@ -205,18 +205,18 @@ export const useOnlineDrive = () => { setBucket, setPrefix, setKeywords, - setSelectedFileKeys, + setSelectedFileIds, } = dataSourceStore.getState() setFileList([]) setBucket('') setPrefix([]) setKeywords('') - setSelectedFileKeys([]) + setSelectedFileIds([]) }, [dataSourceStore]) return { fileList, - selectedFileKeys, + selectedFileIds, selectedOnlineDriveFileList, clearOnlineDriveData, } diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index cafaa668bd..970c6fa694 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -82,7 +82,7 @@ const CreateFormPipeline = () => { } = useWebsiteCrawl() const { fileList: onlineDriveFileList, - selectedFileKeys, + selectedFileIds, selectedOnlineDriveFileList, clearOnlineDriveData, } = useOnlineDrive() @@ -113,9 +113,9 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length if (datasourceType === DatasourceType.onlineDrive) - return isShowVectorSpaceFull || !selectedFileKeys.length + return isShowVectorSpaceFull || !selectedFileIds.length return false - }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileKeys.length]) + }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length]) const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { file_size_limit: 15, @@ -149,8 +149,8 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.onlineDocument) return onlineDocuments.length if (datasourceType === DatasourceType.onlineDrive) - return selectedFileKeys.length - }, [datasourceType, onlineDocuments.length, selectedFileKeys.length]) + return selectedFileIds.length + }, [datasourceType, onlineDocuments.length, selectedFileIds.length]) const tip = useMemo(() => { if (datasourceType === DatasourceType.onlineDocument) @@ -209,10 +209,10 @@ const CreateFormPipeline = () => { } if (datasourceType === DatasourceType.onlineDrive) { const { bucket } = dataSourceStore.getState() - const { key } = previewOnlineDriveFileRef.current! + const { id } = previewOnlineDriveFileRef.current! datasourceInfoList.push({ bucket, - key, + id, credential_id: currentCredentialId, }) } @@ -273,7 +273,7 @@ const CreateFormPipeline = () => { } if (datasourceType === DatasourceType.onlineDrive) { if (datasourceType === DatasourceType.onlineDrive) { - selectedFileKeys.forEach((key) => { + selectedFileIds.forEach((key) => { datasourceInfoList.push({ bucket, key, @@ -296,7 +296,7 @@ const CreateFormPipeline = () => { handleNextStep() }, }) - }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileKeys, websitePages]) + }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileIds, websitePages]) const onClickProcess = useCallback(() => { isPreview.current = false @@ -340,9 +340,9 @@ const CreateFormPipeline = () => { const { onlineDocuments, fileList: onlineDriveFileList, - selectedFileKeys, + selectedFileIds, setOnlineDocuments, - setSelectedFileKeys, + setSelectedFileIds, setSelectedPagesId, } = dataSourceStore.getState() if (datasourceType === DatasourceType.onlineDocument) { @@ -360,11 +360,11 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.onlineDrive) { const allKeys = onlineDriveFileList.filter((item) => { return item.type !== 'bucket' - }).map(file => file.key) - if (selectedFileKeys.length < allKeys.length) - setSelectedFileKeys(allKeys) + }).map(file => file.id) + if (selectedFileIds.length < allKeys.length) + setSelectedFileIds(allKeys) else - setSelectedFileKeys([]) + setSelectedFileIds([]) } }, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType]) diff --git a/web/app/components/header/account-setting/data-source-page-new/configure.tsx b/web/app/components/header/account-setting/data-source-page-new/configure.tsx index f0f98da550..aa4a6e9d1d 100644 --- a/web/app/components/header/account-setting/data-source-page-new/configure.tsx +++ b/web/app/components/header/account-setting/data-source-page-new/configure.tsx @@ -73,7 +73,7 @@ const Configure = ({
{ - canOAuth && ( + !!canOAuth && (
OR @@ -98,7 +98,7 @@ const Configure = ({ ) } { - canApiKey && ( + !!canApiKey && ( { setBucket, setPrefix, setKeywords, - setSelectedFileKeys, + setSelectedFileIds, } = dataSourceStore.getState() setFileList([]) setBucket('') setPrefix([]) setKeywords('') - setSelectedFileKeys([]) + setSelectedFileIds([]) }, [dataSourceStore]) return { diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index 8a06781271..048f13d79b 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -30,12 +30,12 @@ const TestRunPanel = () => { localFileList: fileList, onlineDocuments, websitePages, - selectedFileKeys, + selectedFileIds, } = useDataSourceStoreWithSelector(useShallow(state => ({ localFileList: state.localFileList, onlineDocuments: state.onlineDocuments, websitePages: state.websitePages, - selectedFileKeys: state.selectedFileKeys, + selectedFileIds: state.selectedFileIds, }))) const dataSourceStore = useDataSourceStore() const [datasource, setDatasource] = useState() @@ -62,9 +62,9 @@ const TestRunPanel = () => { if (datasourceType === DatasourceType.websiteCrawl) return !websitePages.length if (datasourceType === DatasourceType.onlineDrive) - return !selectedFileKeys.length + return !selectedFileIds.length return false - }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileKeys.length, websitePages.length]) + }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileIds.length, websitePages.length]) const handleClose = useCallback(() => { setShowDebugAndPreviewPanel(false) @@ -110,7 +110,7 @@ const TestRunPanel = () => { const { bucket } = dataSourceStore.getState() datasourceInfoList.push({ bucket, - key: selectedFileKeys[0], + id: selectedFileIds[0], credential_id: credentialId, }) } @@ -120,7 +120,7 @@ const TestRunPanel = () => { datasource_type: datasourceType, datasource_info_list: datasourceInfoList, }) - }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileKeys, websitePages]) + }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileIds, websitePages]) const clearDataSourceData = useCallback((dataSource: Datasource) => { if (dataSource.nodeData.provider_type === DatasourceType.onlineDocument) diff --git a/web/models/pipeline.ts b/web/models/pipeline.ts index 853c3cc13a..6d6835ee2c 100644 --- a/web/models/pipeline.ts +++ b/web/models/pipeline.ts @@ -282,8 +282,8 @@ export enum OnlineDriveFileType { } export type OnlineDriveFile = { - key: string - displayName: string + id: string + name: string size?: number type: OnlineDriveFileType } diff --git a/web/types/pipeline.tsx b/web/types/pipeline.tsx index 768664d053..9caa762ac5 100644 --- a/web/types/pipeline.tsx +++ b/web/types/pipeline.tsx @@ -13,14 +13,17 @@ export type DataSourceNodeError = { } export type OnlineDriveFile = { - key: string + id: string + name: string size: number + type: 'file' | 'folder' } export type OnlineDriveData = { bucket: string files: OnlineDriveFile[] is_truncated: boolean + next_page_parameters: Record } export type DataSourceNodeCompletedResponse = {