opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: update local file and online drive state management in create-from-pipeline components

This commit is contained in:
twwu 2025-08-28 13:47:20 +08:00
parent 843b14ccc6
commit 048feb4165
11 changed files with 65 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx
View File

@ -32,7 +32,6 @@ const DataSourceOptions = ({
useEffect(() => {
if (options.length > 0 && !datasourceNodeId)
handelSelect(options[0].value)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
return (

12
web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx
View File

@ -32,7 +32,7 @@ const LocalFile = ({
const { t } = useTranslation()
const { notify } = useContext(ToastContext)
const { locale } = useContext(I18n)
const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
const localFileList = useDataSourceStoreWithSelector(state => state.localFileList)
const dataSourceStore = useDataSourceStore()
const [dragging, setDragging] = useState(false)
@ -41,7 +41,7 @@ const LocalFile = ({
const fileUploader = useRef<HTMLInputElement>(null)
const fileListRef = useRef<FileItem[]>([])
const hideUpload = notSupportBatchUpload && fileList.length > 0
const hideUpload = notSupportBatchUpload && localFileList.length > 0
const { data: fileUploadConfigResponse } = useFileUploadConfig()
const supportTypesShowNames = useMemo(() => {
@ -179,7 +179,7 @@ const LocalFile = ({
if (!files.length)
return false
if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
if (files.length + localFileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
return false
}
@ -193,7 +193,7 @@ const LocalFile = ({
updateFileList(newFiles)
fileListRef.current = newFiles
uploadMultipleFiles(preparedFiles)
}, [updateFileList, uploadMultipleFiles, notify, t, fileList])
}, [updateFileList, uploadMultipleFiles, notify, t, localFileList])
const handleDragEnter = (e: DragEvent) => {
e.preventDefault()
@ -297,9 +297,9 @@ const LocalFile = ({
{dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
</div>
)}
{fileList.length > 0 && (
{localFileList.length > 0 && (
<div className='mt-1 flex flex-col gap-y-1'>
{fileList.map((fileItem, index) => {
{localFileList.map((fileItem, index) => {
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
const isError = fileItem.progress === -2
return (

1
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/dropdown/menu.tsx
View File

@ -17,6 +17,7 @@ const Menu = ({
{breadcrumbs.map((breadcrumb, index) => {
return (
<Item
key={`${breadcrumb}-${index}`}
name={breadcrumb}
index={startIndex + index}
onBreadcrumbClick={onBreadcrumbClick}

16
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
View File

@ -45,8 +45,8 @@ const Breadcrumbs = ({
}, [displayBreadcrumbNum, breadcrumbs])
const handleBackToBucketList = useCallback(() => {
const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } = dataSourceStore.getState()
setFileList([])
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix, setBucket } = dataSourceStore.getState()
setOnlineDriveFileList([])
setSelectedFileIds([])
setBucket('')
setBreadcrumbs([])
@ -54,26 +54,26 @@ const Breadcrumbs = ({
}, [dataSourceStore])
const handleClickBucketName = useCallback(() => {
const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
setFileList([])
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
setOnlineDriveFileList([])
setSelectedFileIds([])
setBreadcrumbs([])
setPrefix([])
}, [dataSourceStore])
const handleBackToRoot = useCallback(() => {
const { setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
setFileList([])
const { setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
setOnlineDriveFileList([])
setSelectedFileIds([])
setBreadcrumbs([])
setPrefix([])
}, [dataSourceStore])
const handleClickBreadcrumb = useCallback((index: number) => {
const { breadcrumbs, prefix, setFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
const { breadcrumbs, prefix, setOnlineDriveFileList, setSelectedFileIds, setBreadcrumbs, setPrefix } = dataSourceStore.getState()
const newBreadcrumbs = breadcrumbs.slice(0, index + 1)
const newPrefix = prefix.slice(0, index + 1)
setFileList([])
setOnlineDriveFileList([])
setSelectedFileIds([])
setBreadcrumbs(newBreadcrumbs)
setPrefix(newPrefix)

35
web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
View File

@ -1,6 +1,6 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import Header from '../base/header'
import { useCallback, useEffect, useMemo, useState } from 'react'
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import FileList from './file-list'
import type { OnlineDriveFile } from '@/models/pipeline'
import { DatasourceType, OnlineDriveFileType } from '@/models/pipeline'
@ -38,7 +38,7 @@ const OnlineDrive = ({
keywords,
bucket,
selectedFileIds,
fileList,
onlineDriveFileList,
currentCredentialId,
} = useDataSourceStoreWithSelector(useShallow(state => ({
nextPageParameters: state.nextPageParameters,
@ -47,11 +47,12 @@ const OnlineDrive = ({
keywords: state.keywords,
bucket: state.bucket,
selectedFileIds: state.selectedFileIds,
fileList: state.fileList,
onlineDriveFileList: state.onlineDriveFileList,
currentCredentialId: state.currentCredentialId,
})))
const dataSourceStore = useDataSourceStore()
const [isLoading, setIsLoading] = useState(false)
const isLoadingRef = useRef(false)
const { data: dataSourceAuth } = useGetDataSourceAuth({
pluginId: nodeData.plugin_id,
@ -63,8 +64,10 @@ const OnlineDrive = ({
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
const getOnlineDriveFiles = useCallback(async () => {
const { nextPageParameters, prefix, bucket, fileList, currentCredentialId } = dataSourceStore.getState()
if (isLoadingRef.current) return
const { nextPageParameters, prefix, bucket, onlineDriveFileList, currentCredentialId } = dataSourceStore.getState()
setIsLoading(true)
isLoadingRef.current = true
ssePost(
datasourceNodeRunURL,
{
@ -81,18 +84,19 @@ const OnlineDrive = ({
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
const { setFileList, isTruncated, currentNextPageParametersRef, setHasBucket } = dataSourceStore.getState()
const { setOnlineDriveFileList, isTruncated, currentNextPageParametersRef, setHasBucket } = dataSourceStore.getState()
const {
fileList: newFileList,
isTruncated: newIsTruncated,
nextPageParameters: newNextPageParameters,
hasBucket: newHasBucket,
} = convertOnlineDriveData(documentsData.data, breadcrumbs, bucket)
setFileList([...fileList, ...newFileList])
setOnlineDriveFileList([...onlineDriveFileList, ...newFileList])
isTruncated.current = newIsTruncated
currentNextPageParametersRef.current = newNextPageParameters
setHasBucket(newHasBucket)
setIsLoading(false)
isLoadingRef.current = false
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
Toast.notify({
@ -100,6 +104,7 @@ const OnlineDrive = ({
message: error.error,
})
setIsLoading(false)
isLoadingRef.current = false
},
},
)
@ -109,7 +114,7 @@ const OnlineDrive = ({
if (!currentCredentialId) return
if (isInitialMount) {
// Only fetch files on initial mount if fileList is empty
if (fileList.length === 0)
if (onlineDriveFileList.length === 0)
getOnlineDriveFiles()
setIsInitialMount(false)
}
@ -118,11 +123,11 @@ const OnlineDrive = ({
}
}, [nextPageParameters, prefix, bucket, currentCredentialId])
const onlineDriveFileList = useMemo(() => {
const filteredOnlineDriveFileList = useMemo(() => {
if (keywords)
return fileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase()))
return fileList
}, [fileList, keywords])
return onlineDriveFileList.filter(file => file.name.toLowerCase().includes(keywords.toLowerCase()))
return onlineDriveFileList
}, [onlineDriveFileList, keywords])
const updateKeywords = useCallback((keywords: string) => {
const { setKeywords } = dataSourceStore.getState()
@ -152,9 +157,9 @@ const OnlineDrive = ({
}, [dataSourceStore, isInPipeline])
const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
const { breadcrumbs, setBreadcrumbs, setPrefix, setBucket, setFileList, setSelectedFileIds } = dataSourceStore.getState()
const { breadcrumbs, prefix, setBreadcrumbs, setPrefix, setBucket, setOnlineDriveFileList, setSelectedFileIds } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.file) return
setFileList([])
setOnlineDriveFileList([])
if (file.type === OnlineDriveFileType.bucket) {
setBucket(file.name)
}
@ -189,14 +194,14 @@ const OnlineDrive = ({
credentials={dataSourceAuth?.result || []}
/>
<FileList
fileList={onlineDriveFileList}
fileList={filteredOnlineDriveFileList}
selectedFileIds={selectedFileIds}
breadcrumbs={breadcrumbs}
keywords={keywords}
bucket={bucket}
resetKeywords={resetKeywords}
updateKeywords={updateKeywords}
searchResultsLength={onlineDriveFileList.length}
searchResultsLength={filteredOnlineDriveFileList.length}
handleSelectFile={handleSelectFile}
handleOpenFolder={handleOpenFolder}
isInPipeline={isInPipeline}

14
web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
View File

@ -10,8 +10,8 @@ export type OnlineDriveSliceShape = {
setKeywords: (keywords: string) => void
selectedFileIds: string[]
setSelectedFileIds: (selectedFileIds: string[]) => void
fileList: OnlineDriveFile[]
setFileList: (fileList: OnlineDriveFile[]) => void
onlineDriveFileList: OnlineDriveFile[]
setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => void
bucket: string
setBucket: (bucket: string) => void
nextPageParameters: Record<string, any>
@ -43,12 +43,12 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set,
selectedFileIds,
}))
const id = selectedFileIds[0]
const { fileList, previewOnlineDriveFileRef } = get()
previewOnlineDriveFileRef.current = fileList.find(file => file.id === id)
const { onlineDriveFileList, previewOnlineDriveFileRef } = get()
previewOnlineDriveFileRef.current = onlineDriveFileList.find(file => file.id === id)
},
fileList: [],
setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
fileList,
onlineDriveFileList: [],
setOnlineDriveFileList: (onlineDriveFileList: OnlineDriveFile[]) => set(() => ({
onlineDriveFileList,
})),
bucket: '',
setBucket: (bucket: string) => set(() => ({

20
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -65,7 +65,7 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
export const useLocalFile = () => {
const {
localFileList: fileList,
localFileList,
currentLocalFile,
} = useDataSourceStoreWithSelector(useShallow(state => ({
localFileList: state.localFileList,
@ -73,7 +73,7 @@ export const useLocalFile = () => {
})))
const dataSourceStore = useDataSourceStore()
const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
const allFileLoaded = useMemo(() => (localFileList.length > 0 && localFileList.every(file => file.file.id)), [localFileList])
const hidePreviewLocalFile = useCallback(() => {
const { setCurrentLocalFile } = dataSourceStore.getState()
@ -81,7 +81,7 @@ export const useLocalFile = () => {
}, [dataSourceStore])
return {
fileList,
localFileList,
allFileLoaded,
currentLocalFile,
hidePreviewLocalFile,
@ -187,27 +187,27 @@ export const useWebsiteCrawl = () => {
export const useOnlineDrive = () => {
const {
fileList,
onlineDriveFileList,
selectedFileIds,
} = useDataSourceStoreWithSelector(useShallow(state => ({
fileList: state.fileList,
onlineDriveFileList: state.onlineDriveFileList,
selectedFileIds: state.selectedFileIds,
})))
const dataSourceStore = useDataSourceStore()
const selectedOnlineDriveFileList = useMemo(() => {
return selectedFileIds.map(key => fileList.find(item => item.id === key)!)
}, [fileList, selectedFileIds])
return selectedFileIds.map(key => onlineDriveFileList.find(item => item.id === key)!)
}, [onlineDriveFileList, selectedFileIds])
const clearOnlineDriveData = useCallback(() => {
const {
setFileList,
setOnlineDriveFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileIds,
} = dataSourceStore.getState()
setFileList([])
setOnlineDriveFileList([])
setBucket('')
setPrefix([])
setKeywords('')
@ -215,7 +215,7 @@ export const useOnlineDrive = () => {
}, [dataSourceStore])
return {
fileList,
onlineDriveFileList,
selectedFileIds,
selectedOnlineDriveFileList,
clearOnlineDriveData,

14
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -61,7 +61,7 @@ const CreateFormPipeline = () => {
handleBackStep,
} = useAddDocumentsSteps()
const {
fileList,
localFileList,
allFileLoaded,
currentLocalFile,
hidePreviewLocalFile,
@ -81,7 +81,7 @@ const CreateFormPipeline = () => {
clearWebsiteCrawlData,
} = useWebsiteCrawl()
const {
fileList: onlineDriveFileList,
onlineDriveFileList,
selectedFileIds,
selectedOnlineDriveFileList,
clearOnlineDriveData,
@ -107,7 +107,7 @@ const CreateFormPipeline = () => {
const nextBtnDisabled = useMemo(() => {
if (!datasource) return true
if (datasourceType === DatasourceType.localFile)
return isShowVectorSpaceFull || !fileList.length || !allFileLoaded
return isShowVectorSpaceFull || !localFileList.length || !allFileLoaded
if (datasourceType === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasourceType === DatasourceType.websiteCrawl)
@ -115,7 +115,7 @@ const CreateFormPipeline = () => {
if (datasourceType === DatasourceType.onlineDrive)
return isShowVectorSpaceFull || !selectedFileIds.length
return false
}, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length])
}, [datasource, datasourceType, isShowVectorSpaceFull, localFileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileIds.length])
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
file_size_limit: 15,
@ -285,7 +285,7 @@ const CreateFormPipeline = () => {
const {
bucket,
selectedFileIds,
fileList: onlineDriveFileList,
onlineDriveFileList,
} = dataSourceStore.getState()
selectedFileIds.forEach((id) => {
const file = onlineDriveFileList.find(file => file.id === id)
@ -354,7 +354,7 @@ const CreateFormPipeline = () => {
const handleSelectAll = useCallback(() => {
const {
onlineDocuments,
fileList: onlineDriveFileList,
onlineDriveFileList,
selectedFileIds,
setOnlineDocuments,
setSelectedFileIds,
@ -535,7 +535,7 @@ const CreateFormPipeline = () => {
<div className='flex h-full flex-col pl-2 pt-2'>
<ChunkPreview
dataSourceType={datasourceType as DatasourceType}
localFiles={fileList.map(file => file.file)}
localFiles={localFileList.map(file => file.file)}
onlineDocuments={onlineDocuments}
websitePages={websitePages}
onlineDriveFiles={selectedOnlineDriveFileList}

4
web/app/components/rag-pipeline/components/panel/test-run/preparation/hooks.ts
View File

@ -110,13 +110,13 @@ export const useOnlineDrive = () => {
const clearOnlineDriveData = useCallback(() => {
const {
setFileList,
setOnlineDriveFileList,
setBucket,
setPrefix,
setKeywords,
setSelectedFileIds,
} = dataSourceStore.getState()
setFileList([])
setOnlineDriveFileList([])
setBucket('')
setPrefix([])
setKeywords('')

4
web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx
View File

@ -104,8 +104,8 @@ const Preparation = () => {
})
}
if (datasourceType === DatasourceType.onlineDrive) {
const { bucket, fileList, selectedFileIds } = dataSourceStore.getState()
const file = fileList.find(file => file.id === selectedFileIds[0])
const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState()
const file = onlineDriveFileList.find(file => file.id === selectedFileIds[0])
datasourceInfoList.push({
bucket,
id: file?.id,

4
web/app/components/workflow/nodes/data-source/hooks/use-before-run-form.ts
View File

@ -110,8 +110,8 @@ const useBeforeRunForm = ({
}
}
if (datasourceType === DatasourceType.onlineDrive) {
const { bucket, fileList, selectedFileIds } = dataSourceStore.getState()
const file = fileList.find(file => file.id === selectedFileIds[0])
const { bucket, onlineDriveFileList, selectedFileIds } = dataSourceStore.getState()
const file = onlineDriveFileList.find(file => file.id === selectedFileIds[0])
datasourceInfo = {
bucket,
id: file?.id,