diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
index ffe443b8ab..81cbeb08e1 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx
@@ -21,7 +21,7 @@ const Breadcrumbs = ({
isInPipeline,
}: BreadcrumbsProps) => {
const { t } = useTranslation()
- const { setFileList, setSelectedFileList, setPrefix, setBucket } = useDataSourceStore().getState()
+ const dataSourceStore = useDataSourceStore()
const showSearchResult = !!keywords && searchResultsLength > 0
const isRoot = prefix.length === 0 && bucket === ''
@@ -43,24 +43,27 @@ const Breadcrumbs = ({
}, [displayBreadcrumbNum, prefix])
const handleBackToBucketList = useCallback(() => {
+ const { setFileList, setSelectedFileList, setPrefix, setBucket } = dataSourceStore.getState()
setFileList([])
setSelectedFileList([])
setBucket('')
setPrefix([])
- }, [setBucket, setFileList, setPrefix, setSelectedFileList])
+ }, [dataSourceStore])
const handleClickBucketName = useCallback(() => {
+ const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState()
setFileList([])
setSelectedFileList([])
setPrefix([])
- }, [setFileList, setPrefix, setSelectedFileList])
+ }, [dataSourceStore])
const handleClickBreadcrumb = useCallback((index: number) => {
+ const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState()
const newPrefix = prefix.slice(0, index - 1)
setFileList([])
setSelectedFileList([])
setPrefix(newPrefix)
- }, [prefix, setFileList, setPrefix, setSelectedFileList])
+ }, [dataSourceStore, prefix])
return (
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
index 81b0fb45f2..c47509fb63 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx
@@ -17,6 +17,7 @@ type FileListProps = {
handleSelectFile: (file: OnlineDriveFile) => void
handleOpenFolder: (file: OnlineDriveFile) => void
isLoading: boolean
+ isTruncated: boolean
}
const FileList = ({
@@ -32,6 +33,7 @@ const FileList = ({
handleOpenFolder,
isInPipeline,
isLoading,
+ isTruncated,
}: FileListProps) => {
const [inputValue, setInputValue] = useState(keywords)
@@ -74,6 +76,7 @@ const FileList = ({
handleSelectFile={handleSelectFile}
isInPipeline={isInPipeline}
isLoading={isLoading}
+ isTruncated={isTruncated}
/>
)
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
index 570ccca301..b4a9534367 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/list/index.tsx
@@ -1,4 +1,4 @@
-import React, { useMemo } from 'react'
+import React, { useEffect, useMemo, useRef } from 'react'
import type { OnlineDriveFile } from '@/models/pipeline'
import Item from './item'
import EmptyFolder from './empty-folder'
@@ -8,12 +8,14 @@ import { RiLoader2Line } from '@remixicon/react'
import { useFileSupportTypes } from '@/service/use-common'
import { isFile } from '../../utils'
import { getFileExtension } from './utils'
+import { useDataSourceStore } from '../../../store'
type FileListProps = {
fileList: OnlineDriveFile[]
selectedFileList: string[]
keywords: string
isInPipeline: boolean
+ isTruncated: boolean
isLoading: boolean
handleResetKeywords: () => void
handleSelectFile: (file: OnlineDriveFile) => void
@@ -29,7 +31,26 @@ const List = ({
handleOpenFolder,
isInPipeline,
isLoading,
+ isTruncated,
}: FileListProps) => {
+ const anchorRef = useRef(null)
+ const observerRef = useRef()
+ const dataSourceStore = useDataSourceStore()
+
+ useEffect(() => {
+ if (anchorRef.current) {
+ observerRef.current = new IntersectionObserver((entries) => {
+ const { setStartAfter } = dataSourceStore.getState()
+ if (entries[0].isIntersecting && isTruncated && !isLoading)
+ setStartAfter(fileList[fileList.length - 1].key)
+ }, {
+ rootMargin: '100px',
+ })
+ observerRef.current.observe(anchorRef.current)
+ }
+ return () => observerRef.current?.disconnect()
+ }, [anchorRef, dataSourceStore, isTruncated, isLoading, fileList])
+
const isAllLoading = isLoading && fileList.length === 0 && keywords.length === 0
const isPartLoading = isLoading && fileList.length > 0
const isEmptyFolder = !isLoading && fileList.length === 0 && keywords.length === 0
@@ -84,6 +105,7 @@ const List = ({
)
}
+
)}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
index 3674031f42..aea1fee267 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx
@@ -8,8 +8,8 @@ import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { ssePost } from '@/service/base'
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
import Toast from '@/app/components/base/toast'
-import { useDataSourceStoreWithSelector } from '../store'
-import { convertOnlineDriveDataToFileList } from './utils'
+import { useDataSourceStore, useDataSourceStoreWithSelector } from '../store'
+import { convertOnlineDriveData } from './utils'
import produce from 'immer'
type OnlineDriveProps = {
@@ -25,17 +25,13 @@ const OnlineDrive = ({
}: OnlineDriveProps) => {
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const prefix = useDataSourceStoreWithSelector(state => state.prefix)
- const setPrefix = useDataSourceStoreWithSelector(state => state.setPrefix)
const keywords = useDataSourceStoreWithSelector(state => state.keywords)
- const setKeywords = useDataSourceStoreWithSelector(state => state.setKeywords)
const bucket = useDataSourceStoreWithSelector(state => state.bucket)
- const setBucket = useDataSourceStoreWithSelector(state => state.setBucket)
const startAfter = useDataSourceStoreWithSelector(state => state.startAfter)
- const setStartAfter = useDataSourceStoreWithSelector(state => state.setStartAfter)
const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
- const setSelectedFileList = useDataSourceStoreWithSelector(state => state.setSelectedFileList)
const fileList = useDataSourceStoreWithSelector(state => state.fileList)
- const setFileList = useDataSourceStoreWithSelector(state => state.setFileList)
+ const isTruncated = useDataSourceStoreWithSelector(state => state.isTruncated)
+ const dataSourceStore = useDataSourceStore()
const [isLoading, setIsLoading] = useState(false)
const datasourceNodeRunURL = !isInPipeline
@@ -60,8 +56,10 @@ const OnlineDrive = ({
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
- const newFileList = convertOnlineDriveDataToFileList(documentsData.data, prefix)
+ const { setFileList, setIsTruncated } = dataSourceStore.getState()
+ const { fileList: newFileList, isTruncated } = convertOnlineDriveData(documentsData.data, prefix)
setFileList([...fileList, ...newFileList])
+ setIsTruncated(isTruncated)
setIsLoading(false)
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
@@ -73,7 +71,7 @@ const OnlineDrive = ({
},
},
)
- }, [bucket, datasourceNodeRunURL, prefix, fileList, setFileList, startAfter])
+ }, [prefix, datasourceNodeRunURL, bucket, startAfter, dataSourceStore, fileList])
useEffect(() => {
getOnlineDrive()
@@ -87,14 +85,18 @@ const OnlineDrive = ({
}, [fileList, keywords])
const updateKeywords = useCallback((keywords: string) => {
+ const { setKeywords } = dataSourceStore.getState()
setKeywords(keywords)
- }, [setKeywords])
+ }, [dataSourceStore])
const resetPrefix = useCallback(() => {
+ const { setKeywords } = dataSourceStore.getState()
+
setKeywords('')
- }, [setKeywords])
+ }, [dataSourceStore])
const handleSelectFile = useCallback((file: OnlineDriveFile) => {
+ const { setSelectedFileList } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.bucket) return
const newSelectedFileList = produce(selectedFileList, (draft) => {
if (draft.includes(file.key)) {
@@ -107,9 +109,10 @@ const OnlineDrive = ({
}
})
setSelectedFileList(newSelectedFileList)
- }, [isInPipeline, selectedFileList, setSelectedFileList])
+ }, [dataSourceStore, isInPipeline, selectedFileList])
const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
+ const { setPrefix, setBucket, setFileList } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.file) return
setFileList([])
if (file.type === OnlineDriveFileType.bucket) {
@@ -122,7 +125,7 @@ const OnlineDrive = ({
})
setPrefix(newPrefix)
}
- }, [prefix, setBucket, setFileList, setPrefix])
+ }, [dataSourceStore, prefix])
return (
@@ -143,6 +146,7 @@ const OnlineDrive = ({
handleOpenFolder={handleOpenFolder}
isInPipeline={isInPipeline}
isLoading={isLoading}
+ isTruncated={isTruncated}
/>
)
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
index 24c28618bd..105d8af632 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
@@ -12,11 +12,12 @@ export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[]
return data.length > 1 || (data.length === 1 && data[0].files.length === 0)
}
-export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[], prefix: string[]): OnlineDriveFile[] => {
+export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[]): { fileList: OnlineDriveFile[], isTruncated: boolean } => {
const fileList: OnlineDriveFile[] = []
+ let isTruncated = false
if (data.length === 0)
- return fileList
+ return { fileList, isTruncated }
if (isBucketListInitiation(data, prefix)) {
data.forEach((item) => {
@@ -38,6 +39,7 @@ export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[], prefix
type: isFileType ? OnlineDriveFileType.file : OnlineDriveFileType.folder,
})
})
+ isTruncated = data[0].is_truncated ?? false
}
- return fileList
+ return { fileList, isTruncated }
}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
index 29f794aa13..aef30447d2 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
@@ -14,6 +14,8 @@ export type OnlineDriveSliceShape = {
setFileList: (fileList: OnlineDriveFile[]) => void
bucket: string
setBucket: (bucket: string) => void
+ isTruncated: boolean
+ setIsTruncated: (isTruncated: boolean) => void
}
export const createOnlineDriveSlice: StateCreator = (set) => {
@@ -42,5 +44,9 @@ export const createOnlineDriveSlice: StateCreator = (set)
setBucket: (bucket: string) => set(() => ({
bucket,
})),
+ isTruncated: false,
+ setIsTruncated: (isTruncated: boolean) => set(() => ({
+ isTruncated,
+ })),
})
}
diff --git a/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx b/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx
deleted file mode 100644
index 4b40be2c7f..0000000000
--- a/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx
+++ /dev/null
@@ -1,240 +0,0 @@
-'use client'
-
-import { useContext } from 'use-context-selector'
-import { useRouter } from 'next/navigation'
-import { useCallback, useEffect, useState } from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiMoreFill } from '@remixicon/react'
-import cn from '@/utils/classnames'
-import Confirm from '@/app/components/base/confirm'
-import { ToastContext } from '@/app/components/base/toast'
-import { checkIsUsedInApp, deleteDataset } from '@/service/datasets'
-import type { DataSet } from '@/models/datasets'
-import Tooltip from '@/app/components/base/tooltip'
-import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
-import type { HtmlContentProps } from '@/app/components/base/popover'
-import CustomPopover from '@/app/components/base/popover'
-import Divider from '@/app/components/base/divider'
-import RenameDatasetModal from '@/app/components/datasets/rename-modal'
-import type { Tag } from '@/app/components/base/tag-management/constant'
-import TagSelector from '@/app/components/base/tag-management/selector'
-import CornerLabel from '@/app/components/base/corner-label'
-import { useAppContext } from '@/context/app-context'
-
-export type DatasetCardProps = {
- dataset: DataSet
- onSuccess?: () => void
-}
-
-const DatasetCard = ({
- dataset,
- onSuccess,
-}: DatasetCardProps) => {
- const { t } = useTranslation()
- const { notify } = useContext(ToastContext)
- const { push } = useRouter()
- const EXTERNAL_PROVIDER = 'external' as const
-
- const { isCurrentWorkspaceDatasetOperator } = useAppContext()
- const [tags, setTags] = useState(dataset.tags)
-
- const [showRenameModal, setShowRenameModal] = useState(false)
- const [showConfirmDelete, setShowConfirmDelete] = useState(false)
- const [confirmMessage, setConfirmMessage] = useState('')
- const isExternalProvider = (provider: string): boolean => provider === EXTERNAL_PROVIDER
- const detectIsUsedByApp = useCallback(async () => {
- try {
- const { is_using: isUsedByApp } = await checkIsUsedInApp(dataset.id)
- setConfirmMessage(isUsedByApp ? t('dataset.datasetUsedByApp')! : t('dataset.deleteDatasetConfirmContent')!)
- }
- catch (e: any) {
- const res = await e.json()
- notify({ type: 'error', message: res?.message || 'Unknown error' })
- }
-
- setShowConfirmDelete(true)
- }, [dataset.id, notify, t])
- const onConfirmDelete = useCallback(async () => {
- try {
- await deleteDataset(dataset.id)
- notify({ type: 'success', message: t('dataset.datasetDeleted') })
- if (onSuccess)
- onSuccess()
- }
- catch {
- }
- setShowConfirmDelete(false)
- }, [dataset.id, notify, onSuccess, t])
-
- const Operations = (props: HtmlContentProps & { showDelete: boolean }) => {
- const onMouseLeave = async () => {
- props.onClose?.()
- }
- const onClickRename = async (e: React.MouseEvent) => {
- e.stopPropagation()
- props.onClick?.()
- e.preventDefault()
- setShowRenameModal(true)
- }
- const onClickDelete = async (e: React.MouseEvent) => {
- e.stopPropagation()
- props.onClick?.()
- e.preventDefault()
- detectIsUsedByApp()
- }
- return (
-
-
- {t('common.operation.settings')}
-
- {props.showDelete && (
- <>
-
-
-
- {t('common.operation.delete')}
-
-
- >
- )}
-
- )
- }
-
- useEffect(() => {
- setTags(dataset.tags)
- }, [dataset])
-
- return (
- <>
- {
- e.preventDefault()
- isExternalProvider(dataset.provider)
- ? push(`/datasets/${dataset.id}/hitTesting`)
- : push(`/datasets/${dataset.id}/documents`)
- }}
- >
- {isExternalProvider(dataset.provider) &&
}
-
-
-
-
-
-
-
{dataset.name}
- {!dataset.embedding_available && (
-
- {t('dataset.unavailable')}
-
- )}
-
-
-
- {dataset.provider === 'external'
- ? <>
- {dataset.app_count}{t('dataset.appCount')}
- >
- : <>
- {dataset.document_count}{t('dataset.documentCount')}
- ·
- {Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}
- ·
- {dataset.app_count}{t('dataset.appCount')}
- >
- }
-
-
-
-
-
- {dataset.description}
-
-
-
{
- e.stopPropagation()
- e.preventDefault()
- }}>
-
- tag.id)}
- selectedTags={tags}
- onCacheUpdate={setTags}
- onChange={onSuccess}
- />
-
-
-
-
-
}
- position="br"
- trigger="click"
- btnElement={
-
-
-
- }
- btnClassName={open =>
- cn(
- open ? '!bg-black/5 !shadow-none' : '!bg-transparent',
- 'h-8 w-8 rounded-md border-none !p-2 hover:!bg-black/5',
- )
- }
- className={'!z-20 h-fit !w-[128px]'}
- />
-
-
-
- {showRenameModal && (
- setShowRenameModal(false)}
- onSuccess={onSuccess}
- />
- )}
- {showConfirmDelete && (
- setShowConfirmDelete(false)}
- />
- )}
- >
- )
-}
-
-export default DatasetCard
diff --git a/web/app/components/datasets/list/datasets.tsx b/web/app/components/datasets/list/datasets.tsx
index 3265279c9f..70d772466a 100644
--- a/web/app/components/datasets/list/datasets.tsx
+++ b/web/app/components/datasets/list/datasets.tsx
@@ -34,19 +34,17 @@ const Datasets = ({
keyword: keywords,
})
const resetDatasetList = useResetDatasetList()
- const loadingStateRef = useRef(false)
const anchorRef = useRef(null)
const observerRef = useRef()
useEffect(() => {
- loadingStateRef.current = isFetching
document.title = `${t('dataset.knowledge')} - Dify`
- }, [isFetching, t])
+ }, [t])
useEffect(() => {
if (anchorRef.current) {
observerRef.current = new IntersectionObserver((entries) => {
- if (entries[0].isIntersecting && hasNextPage)
+ if (entries[0].isIntersecting && hasNextPage && !isFetching)
fetchNextPage()
}, {
rootMargin: '100px',
@@ -54,7 +52,7 @@ const Datasets = ({
observerRef.current.observe(anchorRef.current)
}
return () => observerRef.current?.disconnect()
- }, [anchorRef, datasetList, hasNextPage, fetchNextPage])
+ }, [anchorRef, datasetList, hasNextPage, fetchNextPage, isFetching])
return (
<>
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
index fe6c778319..4fc09a61ee 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
@@ -24,7 +24,7 @@ const TestRunPanel = () => {
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
- const { bucket } = useDataSourceStore().getState()
+ const dataSourceStore = useDataSourceStore()
const [datasource, setDatasource] = useState()
const {
@@ -84,6 +84,7 @@ const TestRunPanel = () => {
if (datasourceType === DatasourceType.websiteCrawl)
datasourceInfoList.push(websitePages[0])
if (datasourceType === DatasourceType.onlineDrive) {
+ const { bucket } = dataSourceStore.getState()
datasourceInfoList.push({
bucket,
key: selectedFileList[0],
@@ -95,7 +96,7 @@ const TestRunPanel = () => {
datasource_type: datasourceType,
datasource_info_list: datasourceInfoList,
})
- }, [bucket, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileList, websitePages])
+ }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileList, websitePages])
return (