{
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const prefix = useDataSourceStoreWithSelector(state => state.prefix)
- const setPrefix = useDataSourceStoreWithSelector(state => state.setPrefix)
const keywords = useDataSourceStoreWithSelector(state => state.keywords)
- const setKeywords = useDataSourceStoreWithSelector(state => state.setKeywords)
const bucket = useDataSourceStoreWithSelector(state => state.bucket)
- const setBucket = useDataSourceStoreWithSelector(state => state.setBucket)
const startAfter = useDataSourceStoreWithSelector(state => state.startAfter)
- const setStartAfter = useDataSourceStoreWithSelector(state => state.setStartAfter)
const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
- const setSelectedFileList = useDataSourceStoreWithSelector(state => state.setSelectedFileList)
const fileList = useDataSourceStoreWithSelector(state => state.fileList)
- const setFileList = useDataSourceStoreWithSelector(state => state.setFileList)
+ const isTruncated = useDataSourceStoreWithSelector(state => state.isTruncated)
+ const dataSourceStore = useDataSourceStore()
const [isLoading, setIsLoading] = useState(false)
const datasourceNodeRunURL = !isInPipeline
@@ -60,8 +56,10 @@ const OnlineDrive = ({
},
{
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
- const newFileList = convertOnlineDriveDataToFileList(documentsData.data)
+ const { setFileList, setIsTruncated } = dataSourceStore.getState()
+ const { fileList: newFileList, isTruncated } = convertOnlineDriveData(documentsData.data, prefix)
setFileList([...fileList, ...newFileList])
+ setIsTruncated(isTruncated)
setIsLoading(false)
},
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
@@ -73,7 +71,7 @@ const OnlineDrive = ({
},
},
)
- }, [bucket, datasourceNodeRunURL, prefix, fileList, setFileList, startAfter])
+ }, [prefix, datasourceNodeRunURL, bucket, startAfter, dataSourceStore, fileList])
useEffect(() => {
getOnlineDrive()
@@ -87,14 +85,18 @@ const OnlineDrive = ({
}, [fileList, keywords])
const updateKeywords = useCallback((keywords: string) => {
+ const { setKeywords } = dataSourceStore.getState()
setKeywords(keywords)
- }, [setKeywords])
+ }, [dataSourceStore])
const resetPrefix = useCallback(() => {
+ const { setKeywords } = dataSourceStore.getState()
+
setKeywords('')
- }, [setKeywords])
+ }, [dataSourceStore])
const handleSelectFile = useCallback((file: OnlineDriveFile) => {
+ const { setSelectedFileList } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.bucket) return
const newSelectedFileList = produce(selectedFileList, (draft) => {
if (draft.includes(file.key)) {
@@ -107,9 +109,10 @@ const OnlineDrive = ({
}
})
setSelectedFileList(newSelectedFileList)
- }, [isInPipeline, selectedFileList, setSelectedFileList])
+ }, [dataSourceStore, isInPipeline, selectedFileList])
const handleOpenFolder = useCallback((file: OnlineDriveFile) => {
+ const { setPrefix, setBucket, setFileList } = dataSourceStore.getState()
if (file.type === OnlineDriveFileType.file) return
setFileList([])
if (file.type === OnlineDriveFileType.bucket) {
@@ -122,7 +125,7 @@ const OnlineDrive = ({
})
setPrefix(newPrefix)
}
- }, [prefix, setBucket, setFileList, setPrefix])
+ }, [dataSourceStore, prefix])
return (
@@ -143,6 +146,7 @@ const OnlineDrive = ({
handleOpenFolder={handleOpenFolder}
isInPipeline={isInPipeline}
isLoading={isLoading}
+ isTruncated={isTruncated}
/>
)
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
index 47286850f7..105d8af632 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/utils.ts
@@ -7,14 +7,19 @@ export const isFile = (path: string): boolean => {
return filePathRegex.test(path)
}
-export const hasBuckets = (data: OnlineDriveData[]): boolean => {
+export const isBucketListInitiation = (data: OnlineDriveData[], prefix: string[]): boolean => {
+ if (prefix.length > 0) return false
return data.length > 1 || (data.length === 1 && data[0].files.length === 0)
}
-export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[]): OnlineDriveFile[] => {
+export const convertOnlineDriveData = (data: OnlineDriveData[], prefix: string[]): { fileList: OnlineDriveFile[], isTruncated: boolean } => {
const fileList: OnlineDriveFile[] = []
+ let isTruncated = false
- if (hasBuckets(data)) {
+ if (data.length === 0)
+ return { fileList, isTruncated }
+
+ if (isBucketListInitiation(data, prefix)) {
data.forEach((item) => {
fileList.push({
key: item.bucket,
@@ -34,6 +39,7 @@ export const convertOnlineDriveDataToFileList = (data: OnlineDriveData[]): Onlin
type: isFileType ? OnlineDriveFileType.file : OnlineDriveFileType.folder,
})
})
+ isTruncated = data[0].is_truncated ?? false
}
- return fileList
+ return { fileList, isTruncated }
}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
index 29f794aa13..aef30447d2 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts
@@ -14,6 +14,8 @@ export type OnlineDriveSliceShape = {
setFileList: (fileList: OnlineDriveFile[]) => void
bucket: string
setBucket: (bucket: string) => void
+ isTruncated: boolean
+ setIsTruncated: (isTruncated: boolean) => void
}
export const createOnlineDriveSlice: StateCreator
= (set) => {
@@ -42,5 +44,9 @@ export const createOnlineDriveSlice: StateCreator = (set)
setBucket: (bucket: string) => set(() => ({
bucket,
})),
+ isTruncated: false,
+ setIsTruncated: (isTruncated: boolean) => set(() => ({
+ isTruncated,
+ })),
})
}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
index 526a1159c8..05cbabcdbf 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
@@ -1,12 +1,10 @@
import { useTranslation } from 'react-i18next'
import { AddDocumentsStep } from './types'
import type { DataSourceOption } from '@/app/components/rag-pipeline/components/panel/test-run/types'
-import { useCallback, useMemo, useRef, useState } from 'react'
+import { useCallback, useMemo, useState } from 'react'
import { BlockEnum, type Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
-import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
-import { CrawlStep } from '@/models/datasets'
-import { useDataSourceStoreWithSelector } from './data-source/store'
+import { useDataSourceStore, useDataSourceStoreWithSelector } from './data-source/store'
export const useAddDocumentsSteps = () => {
const { t } = useTranslation()
@@ -66,13 +64,14 @@ export const useLocalFile = () => {
const fileList = useDataSourceStoreWithSelector(state => state.localFileList)
const previewFileRef = useDataSourceStoreWithSelector(state => state.previewLocalFileRef)
const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile)
- const setCurrentFile = useDataSourceStoreWithSelector(state => state.setCurrentLocalFile)
+ const dataSourceStore = useDataSourceStore()
const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
const hidePreviewLocalFile = useCallback(() => {
- setCurrentFile(undefined)
- }, [setCurrentFile])
+ const { setCurrentLocalFile } = dataSourceStore.getState()
+ setCurrentLocalFile(undefined)
+ }, [dataSourceStore])
return {
fileList,
@@ -86,12 +85,13 @@ export const useLocalFile = () => {
export const useOnlineDocuments = () => {
const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments)
const previewOnlineDocumentRef = useDataSourceStoreWithSelector(state => state.previewOnlineDocumentRef)
- const setCurrentDocument = useDataSourceStoreWithSelector(state => state.setCurrentDocument)
const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument)
+ const dataSourceStore = useDataSourceStore()
const hidePreviewOnlineDocument = useCallback(() => {
+ const { setCurrentDocument } = dataSourceStore.getState()
setCurrentDocument(undefined)
- }, [setCurrentDocument])
+ }, [dataSourceStore])
return {
onlineDocuments,
@@ -102,44 +102,31 @@ export const useOnlineDocuments = () => {
}
export const useWebsiteCrawl = () => {
- const [websitePages, setWebsitePages] = useState([])
- const [currentWebsite, setCurrentWebsite] = useState()
- const [crawlResult, setCrawlResult] = useState()
- const [step, setStep] = useState(CrawlStep.init)
- const [previewIndex, setPreviewIndex] = useState(-1)
-
- const previewWebsitePage = useRef(websitePages[0])
-
- const updateCurrentWebsite = useCallback((website: CrawlResultItem, index: number) => {
- setCurrentWebsite(website)
- setPreviewIndex(index)
- }, [])
+ const websitePages = useDataSourceStoreWithSelector(state => state.websitePages)
+ const currentWebsite = useDataSourceStoreWithSelector(state => state.currentWebsite)
+ const previewWebsitePageRef = useDataSourceStoreWithSelector(state => state.previewWebsitePageRef)
+ const dataSourceStore = useDataSourceStore()
const hideWebsitePreview = useCallback(() => {
+ const { setCurrentWebsite, setPreviewIndex } = dataSourceStore.getState()
setCurrentWebsite(undefined)
setPreviewIndex(-1)
- }, [])
-
- const updataCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
- setWebsitePages(checkedCrawlResult)
- previewWebsitePage.current = checkedCrawlResult[0]
- }, [])
+ }, [dataSourceStore])
return {
websitePages,
- crawlResult,
- setCrawlResult,
- step,
- setStep,
- previewWebsitePage,
- updataCheckedCrawlResultChange,
+ previewWebsitePageRef,
currentWebsite,
- updateCurrentWebsite,
- previewIndex,
hideWebsitePreview,
}
}
export const useOnlineDrive = () => {
- return {}
+ const fileList = useDataSourceStoreWithSelector(state => state.fileList)
+ const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList)
+
+ return {
+ fileList,
+ selectedFileList,
+ }
}
diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx
index 69d36b5365..8db4014475 100644
--- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx
@@ -27,8 +27,9 @@ import Processing from './processing'
import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline'
import { TransferMethod } from '@/types/app'
-import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
+import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
import DataSourceProvider from './data-source/store/provider'
+import { useDataSourceStore } from './data-source/store'
const CreateFormPipeline = () => {
const { t } = useTranslation()
@@ -39,6 +40,7 @@ const CreateFormPipeline = () => {
const [estimateData, setEstimateData] = useState(undefined)
const [batchId, setBatchId] = useState('')
const [documents, setDocuments] = useState([])
+ const dataSourceStore = useDataSourceStore()
const isPreview = useRef(false)
const formRef = useRef(null)
@@ -66,27 +68,44 @@ const CreateFormPipeline = () => {
} = useOnlineDocuments()
const {
websitePages,
- previewWebsitePage,
+ previewWebsitePageRef,
currentWebsite,
hideWebsitePreview,
} = useWebsiteCrawl()
- // const { } = useOnlineDrive()
+ const {
+ fileList: onlineDriveFileList,
+ selectedFileList,
+ } = useOnlineDrive()
- const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
- const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
- const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
const datasourceType = datasource?.nodeData.provider_type
+ const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
+ const isShowVectorSpaceFull = useMemo(() => {
+ if (!datasource)
+ return false
+ if (datasourceType === DatasourceType.localFile)
+ return allFileLoaded && isVectorSpaceFull && enableBilling
+ if (datasourceType === DatasourceType.onlineDocument)
+ return onlineDocuments.length > 0 && isVectorSpaceFull && enableBilling
+ if (datasourceType === DatasourceType.websiteCrawl)
+ return websitePages.length > 0 && isVectorSpaceFull && enableBilling
+ if (datasourceType === DatasourceType.onlineDrive)
+ return onlineDriveFileList.length > 0 && isVectorSpaceFull && enableBilling
+ return false
+ }, [allFileLoaded, datasource, datasourceType, enableBilling, isVectorSpaceFull, onlineDocuments.length, onlineDriveFileList.length, websitePages.length])
+ const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
const nextBtnDisabled = useMemo(() => {
if (!datasource) return true
if (datasourceType === DatasourceType.localFile)
- return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
+ return isShowVectorSpaceFull || !fileList.length || !allFileLoaded
if (datasourceType === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasourceType === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length
+ if (datasourceType === DatasourceType.onlineDrive)
+ return isShowVectorSpaceFull || !selectedFileList.length
return false
- }, [datasource, datasourceType, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
+ }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileList.length])
const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
@@ -117,7 +136,7 @@ const CreateFormPipeline = () => {
datasourceInfoList.push(documentInfo)
}
if (datasourceType === DatasourceType.websiteCrawl)
- datasourceInfoList.push(previewWebsitePage.current)
+ datasourceInfoList.push(previewWebsitePageRef.current!)
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
@@ -130,7 +149,7 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
- }, [datasource, datasourceType, pipelineId, previewFileRef, previewOnlineDocumentRef, previewWebsitePage, runPublishedPipeline])
+ }, [datasource, datasourceType, previewWebsitePageRef, runPublishedPipeline, pipelineId, previewFileRef, previewOnlineDocumentRef])
const handleProcess = useCallback(async (data: Record) => {
if (!datasource)
@@ -167,6 +186,17 @@ const CreateFormPipeline = () => {
datasourceInfoList.push(websitePage)
})
}
+ if (datasourceType === DatasourceType.onlineDrive) {
+ if (datasourceType === DatasourceType.onlineDrive) {
+ const { bucket } = dataSourceStore.getState()
+ selectedFileList.forEach((key) => {
+ datasourceInfoList.push({
+ bucket,
+ key,
+ })
+ })
+ }
+ }
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
@@ -181,7 +211,7 @@ const CreateFormPipeline = () => {
handleNextStep()
},
})
- }, [datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
+ }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileList, websitePages])
const onClickProcess = useCallback(() => {
isPreview.current = false
@@ -208,9 +238,9 @@ const CreateFormPipeline = () => {
}, [onClickPreview, previewOnlineDocumentRef])
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
- previewWebsitePage.current = website
+ previewWebsitePageRef.current = website
onClickPreview()
- }, [onClickPreview, previewWebsitePage])
+ }, [onClickPreview, previewWebsitePageRef])
if (isFetchingPipelineInfo) {
return (
diff --git a/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx b/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx
deleted file mode 100644
index 4b40be2c7f..0000000000
--- a/web/app/components/datasets/list/dataset-card/dataset-card-deprecated.tsx
+++ /dev/null
@@ -1,240 +0,0 @@
-'use client'
-
-import { useContext } from 'use-context-selector'
-import { useRouter } from 'next/navigation'
-import { useCallback, useEffect, useState } from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiMoreFill } from '@remixicon/react'
-import cn from '@/utils/classnames'
-import Confirm from '@/app/components/base/confirm'
-import { ToastContext } from '@/app/components/base/toast'
-import { checkIsUsedInApp, deleteDataset } from '@/service/datasets'
-import type { DataSet } from '@/models/datasets'
-import Tooltip from '@/app/components/base/tooltip'
-import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
-import type { HtmlContentProps } from '@/app/components/base/popover'
-import CustomPopover from '@/app/components/base/popover'
-import Divider from '@/app/components/base/divider'
-import RenameDatasetModal from '@/app/components/datasets/rename-modal'
-import type { Tag } from '@/app/components/base/tag-management/constant'
-import TagSelector from '@/app/components/base/tag-management/selector'
-import CornerLabel from '@/app/components/base/corner-label'
-import { useAppContext } from '@/context/app-context'
-
-export type DatasetCardProps = {
- dataset: DataSet
- onSuccess?: () => void
-}
-
-const DatasetCard = ({
- dataset,
- onSuccess,
-}: DatasetCardProps) => {
- const { t } = useTranslation()
- const { notify } = useContext(ToastContext)
- const { push } = useRouter()
- const EXTERNAL_PROVIDER = 'external' as const
-
- const { isCurrentWorkspaceDatasetOperator } = useAppContext()
- const [tags, setTags] = useState(dataset.tags)
-
- const [showRenameModal, setShowRenameModal] = useState(false)
- const [showConfirmDelete, setShowConfirmDelete] = useState(false)
- const [confirmMessage, setConfirmMessage] = useState('')
- const isExternalProvider = (provider: string): boolean => provider === EXTERNAL_PROVIDER
- const detectIsUsedByApp = useCallback(async () => {
- try {
- const { is_using: isUsedByApp } = await checkIsUsedInApp(dataset.id)
- setConfirmMessage(isUsedByApp ? t('dataset.datasetUsedByApp')! : t('dataset.deleteDatasetConfirmContent')!)
- }
- catch (e: any) {
- const res = await e.json()
- notify({ type: 'error', message: res?.message || 'Unknown error' })
- }
-
- setShowConfirmDelete(true)
- }, [dataset.id, notify, t])
- const onConfirmDelete = useCallback(async () => {
- try {
- await deleteDataset(dataset.id)
- notify({ type: 'success', message: t('dataset.datasetDeleted') })
- if (onSuccess)
- onSuccess()
- }
- catch {
- }
- setShowConfirmDelete(false)
- }, [dataset.id, notify, onSuccess, t])
-
- const Operations = (props: HtmlContentProps & { showDelete: boolean }) => {
- const onMouseLeave = async () => {
- props.onClose?.()
- }
- const onClickRename = async (e: React.MouseEvent) => {
- e.stopPropagation()
- props.onClick?.()
- e.preventDefault()
- setShowRenameModal(true)
- }
- const onClickDelete = async (e: React.MouseEvent) => {
- e.stopPropagation()
- props.onClick?.()
- e.preventDefault()
- detectIsUsedByApp()
- }
- return (
-
-
- {t('common.operation.settings')}
-
- {props.showDelete && (
- <>
-
-
-
- {t('common.operation.delete')}
-
-
-
- )}
-
{
- e.preventDefault()
- isExternalProvider(dataset.provider)
- ? push(`/datasets/${dataset.id}/hitTesting`)
- : push(`/datasets/${dataset.id}/documents`)
- }}
- >
- {isExternalProvider(dataset.provider) &&
}
-
-
-
-
-
-
-
{dataset.name}
- {!dataset.embedding_available && (
-
- {t('dataset.unavailable')}
-
- )}
-
-
-
- {dataset.provider === 'external'
- ? <>
- {dataset.app_count}{t('dataset.appCount')}
-
- : <>
- {dataset.document_count}{t('dataset.documentCount')}
- ·
- {Math.round(dataset.word_count / 1000)}{t('dataset.wordCount')}
- ·
- {dataset.app_count}{t('dataset.appCount')}
-
- }
-
-
-
-
-
- {dataset.description}
-
-
-
{
- e.stopPropagation()
- e.preventDefault()
- }}>
-
- tag.id)}
- selectedTags={tags}
- onCacheUpdate={setTags}
- onChange={onSuccess}
- />
-
-
-
-
-
}
- position="br"
- trigger="click"
- btnElement={
-
-
-
- }
- btnClassName={open =>
- cn(
- open ? '!bg-black/5 !shadow-none' : '!bg-transparent',
- 'h-8 w-8 rounded-md border-none !p-2 hover:!bg-black/5',
- )
- }
- className={'!z-20 h-fit !w-[128px]'}
- />
-
-
-