diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx index 7752b754d2..6e01f2123b 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/header/breadcrumbs/index.tsx @@ -51,9 +51,9 @@ const Breadcrumbs = ({ }, [displayBreadcrumbNum, prefix]) const handleBackToBucketList = useCallback(() => { - const { setFileList, setSelectedFileList, setPrefix, setBucket } = dataSourceStore.getState() + const { setFileList, setSelectedFileKeys, setPrefix, setBucket } = dataSourceStore.getState() setFileList([]) - setSelectedFileList([]) + setSelectedFileKeys([]) setBucket('') setPrefix([]) getOnlineDriveFiles({ @@ -64,9 +64,9 @@ const Breadcrumbs = ({ }, [dataSourceStore, getOnlineDriveFiles]) const handleClickBucketName = useCallback(() => { - const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState() + const { setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState() setFileList([]) - setSelectedFileList([]) + setSelectedFileKeys([]) setPrefix([]) getOnlineDriveFiles({ prefix: [], @@ -75,16 +75,16 @@ const Breadcrumbs = ({ }, [dataSourceStore, getOnlineDriveFiles]) const handleClickBreadcrumb = useCallback((index: number) => { - const { setFileList, setSelectedFileList, setPrefix } = dataSourceStore.getState() + const { prefix, setFileList, setSelectedFileKeys, setPrefix } = dataSourceStore.getState() const newPrefix = prefix.slice(0, index + 1) setFileList([]) - setSelectedFileList([]) + setSelectedFileKeys([]) setPrefix(newPrefix) getOnlineDriveFiles({ prefix: newPrefix, fileList: [], }) - }, [dataSourceStore, getOnlineDriveFiles, prefix]) + }, [dataSourceStore, getOnlineDriveFiles]) return (
diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx index cbe565e56d..a8aa23066b 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/file-list/index.tsx @@ -6,7 +6,7 @@ import { useDebounceFn } from 'ahooks' type FileListProps = { fileList: OnlineDriveFile[] - selectedFileList: string[] + selectedFileKeys: string[] prefix: string[] keywords: string bucket: string @@ -27,7 +27,7 @@ type FileListProps = { const FileList = ({ fileList, - selectedFileList, + selectedFileKeys, prefix, keywords, bucket, @@ -75,7 +75,7 @@ const FileList = ({ /> { fileList.map((file) => { - const isSelected = selectedFileList.includes(file.key) + const isSelected = selectedFileKeys.includes(file.key) return ( state.prefix) const keywords = useDataSourceStoreWithSelector(state => state.keywords) const bucket = useDataSourceStoreWithSelector(state => state.bucket) - const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) + const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys) const fileList = useDataSourceStoreWithSelector(state => state.fileList) const currentNodeIdRef = useDataSourceStoreWithSelector(state => state.currentNodeIdRef) const dataSourceStore = useDataSourceStore() @@ -43,7 +43,7 @@ const OnlineDrive = ({ startAfter?: string fileList?: OnlineDriveFile[] }) => { - const { startAfter } = dataSourceStore.getState() + const { startAfter, prefix, bucket, fileList } = dataSourceStore.getState() const _prefix = params.prefix ?? prefix const _bucket = params.bucket ?? bucket const _startAfter = params.startAfter ?? startAfter.current @@ -80,16 +80,22 @@ const OnlineDrive = ({ }, }, ) - }, [prefix, bucket, datasourceNodeRunURL, dataSourceStore, fileList]) + }, [datasourceNodeRunURL, dataSourceStore]) useEffect(() => { if (nodeId !== currentNodeIdRef.current) { - const { setFileList, setBucket, setPrefix, setKeywords, setSelectedFileList } = dataSourceStore.getState() + const { + setFileList, + setBucket, + setPrefix, + setKeywords, + setSelectedFileKeys, + } = dataSourceStore.getState() setFileList([]) setBucket('') setPrefix([]) setKeywords('') - setSelectedFileList([]) + setSelectedFileKeys([]) currentNodeIdRef.current = nodeId getOnlineDriveFiles({ prefix: [], @@ -124,9 +130,9 @@ const OnlineDrive = ({ }, [dataSourceStore]) const handleSelectFile = useCallback((file: OnlineDriveFile) => { - const { setSelectedFileList } = dataSourceStore.getState() + const { selectedFileKeys, setSelectedFileKeys } = dataSourceStore.getState() if (file.type === OnlineDriveFileType.bucket) return - const newSelectedFileList = produce(selectedFileList, (draft) => { + const newSelectedFileList = produce(selectedFileKeys, (draft) => { if (draft.includes(file.key)) { const index = draft.indexOf(file.key) draft.splice(index, 1) @@ -136,11 +142,11 @@ const OnlineDrive = ({ draft.push(file.key) } }) - setSelectedFileList(newSelectedFileList) - }, [dataSourceStore, isInPipeline, selectedFileList]) + setSelectedFileKeys(newSelectedFileList) + }, [dataSourceStore, isInPipeline]) const handleOpenFolder = useCallback((file: OnlineDriveFile) => { - const { setPrefix, setBucket, setFileList, setSelectedFileList } = dataSourceStore.getState() + const { prefix, setPrefix, setBucket, setFileList, setSelectedFileKeys } = dataSourceStore.getState() if (file.type === OnlineDriveFileType.file) return setFileList([]) if (file.type === OnlineDriveFileType.bucket) { @@ -148,7 +154,7 @@ const OnlineDrive = ({ getOnlineDriveFiles({ bucket: file.displayName, fileList: [] }) } else { - setSelectedFileList([]) + setSelectedFileKeys([]) const displayName = file.displayName.endsWith('/') ? file.displayName.slice(0, -1) : file.displayName const newPrefix = produce(prefix, (draft) => { draft.push(displayName) @@ -156,7 +162,7 @@ const OnlineDrive = ({ setPrefix(newPrefix) getOnlineDriveFiles({ prefix: newPrefix, fileList: [] }) } - }, [dataSourceStore, getOnlineDriveFiles, prefix]) + }, [dataSourceStore, getOnlineDriveFiles]) return (
@@ -166,7 +172,7 @@ const OnlineDrive = ({ /> } -export const createOnlineDocumentSlice: StateCreator = (set) => { +export const createOnlineDocumentSlice: StateCreator = (set, get) => { return ({ documentsData: [], setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({ @@ -32,9 +32,13 @@ export const createOnlineDocumentSlice: StateCreator = currentWorkspaceId: workspaceId, })), onlineDocuments: [], - setOnlineDocuments: (documents: NotionPage[]) => set(() => ({ - onlineDocuments: documents, - })), + setOnlineDocuments: (documents: NotionPage[]) => { + set(() => ({ + onlineDocuments: documents, + })) + const { previewOnlineDocumentRef } = get() + previewOnlineDocumentRef.current = documents[0] + }, currentDocument: undefined, setCurrentDocument: (document: NotionPage | undefined) => set(() => ({ currentDocument: document, diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts index 186f23fcc4..46d83599ab 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts @@ -6,17 +6,18 @@ export type OnlineDriveSliceShape = { setPrefix: (prefix: string[]) => void keywords: string setKeywords: (keywords: string) => void - selectedFileList: string[] - setSelectedFileList: (selectedFileList: string[]) => void + selectedFileKeys: string[] + setSelectedFileKeys: (selectedFileKeys: string[]) => void fileList: OnlineDriveFile[] setFileList: (fileList: OnlineDriveFile[]) => void bucket: string setBucket: (bucket: string) => void startAfter: React.MutableRefObject isTruncated: React.MutableRefObject + previewOnlineDriveFileRef: React.MutableRefObject } -export const createOnlineDriveSlice: StateCreator = (set) => { +export const createOnlineDriveSlice: StateCreator = (set, get) => { return ({ prefix: [], setPrefix: (prefix: string[]) => set(() => ({ @@ -26,11 +27,15 @@ export const createOnlineDriveSlice: StateCreator = (set) setKeywords: (keywords: string) => set(() => ({ keywords, })), - startAfter: { current: '' }, - selectedFileList: [], - setSelectedFileList: (selectedFileList: string[]) => set(() => ({ - selectedFileList, - })), + selectedFileKeys: [], + setSelectedFileKeys: (selectedFileKeys: string[]) => { + set(() => ({ + selectedFileKeys, + })) + const key = selectedFileKeys[0] + const { fileList, previewOnlineDriveFileRef } = get() + previewOnlineDriveFileRef.current = fileList.find(file => file.key === key) + }, fileList: [], setFileList: (fileList: OnlineDriveFile[]) => set(() => ({ fileList, @@ -39,6 +44,8 @@ export const createOnlineDriveSlice: StateCreator = (set) setBucket: (bucket: string) => set(() => ({ bucket, })), + startAfter: { current: '' }, isTruncated: { current: false }, + previewOnlineDriveFileRef: { current: undefined }, }) } diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts index a40acd4f48..5286e5eef9 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts @@ -16,12 +16,16 @@ export type WebsiteCrawlSliceShape = { previewWebsitePageRef: React.MutableRefObject } -export const createWebsiteCrawlSlice: StateCreator = (set) => { +export const createWebsiteCrawlSlice: StateCreator = (set, get) => { return ({ websitePages: [], - setWebsitePages: (pages: CrawlResultItem[]) => set(() => ({ - websitePages: pages, - })), + setWebsitePages: (pages: CrawlResultItem[]) => { + set(() => ({ + websitePages: pages, + })) + const { previewWebsitePageRef } = get() + previewWebsitePageRef.current = pages[0] + }, currentWebsite: undefined, setCurrentWebsite: (website: CrawlResultItem | undefined) => set(() => ({ currentWebsite: website, diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx index d28e1cbf55..7b925c32a6 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx @@ -67,7 +67,6 @@ const WebsiteCrawl = ({ setPreviewIndex, setCurrentWebsite, currentNodeIdRef, - previewWebsitePageRef, } = dataSourceStore.getState() if (nodeId !== currentNodeIdRef.current) { setStep(CrawlStep.init) @@ -75,7 +74,6 @@ const WebsiteCrawl = ({ setCurrentWebsite(undefined) setWebsitePages([]) setPreviewIndex(0) - previewWebsitePageRef.current = undefined setCrawledNum(0) setTotalNum(0) setCrawlErrorMessage('') diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 7a77b7a08b..6007402cc8 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -63,7 +63,6 @@ export const useDatasourceOptions = (pipelineNodes: Node[]) export const useLocalFile = () => { const fileList = useDataSourceStoreWithSelector(state => state.localFileList) - const previewFileRef = useDataSourceStoreWithSelector(state => state.previewLocalFileRef) const currentLocalFile = useDataSourceStoreWithSelector(state => state.currentLocalFile) const dataSourceStore = useDataSourceStore() @@ -76,7 +75,6 @@ export const useLocalFile = () => { return { fileList, - previewFileRef, allFileLoaded, currentLocalFile, hidePreviewLocalFile, @@ -87,7 +85,6 @@ export const useOnlineDocuments = () => { const documentsData = useDataSourceStoreWithSelector(state => state.documentsData) const currentWorkspaceId = useDataSourceStoreWithSelector(state => state.currentWorkspaceId) const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments) - const previewOnlineDocumentRef = useDataSourceStoreWithSelector(state => state.previewOnlineDocumentRef) const currentDocument = useDataSourceStoreWithSelector(state => state.currentDocument) const dataSourceStore = useDataSourceStore() @@ -117,7 +114,6 @@ export const useOnlineDocuments = () => { onlineDocuments, currentDocument, PagesMapAndSelectedPagesId, - previewOnlineDocumentRef, hidePreviewOnlineDocument, } } @@ -125,7 +121,6 @@ export const useOnlineDocuments = () => { export const useWebsiteCrawl = () => { const websitePages = useDataSourceStoreWithSelector(state => state.websitePages) const currentWebsite = useDataSourceStoreWithSelector(state => state.currentWebsite) - const previewWebsitePageRef = useDataSourceStoreWithSelector(state => state.previewWebsitePageRef) const dataSourceStore = useDataSourceStore() const hideWebsitePreview = useCallback(() => { @@ -136,7 +131,6 @@ export const useWebsiteCrawl = () => { return { websitePages, - previewWebsitePageRef, currentWebsite, hideWebsitePreview, } @@ -144,10 +138,15 @@ export const useWebsiteCrawl = () => { export const useOnlineDrive = () => { const fileList = useDataSourceStoreWithSelector(state => state.fileList) - const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) + const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys) + + const selectedOnlineDriveFileList = useMemo(() => { + return selectedFileKeys.map(key => fileList.find(item => item.key === key)!) + }, [fileList, selectedFileKeys]) return { fileList, - selectedFileList, + selectedFileKeys, + selectedOnlineDriveFileList, } } diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index aef577c74b..c80c6a510d 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -24,7 +24,7 @@ import WebsitePreview from './preview/web-preview' import ProcessDocuments from './process-documents' import ChunkPreview from './preview/chunk-preview' import Processing from './processing' -import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline' +import type { InitialDocumentDetail, OnlineDriveFile, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks' @@ -57,7 +57,6 @@ const CreateFormPipeline = () => { } = useAddDocumentsSteps() const { fileList, - previewFileRef, allFileLoaded, currentLocalFile, hidePreviewLocalFile, @@ -67,18 +66,17 @@ const CreateFormPipeline = () => { onlineDocuments, currentDocument, PagesMapAndSelectedPagesId, - previewOnlineDocumentRef, hidePreviewOnlineDocument, } = useOnlineDocuments() const { websitePages, - previewWebsitePageRef, currentWebsite, hideWebsitePreview, } = useWebsiteCrawl() const { fileList: onlineDriveFileList, - selectedFileList, + selectedFileKeys, + selectedOnlineDriveFileList, } = useOnlineDrive() const datasourceType = datasource?.nodeData.provider_type @@ -107,9 +105,9 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length if (datasourceType === DatasourceType.onlineDrive) - return isShowVectorSpaceFull || !selectedFileList.length + return isShowVectorSpaceFull || !selectedFileKeys.length return false - }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileList.length]) + }, [datasource, datasourceType, isShowVectorSpaceFull, fileList.length, allFileLoaded, onlineDocuments.length, websitePages.length, selectedFileKeys.length]) const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { file_size_limit: 15, @@ -143,8 +141,8 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.onlineDocument) return onlineDocuments.length if (datasourceType === DatasourceType.onlineDrive) - return selectedFileList.length - }, [datasourceType, onlineDocuments.length, selectedFileList.length]) + return selectedFileKeys.length + }, [datasourceType, onlineDocuments.length, selectedFileKeys.length]) const tip = useMemo(() => { if (datasourceType === DatasourceType.onlineDocument) @@ -163,9 +161,15 @@ const CreateFormPipeline = () => { const handlePreviewChunks = useCallback(async (data: Record) => { if (!datasource) return + const { + previewLocalFileRef, + previewOnlineDocumentRef, + previewWebsitePageRef, + previewOnlineDriveFileRef, + } = dataSourceStore.getState() const datasourceInfoList: Record[] = [] if (datasourceType === DatasourceType.localFile) { - const { id, name, type, size, extension, mime_type } = previewFileRef.current as File + const { id, name, type, size, extension, mime_type } = previewLocalFileRef.current as File const documentInfo = { related_id: id, name, @@ -188,6 +192,14 @@ const CreateFormPipeline = () => { } if (datasourceType === DatasourceType.websiteCrawl) datasourceInfoList.push(previewWebsitePageRef.current!) + if (datasourceType === DatasourceType.onlineDrive) { + const { bucket } = dataSourceStore.getState() + const { key } = previewOnlineDriveFileRef.current! + datasourceInfoList.push({ + bucket, + key, + }) + } await runPublishedPipeline({ pipeline_id: pipelineId!, inputs: data, @@ -200,7 +212,7 @@ const CreateFormPipeline = () => { setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) }, }) - }, [datasource, datasourceType, previewWebsitePageRef, runPublishedPipeline, pipelineId, previewFileRef, previewOnlineDocumentRef]) + }, [datasource, datasourceType, runPublishedPipeline, pipelineId, dataSourceStore]) const handleProcess = useCallback(async (data: Record) => { if (!datasource) @@ -240,7 +252,7 @@ const CreateFormPipeline = () => { if (datasourceType === DatasourceType.onlineDrive) { if (datasourceType === DatasourceType.onlineDrive) { const { bucket } = dataSourceStore.getState() - selectedFileList.forEach((key) => { + selectedFileKeys.forEach((key) => { datasourceInfoList.push({ bucket, key, @@ -262,7 +274,7 @@ const CreateFormPipeline = () => { handleNextStep() }, }) - }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileList, websitePages]) + }, [dataSourceStore, datasource, datasourceType, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, selectedFileKeys, websitePages]) const onClickProcess = useCallback(() => { isPreview.current = false @@ -279,22 +291,31 @@ const CreateFormPipeline = () => { }, [handlePreviewChunks, handleProcess]) const handlePreviewFileChange = useCallback((file: DocumentItem) => { - previewFileRef.current = file + const { previewLocalFileRef } = dataSourceStore.getState() + previewLocalFileRef.current = file onClickPreview() - }, [onClickPreview, previewFileRef]) + }, [dataSourceStore, onClickPreview]) const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { + const { previewOnlineDocumentRef } = dataSourceStore.getState() previewOnlineDocumentRef.current = page onClickPreview() - }, [onClickPreview, previewOnlineDocumentRef]) + }, [dataSourceStore, onClickPreview]) const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { + const { previewWebsitePageRef } = dataSourceStore.getState() previewWebsitePageRef.current = website onClickPreview() - }, [onClickPreview, previewWebsitePageRef]) + }, [dataSourceStore, onClickPreview]) + + const handlePreviewOnlineDriveFileChange = useCallback((file: OnlineDriveFile) => { + const { previewOnlineDriveFileRef } = dataSourceStore.getState() + previewOnlineDriveFileRef.current = file + onClickPreview() + }, [dataSourceStore, onClickPreview]) const handleSelectAll = useCallback(() => { - const { setOnlineDocuments, setSelectedFileList, setSelectedPagesId } = dataSourceStore.getState() + const { setOnlineDocuments, setSelectedFileKeys, setSelectedPagesId } = dataSourceStore.getState() if (datasourceType === DatasourceType.onlineDocument) { const allIds = currentWorkspace?.pages.map(page => page.page_id) || [] if (onlineDocuments.length < allIds.length) { @@ -311,12 +332,12 @@ const CreateFormPipeline = () => { const allKeys = onlineDriveFileList.filter((item) => { return item.type !== 'bucket' }).map(file => file.key) - if (selectedFileList.length < allKeys.length) - setSelectedFileList(allKeys) + if (selectedFileKeys.length < allKeys.length) + setSelectedFileKeys(allKeys) else - setSelectedFileList([]) + setSelectedFileKeys([]) } - }, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType, onlineDocuments.length, onlineDriveFileList, selectedFileList.length]) + }, [PagesMapAndSelectedPagesId, currentWorkspace?.pages, dataSourceStore, datasourceType, onlineDocuments.length, onlineDriveFileList, selectedFileKeys.length]) if (isFetchingPipelineInfo) { return ( @@ -440,9 +461,10 @@ const CreateFormPipeline = () => {
file.file)} + localFiles={fileList.map(file => file.file)} onlineDocuments={onlineDocuments} websitePages={websitePages} + onlineDriveFiles={selectedOnlineDriveFileList} isIdle={isIdle} isPending={isPending && isPreview.current} estimateData={estimateData} @@ -450,6 +472,7 @@ const CreateFormPipeline = () => { handlePreviewFileChange={handlePreviewFileChange} handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange} handlePreviewWebsitePageChange={handlePreviewWebsiteChange} + handlePreviewOnlineDriveFileChange={handlePreviewOnlineDriveFileChange} />
diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx index e926422b1d..763b5f38af 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx @@ -14,13 +14,15 @@ import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from import { RiSearchEyeLine } from '@remixicon/react' import Badge from '@/app/components/base/badge' import Button from '@/app/components/base/button' +import type { OnlineDriveFile } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline' type ChunkPreviewProps = { dataSourceType: DatasourceType - files: CustomFile[] + localFiles: CustomFile[] onlineDocuments: NotionPage[] websitePages: CrawlResultItem[] + onlineDriveFiles: OnlineDriveFile[] isIdle: boolean isPending: boolean estimateData: FileIndexingEstimateResponse | undefined @@ -28,13 +30,15 @@ type ChunkPreviewProps = { handlePreviewFileChange: (file: DocumentItem) => void handlePreviewOnlineDocumentChange: (page: NotionPage) => void handlePreviewWebsitePageChange: (page: CrawlResultItem) => void + handlePreviewOnlineDriveFileChange: (file: OnlineDriveFile) => void } const ChunkPreview = ({ dataSourceType, - files, + localFiles, onlineDocuments, websitePages, + onlineDriveFiles, isIdle, isPending, estimateData, @@ -42,13 +46,15 @@ const ChunkPreview = ({ handlePreviewFileChange, handlePreviewOnlineDocumentChange, handlePreviewWebsitePageChange, + handlePreviewOnlineDriveFileChange, }: ChunkPreviewProps) => { const { t } = useTranslation() const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form) - const [previewFile, setPreviewFile] = useState(files[0] as DocumentItem) + const [previewFile, setPreviewFile] = useState(localFiles[0] as DocumentItem) const [previewOnlineDocument, setPreviewOnlineDocument] = useState(onlineDocuments[0]) const [previewWebsitePage, setPreviewWebsitePage] = useState(websitePages[0]) + const [previewOnlineDriveFile, setPreviewOnlineDriveFile] = useState(onlineDriveFiles[0]) return ( {dataSourceType === DatasourceType.localFile && >} + files={localFiles as Array>} onChange={(selected) => { setPreviewFile(selected) handlePreviewFileChange(selected) @@ -110,6 +116,29 @@ const ChunkPreview = ({ } /> } + {dataSourceType === DatasourceType.onlineDrive + && ({ + id: file.key, + name: file.displayName, + extension: 'md', + })) + } + onChange={(selected) => { + const selectedFile = onlineDriveFiles.find(file => file.key === selected.id) + setPreviewOnlineDriveFile(selectedFile!) + handlePreviewOnlineDriveFileChange(selectedFile!) + }} + value={ + { + id: previewOnlineDriveFile?.key || '', + name: previewOnlineDriveFile?.displayName || '', + extension: 'md', + } + } + /> + } { currentDocForm !== ChunkingMode.qa && { const fileList = useDataSourceStoreWithSelector(state => state.localFileList) const onlineDocuments = useDataSourceStoreWithSelector(state => state.onlineDocuments) const websitePages = useDataSourceStoreWithSelector(state => state.websitePages) - const selectedFileList = useDataSourceStoreWithSelector(state => state.selectedFileList) + const selectedFileKeys = useDataSourceStoreWithSelector(state => state.selectedFileKeys) const dataSourceStore = useDataSourceStore() const [datasource, setDatasource] = useState() @@ -45,9 +45,9 @@ const TestRunPanel = () => { if (datasourceType === DatasourceType.websiteCrawl) return !websitePages.length if (datasourceType === DatasourceType.onlineDrive) - return !selectedFileList.length + return !selectedFileKeys.length return false - }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileList.length, websitePages.length]) + }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileKeys.length, websitePages.length]) const handleClose = useCallback(() => { setShowDebugAndPreviewPanel(false) @@ -87,7 +87,7 @@ const TestRunPanel = () => { const { bucket } = dataSourceStore.getState() datasourceInfoList.push({ bucket, - key: selectedFileList[0], + key: selectedFileKeys[0], }) } handleRun({ @@ -96,7 +96,7 @@ const TestRunPanel = () => { datasource_type: datasourceType, datasource_info_list: datasourceInfoList, }) - }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileList, websitePages]) + }, [dataSourceStore, datasource, datasourceType, fileList, handleRun, onlineDocuments, selectedFileKeys, websitePages]) return (
{ const { mutateAsync: checkPipelineDependencies } = useCheckPipelineDependencies() const handleCheckPluginDependencies = useCallback(async (id: string, isPipeline = false) => { - const mutateAsync = isPipeline ? checkPipelineDependencies : checkWorkflowDependencies - const { leaked_dependencies } = await mutateAsync(id) + const checkDependencies = isPipeline ? checkPipelineDependencies : checkWorkflowDependencies + const { leaked_dependencies } = await checkDependencies(id) const { setDependencies } = usePluginDependenciesStore.getState() setDependencies(leaked_dependencies) }, [checkWorkflowDependencies, checkPipelineDependencies])