From f2960989c1a7e91d3bf7131c7820559443273f48 Mon Sep 17 00:00:00 2001 From: twwu Date: Thu, 3 Jul 2025 18:34:54 +0800 Subject: [PATCH] refactor: Refactor data source components --- .../data-source/local-file/index.tsx | 62 +++++--- .../data-source/online-documents/index.tsx | 61 ++++---- .../online-documents/page-selector/index.tsx | 7 +- .../data-source/online-drive/index.tsx | 85 +++++++--- .../data-source/store/slices/local-file.ts | 2 +- .../store/slices/online-document.ts | 12 +- .../data-source/store/slices/online-drive.ts | 6 + .../data-source/store/slices/website-crawl.ts | 2 + .../data-source/website-crawl/index.tsx | 50 +++--- .../documents/create-from-pipeline/hooks.ts | 146 +++--------------- .../documents/create-from-pipeline/index.tsx | 111 ++++--------- .../preview/file-preview.tsx | 28 +--- .../preview/web-preview.tsx | 12 +- .../components/panel/test-run/hooks.ts | 117 +------------- .../components/panel/test-run/index.tsx | 88 ++--------- 15 files changed, 259 insertions(+), 530 deletions(-) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx index 794913eb14..2f97cebafd 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/local-file/index.tsx @@ -5,7 +5,7 @@ import { useContext } from 'use-context-selector' import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react' import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon' import cn from '@/utils/classnames' -import type { CustomFile as File, FileItem } from '@/models/datasets' +import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets' import { ToastContext } from '@/app/components/base/toast' import SimplePieChart from '@/app/components/base/simple-pie-chart' import { upload } from '@/service/base' @@ -15,35 +15,34 @@ import { IS_CE_EDITION } from '@/config' import { Theme } from '@/types/app' import useTheme from '@/hooks/use-theme' import { useFileUploadConfig } from '@/service/use-common' +import { useDataSourceStore } from '../store' +import produce from 'immer' const FILES_NUMBER_LIMIT = 20 export type LocalFileProps = { - fileList: FileItem[] allowedExtensions: string[] - prepareFileList: (files: FileItem[]) => void - onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void - onFileListUpdate?: (files: FileItem[]) => void - onPreview?: (file: File) => void notSupportBatchUpload?: boolean } const LocalFile = ({ - fileList, allowedExtensions, - prepareFileList, - onFileUpdate, - onFileListUpdate, - onPreview, notSupportBatchUpload, }: LocalFileProps) => { const { t } = useTranslation() const { notify } = useContext(ToastContext) const { locale } = useContext(I18n) + const fileList = useDataSourceStore(state => state.localFileList) + const setFileList = useDataSourceStore(state => state.setLocalFileList) + const setCurrentFile = useDataSourceStore(state => state.setCurrentLocalFile) + const previewFileRef = useDataSourceStore(state => state.previewLocalFileRef) const [dragging, setDragging] = useState(false) + const dropRef = useRef(null) const dragRef = useRef(null) const fileUploader = useRef(null) + const fileListRef = useRef([]) + const hideUpload = notSupportBatchUpload && fileList.length > 0 const { data: fileUploadConfigResponse } = useFileUploadConfig() @@ -69,7 +68,26 @@ const LocalFile = ({ batch_count_limit: 5, }, [fileUploadConfigResponse]) - const fileListRef = useRef([]) + const updateFile = useCallback((fileItem: FileItem, progress: number, list: FileItem[]) => { + const newList = produce(list, (draft) => { + const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) + draft[targetIndex] = { + ...draft[targetIndex], + progress, + } + }) + setFileList(newList) + previewFileRef.current = newList[0].file as DocumentItem + }, [previewFileRef, setFileList]) + + const updateFileList = useCallback((preparedFiles: FileItem[]) => { + setFileList(preparedFiles) + }, [setFileList]) + + const handlePreview = useCallback((file: File) => { + if (file.id) + setCurrentFile(file) + }, [setCurrentFile]) // utils const getFileType = (currentFile: File) => { @@ -107,7 +125,7 @@ const LocalFile = ({ const onProgress = (e: ProgressEvent) => { if (e.lengthComputable) { const percent = Math.floor(e.loaded / e.total * 100) - onFileUpdate(fileItem, percent, fileListRef.current) + updateFile(fileItem, percent, fileListRef.current) } } @@ -124,16 +142,16 @@ const LocalFile = ({ } const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID) fileListRef.current[index] = completeFile - onFileUpdate(completeFile, 100, fileListRef.current) + updateFile(completeFile, 100, fileListRef.current) return Promise.resolve({ ...completeFile }) }) .catch((e) => { notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') }) - onFileUpdate(fileItem, -2, fileListRef.current) + updateFile(fileItem, -2, fileListRef.current) return Promise.resolve({ ...fileItem }) }) .finally() - }, [fileListRef, notify, onFileUpdate, t]) + }, [fileListRef, notify, updateFile, t]) const uploadBatchFiles = useCallback((bFiles: FileItem[]) => { bFiles.forEach(bf => (bf.progress = 0)) @@ -172,10 +190,10 @@ const LocalFile = ({ progress: -1, })) const newFiles = [...fileListRef.current, ...preparedFiles] - prepareFileList(newFiles) + updateFileList(newFiles) fileListRef.current = newFiles uploadMultipleFiles(preparedFiles) - }, [prepareFileList, uploadMultipleFiles, notify, t, fileList]) + }, [updateFileList, uploadMultipleFiles, notify, t, fileList]) const handleDragEnter = (e: DragEvent) => { e.preventDefault() @@ -207,17 +225,17 @@ const LocalFile = ({ initialUpload(validFiles) }, [initialUpload, isValid, notSupportBatchUpload]) - const selectHandle = () => { + const selectHandle = useCallback(() => { if (fileUploader.current) fileUploader.current.click() - } + }, []) const removeFile = (fileID: string) => { if (fileUploader.current) fileUploader.current.value = '' fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID) - onFileListUpdate?.([...fileListRef.current]) + updateFileList([...fileListRef.current]) } const fileChangeHandle = useCallback((e: React.ChangeEvent) => { const files = [...(e.target.files ?? [])] as File[] @@ -287,7 +305,7 @@ const LocalFile = ({ return (
fileItem.file?.id && onPreview?.(fileItem.file)} + onClick={handlePreview.bind(null, fileItem.file)} className={cn( 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', isError && 'border-state-destructive-border bg-state-destructive-hover', diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx index d67e0647ce..c88f2095c6 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo } from 'react' import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector' import SearchInput from '@/app/components/base/notion-page-selector/search-input' import PageSelector from './page-selector' -import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' +import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common' import Header from '@/app/components/datasets/create/website/base/header' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { DatasourceType } from '@/models/pipeline' @@ -10,43 +10,44 @@ import { ssePost } from '@/service/base' import Toast from '@/app/components/base/toast' import type { DataSourceNodeCompletedResponse } from '@/types/pipeline' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import { useDataSourceStore } from '../store' type OnlineDocumentsProps = { - onSelect: (selectedPages: NotionPage[]) => void - previewPageId?: string - onPreview?: (selectedPage: NotionPage) => void isInPipeline?: boolean nodeId: string nodeData: DataSourceNodeType - documentsData: DataSourceNotionWorkspace[] - setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => void - searchValue: string - setSearchValue: (value: string) => void - currentWorkspaceId: string - setCurrentWorkspaceId: (workspaceId: string) => void - PagesMapAndSelectedPagesId: DataSourceNotionPageMap - selectedPagesId: Set - setSelectedPagesId: (selectedPagesId: Set) => void } const OnlineDocuments = ({ - onSelect, - previewPageId, - onPreview, isInPipeline = false, nodeId, nodeData, - documentsData, - setDocumentsData, - searchValue, - setSearchValue, - currentWorkspaceId, - setCurrentWorkspaceId, - PagesMapAndSelectedPagesId, - selectedPagesId, - setSelectedPagesId, }: OnlineDocumentsProps) => { const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const documentsData = useDataSourceStore(state => state.documentsData) + const setDocumentsData = useDataSourceStore(state => state.setDocumentsData) + const searchValue = useDataSourceStore(state => state.searchValue) + const setSearchValue = useDataSourceStore(state => state.setSearchValue) + const currentWorkspaceId = useDataSourceStore(state => state.currentWorkspaceId) + const setCurrentWorkspaceId = useDataSourceStore(state => state.setCurrentWorkspaceId) + const setOnlineDocuments = useDataSourceStore(state => state.setOnlineDocuments) + const setCurrentDocument = useDataSourceStore(state => state.setCurrentDocument) + const selectedPagesId = useDataSourceStore(state => state.selectedPagesId) + const setSelectedPagesId = useDataSourceStore(state => state.setSelectedPagesId) + + const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => { + const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { + next.pages.forEach((page) => { + prev[page.page_id] = { + ...page, + workspace_id: next.workspace_id, + } + }) + + return prev + }, {}) + return pagesMap + }, [documentsData]) const datasourceNodeRunURL = !isInPipeline ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` @@ -96,13 +97,12 @@ const OnlineDocuments = ({ const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[pageId]) setSelectedPagesId(new Set(Array.from(newSelectedPagesId))) - onSelect(selectedPages) - }, [setSelectedPagesId, onSelect, PagesMapAndSelectedPagesId]) + setOnlineDocuments(selectedPages) + }, [setSelectedPagesId, setOnlineDocuments, PagesMapAndSelectedPagesId]) const handlePreviewPage = useCallback((previewPageId: string) => { - if (onPreview) - onPreview(PagesMapAndSelectedPagesId[previewPageId]) - }, [PagesMapAndSelectedPagesId, onPreview]) + setCurrentDocument(PagesMapAndSelectedPagesId[previewPageId]) + }, [PagesMapAndSelectedPagesId, setCurrentDocument]) const headerInfo = useMemo(() => { return { @@ -144,7 +144,6 @@ const OnlineDocuments = ({ pagesMap={PagesMapAndSelectedPagesId} onSelect={handleSelectPages} canPreview={!isInPipeline} - previewPageId={previewPageId} onPreview={handlePreviewPage} isMultipleChoice={!isInPipeline} currentWorkspaceId={currentWorkspaceId} diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx index 8c12c15c1d..a096d13329 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/page-selector/index.tsx @@ -13,7 +13,6 @@ type PageSelectorProps = { list: DataSourceNotionPage[] onSelect: (selectedPagesId: Set) => void canPreview?: boolean - previewPageId?: string onPreview?: (selectedPageId: string) => void isMultipleChoice?: boolean currentWorkspaceId: string @@ -41,14 +40,13 @@ const PageSelector = ({ list, onSelect, canPreview = true, - previewPageId, onPreview, isMultipleChoice = true, currentWorkspaceId, }: PageSelectorProps) => { const { t } = useTranslation() const [dataList, setDataList] = useState([]) - const [localPreviewPageId, setLocalPreviewPageId] = useState('') + const [currentPreviewPageId, setCurrentPreviewPageId] = useState('') useEffect(() => { setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => { @@ -71,7 +69,6 @@ const PageSelector = ({ } }) const currentDataList = searchValue ? searchDataList : dataList - const currentPreviewPageId = previewPageId === undefined ? localPreviewPageId : previewPageId const listMapWithChildrenAndDescendants = useMemo(() => { return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => { @@ -149,7 +146,7 @@ const PageSelector = ({ const current = currentDataList[index] const pageId = current.page_id - setLocalPreviewPageId(pageId) + setCurrentPreviewPageId(pageId) if (onPreview) onPreview(pageId) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx index 59dafeba47..c453498529 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/online-drive/index.tsx @@ -1,36 +1,77 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' import Header from './header' -import { useCallback } from 'react' +import { useCallback, useEffect } from 'react' import FileList from './file-list' -import type { OnlineDriveFile } from '@/models/pipeline' +import { DatasourceType } from '@/models/pipeline' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { ssePost } from '@/service/base' +import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline' +import Toast from '@/app/components/base/toast' +import { useDataSourceStore } from '../store' type OnlineDriveProps = { + nodeId: string nodeData: DataSourceNodeType - prefix: string[] - setPrefix: (prefix: string[]) => void - keywords: string - setKeywords: (keywords: string) => void - startAfter: string - setStartAfter: (startAfter: string) => void - selectedFileList: string[] - setSelectedFileList: (selectedFileList: string[]) => void - fileList: OnlineDriveFile[] - setFileList: (fileList: OnlineDriveFile[]) => void + isInPipeline?: boolean } const OnlineDrive = ({ + nodeId, nodeData, - prefix, - setPrefix, - keywords, - setKeywords, - startAfter, - setStartAfter, - selectedFileList, - setSelectedFileList, - fileList, - setFileList, + isInPipeline = false, }: OnlineDriveProps) => { + const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const prefix = useDataSourceStore(state => state.prefix) + const setPrefix = useDataSourceStore(state => state.setPrefix) + const keywords = useDataSourceStore(state => state.keywords) + const setKeywords = useDataSourceStore(state => state.setKeywords) + const bucket = useDataSourceStore(state => state.bucket) + const setBucket = useDataSourceStore(state => state.setBucket) + const startAfter = useDataSourceStore(state => state.startAfter) + const setStartAfter = useDataSourceStore(state => state.setStartAfter) + const selectedFileList = useDataSourceStore(state => state.selectedFileList) + const setSelectedFileList = useDataSourceStore(state => state.setSelectedFileList) + const fileList = useDataSourceStore(state => state.fileList) + const setFileList = useDataSourceStore(state => state.setFileList) + + const datasourceNodeRunURL = !isInPipeline + ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` + : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` + + const getOnlineDrive = useCallback(async () => { + ssePost( + datasourceNodeRunURL, + { + body: { + inputs: { + prefix: prefix.join('/'), + bucket, + start_after: startAfter, + max_keys: 30, // Adjust as needed + }, + datasource_type: DatasourceType.onlineDrive, + }, + }, + { + onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { + console.log('Online Drive documents data:', documentsData) + }, + onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => { + Toast.notify({ + type: 'error', + message: error.error, + }) + }, + }, + ) + }, [bucket, datasourceNodeRunURL, prefix, startAfter]) + + useEffect(() => { + if (!fileList.length) + getOnlineDrive() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + const updateKeywords = useCallback((keywords: string) => { setKeywords(keywords) }, [setKeywords]) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts index 57fcba2351..ff13b0cf28 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts @@ -1,5 +1,5 @@ import type { StateCreator } from 'zustand' -import type { DocumentItem, FileItem } from '@/models/datasets' +import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets' export type LocalFileSliceShape = { localFileList: FileItem[] diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts index fead7578d1..eef81764e6 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts @@ -2,8 +2,8 @@ import type { StateCreator } from 'zustand' import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common' export type OnlineDocumentSliceShape = { - documentData: DataSourceNotionWorkspace[] - setDocumentData: (documentData: DataSourceNotionWorkspace[]) => void + documentsData: DataSourceNotionWorkspace[] + setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void searchValue: string setSearchValue: (searchValue: string) => void currentWorkspaceId: string @@ -14,13 +14,14 @@ export type OnlineDocumentSliceShape = { setCurrentDocument: (document: NotionPage | undefined) => void selectedPagesId: Set setSelectedPagesId: (selectedPagesId: Set) => void + previewOnlineDocumentRef: React.MutableRefObject } export const createOnlineDocumentSlice: StateCreator = (set) => { return ({ - documentData: [], - setDocumentData: (documentData: DataSourceNotionWorkspace[]) => set(() => ({ - documentData, + documentsData: [], + setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({ + documentsData, })), searchValue: '', setSearchValue: (searchValue: string) => set(() => ({ @@ -42,5 +43,6 @@ export const createOnlineDocumentSlice: StateCreator = setSelectedPagesId: (selectedPagesId: Set) => set(() => ({ selectedPagesId, })), + previewOnlineDocumentRef: { current: undefined }, }) } diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts index ee3531cf5a..29f794aa13 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts @@ -12,6 +12,8 @@ export type OnlineDriveSliceShape = { setSelectedFileList: (selectedFileList: string[]) => void fileList: OnlineDriveFile[] setFileList: (fileList: OnlineDriveFile[]) => void + bucket: string + setBucket: (bucket: string) => void } export const createOnlineDriveSlice: StateCreator = (set) => { @@ -36,5 +38,9 @@ export const createOnlineDriveSlice: StateCreator = (set) setFileList: (fileList: OnlineDriveFile[]) => set(() => ({ fileList, })), + bucket: '', + setBucket: (bucket: string) => set(() => ({ + bucket, + })), }) } diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts index 715f85d461..a40acd4f48 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/website-crawl.ts @@ -13,6 +13,7 @@ export type WebsiteCrawlSliceShape = { setStep: (step: CrawlStep) => void previewIndex: number setPreviewIndex: (index: number) => void + previewWebsitePageRef: React.MutableRefObject } export const createWebsiteCrawlSlice: StateCreator = (set) => { @@ -37,5 +38,6 @@ export const createWebsiteCrawlSlice: StateCreator = (se setPreviewIndex: (index: number) => set(() => ({ previewIndex: index, })), + previewWebsitePageRef: { current: undefined }, }) } diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx index a24a737aee..a18c011b8f 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl/index.tsx @@ -1,7 +1,7 @@ 'use client' import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' -import type { CrawlResult, CrawlResultItem } from '@/models/datasets' +import type { CrawlResultItem } from '@/models/datasets' import { CrawlStep } from '@/models/datasets' import Header from '@/app/components/datasets/create/website/base/header' import Options from './base/options' @@ -20,41 +20,37 @@ import type { DataSourceNodeProcessingResponse, } from '@/types/pipeline' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import { useDataSourceStore } from '../store' const I18N_PREFIX = 'datasetCreation.stepOne.website' export type WebsiteCrawlProps = { nodeId: string nodeData: DataSourceNodeType - crawlResult: CrawlResult | undefined - setCrawlResult: (payload: CrawlResult) => void - step: CrawlStep - setStep: (step: CrawlStep) => void - checkedCrawlResult: CrawlResultItem[] - onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void - previewIndex?: number - onPreview?: (payload: CrawlResultItem, index: number) => void isInPipeline?: boolean } const WebsiteCrawl = ({ nodeId, nodeData, - crawlResult, - setCrawlResult, - step, - setStep, - checkedCrawlResult, - onCheckedCrawlResultChange, - previewIndex, - onPreview, isInPipeline = false, }: WebsiteCrawlProps) => { const { t } = useTranslation() const [controlFoldOptions, setControlFoldOptions] = useState(0) const [totalNum, setTotalNum] = useState(0) const [crawledNum, setCrawledNum] = useState(0) + const [crawlErrorMessage, setCrawlErrorMessage] = useState('') const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const crawlResult = useDataSourceStore(state => state.crawlResult) + const setCrawlResult = useDataSourceStore(state => state.setCrawlResult) + const step = useDataSourceStore(state => state.step) + const setStep = useDataSourceStore(state => state.setStep) + const checkedCrawlResult = useDataSourceStore(state => state.websitePages) + const setWebsitePages = useDataSourceStore(state => state.setWebsitePages) + const previewWebsitePageRef = useDataSourceStore(state => state.previewWebsitePageRef) + const previewIndex = useDataSourceStore(state => state.previewIndex) + const setCurrentWebsite = useDataSourceStore(state => state.setCurrentWebsite) + const setPreviewIndex = useDataSourceStore(state => state.setPreviewIndex) const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams) const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({ @@ -70,13 +66,21 @@ const WebsiteCrawl = ({ const isInit = step === CrawlStep.init const isCrawlFinished = step === CrawlStep.finished const isRunning = step === CrawlStep.running - const [crawlErrorMessage, setCrawlErrorMessage] = useState('') const showError = isCrawlFinished && crawlErrorMessage - const datasourceNodeRunURL = !isInPipeline ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` + const handleCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => { + setWebsitePages(checkedCrawlResult) + previewWebsitePageRef.current = checkedCrawlResult[0] + }, [previewWebsitePageRef, setWebsitePages]) + + const handlePreview = useCallback((website: CrawlResultItem, index: number) => { + setCurrentWebsite(website) + setPreviewIndex(index) + }, [setCurrentWebsite, setPreviewIndex]) + const handleRun = useCallback(async (value: Record) => { setStep(CrawlStep.running) ssePost( @@ -106,7 +110,7 @@ const WebsiteCrawl = ({ time_consuming: time_consuming ?? 0, } setCrawlResult(crawlResultData) - onCheckedCrawlResultChange(crawlData || []) // default select the crawl result + handleCheckedCrawlResultChange(crawlData || []) // default select the crawl result setCrawlErrorMessage('') setStep(CrawlStep.finished) }, @@ -116,7 +120,7 @@ const WebsiteCrawl = ({ }, }, ) - }, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t]) + }, [datasourceNodeRunURL, handleCheckedCrawlResultChange, setCrawlResult, setStep, t]) const handleSubmit = useCallback((value: Record) => { handleRun(value) @@ -165,10 +169,10 @@ const WebsiteCrawl = ({ className='mt-2' list={crawlResult?.data || []} checkedList={checkedCrawlResult} - onSelectedChange={onCheckedCrawlResultChange} + onSelectedChange={handleCheckedCrawlResultChange} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} previewIndex={previewIndex} - onPreview={onPreview} + onPreview={handlePreview} isMultipleChoice={!isInPipeline} // only support single choice in test run /> )} diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 13a4fbd7c6..b99fd51777 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -4,11 +4,9 @@ import type { DataSourceOption } from '@/app/components/rag-pipeline/components/ import { useCallback, useMemo, useRef, useState } from 'react' import { BlockEnum, type Node } from '@/app/components/workflow/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets' +import type { CrawlResult, CrawlResultItem } from '@/models/datasets' import { CrawlStep } from '@/models/datasets' -import produce from 'immer' -import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' -import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline' +import { useDataSourceStore } from './data-source/store' export const useAddDocumentsSteps = () => { const { t } = useTranslation() @@ -58,26 +56,6 @@ export const useDatasourceOptions = (pipelineNodes: Node[]) data: node.data, }) }) - if (process.env.NODE_ENV === 'development') { - // todo: delete mock data - options.push({ - label: 'Google Drive', - value: '123456', - // @ts-expect-error mock data - data: { - datasource_parameters: {}, - datasource_configurations: {}, - type: BlockEnum.DataSource, - title: 'Google Drive', - plugin_id: 'langgenius/google-drive', - provider_type: 'online_drive', - provider_name: 'google_drive', - datasource_name: 'google-drive', - datasource_label: 'Google Drive', - selected: false, - }, - }) - } return options }, [datasourceNodes]) @@ -85,102 +63,41 @@ export const useDatasourceOptions = (pipelineNodes: Node[]) } export const useLocalFile = () => { - const [fileList, setFileList] = useState([]) - const [currentFile, setCurrentFile] = useState() - - const previewFile = useRef() + const fileList = useDataSourceStore(state => state.localFileList) + const previewFileRef = useDataSourceStore(state => state.previewLocalFileRef) + const currentLocalFile = useDataSourceStore(state => state.currentLocalFile) + const setCurrentFile = useDataSourceStore(state => state.setCurrentLocalFile) const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList]) - const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { - const newList = produce(list, (draft) => { - const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) - draft[targetIndex] = { - ...draft[targetIndex], - progress, - } - }) - setFileList(newList) - previewFile.current = newList[0].file as DocumentItem - } - - const updateFileList = useCallback((preparedFiles: FileItem[]) => { - setFileList(preparedFiles) - }, []) - - const updateCurrentFile = useCallback((file: File) => { - setCurrentFile(file) - }, []) - - const hideFilePreview = useCallback(() => { + const hidePreviewLocalFile = useCallback(() => { setCurrentFile(undefined) - }, []) + }, [setCurrentFile]) return { fileList, - previewFile, + previewFileRef, allFileLoaded, - updateFile, - updateFileList, - currentFile, - updateCurrentFile, - hideFilePreview, + currentLocalFile, + hidePreviewLocalFile, } } export const useOnlineDocuments = () => { - const [documentsData, setDocumentsData] = useState([]) - const [searchValue, setSearchValue] = useState('') - const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') - const [onlineDocuments, setOnlineDocuments] = useState([]) - const [currentDocument, setCurrentDocument] = useState() + const onlineDocuments = useDataSourceStore(state => state.onlineDocuments) + const previewOnlineDocumentRef = useDataSourceStore(state => state.previewOnlineDocumentRef) + const setCurrentDocument = useDataSourceStore(state => state.setCurrentDocument) + const currentDocument = useDataSourceStore(state => state.currentDocument) - const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => { - const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { - next.pages.forEach((page) => { - prev[page.page_id] = { - ...page, - workspace_id: next.workspace_id, - } - }) - - return prev - }, {}) - return pagesMap - }, [documentsData]) - const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])] - const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) - - const previewOnlineDocument = useRef(onlineDocuments[0]) - - const updateOnlineDocuments = (value: NotionPage[]) => { - setOnlineDocuments(value) - } - - const updateCurrentPage = useCallback((page: NotionPage) => { - setCurrentDocument(page) - }, []) - - const hideOnlineDocumentPreview = useCallback(() => { + const hidePreviewOnlineDocument = useCallback(() => { setCurrentDocument(undefined) - }, []) + }, [setCurrentDocument]) return { - documentsData, - setDocumentsData, - searchValue, - setSearchValue, - currentWorkspaceId, - setCurrentWorkspaceId, - PagesMapAndSelectedPagesId, - selectedPagesId, - setSelectedPagesId, onlineDocuments, - previewOnlineDocument, - updateOnlineDocuments, currentDocument, - updateCurrentPage, - hideOnlineDocumentPreview, + previewOnlineDocumentRef, + hidePreviewOnlineDocument, } } @@ -224,28 +141,5 @@ export const useWebsiteCrawl = () => { } export const useOnlineDrive = () => { - const [prefix, setPrefix] = useState([]) - const [keywords, setKeywords] = useState('') - const [startAfter, setStartAfter] = useState('') - const [selectedFileList, setSelectedFileList] = useState([]) - const [fileList, setFileList] = useState([ - { - key: 'Bucket_1', - size: 1024, // unit bytes - type: OnlineDriveFileType.bucket, - }, - ]) - - return { - prefix, - setPrefix, - keywords, - setKeywords, - startAfter, - setStartAfter, - selectedFileList, - setSelectedFileList, - fileList, - setFileList, - } + return {} } diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 3a7f237a1a..69d36b5365 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -27,7 +27,7 @@ import Processing from './processing' import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' -import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks' +import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks' import DataSourceProvider from './data-source/store/provider' const CreateFormPipeline = () => { @@ -53,56 +53,24 @@ const CreateFormPipeline = () => { } = useAddDocumentsSteps() const { fileList, - previewFile, + previewFileRef, allFileLoaded, - updateFile, - updateFileList, - currentFile, - updateCurrentFile, - hideFilePreview, + currentLocalFile, + hidePreviewLocalFile, } = useLocalFile() const { - documentsData, - setDocumentsData, - searchValue, - setSearchValue, - currentWorkspaceId, - setCurrentWorkspaceId, - PagesMapAndSelectedPagesId, - selectedPagesId, - setSelectedPagesId, onlineDocuments, - previewOnlineDocument, - updateOnlineDocuments, currentDocument, - updateCurrentPage, - hideOnlineDocumentPreview, + previewOnlineDocumentRef, + hidePreviewOnlineDocument, } = useOnlineDocuments() const { websitePages, - crawlResult, - setCrawlResult, - step, - setStep, previewWebsitePage, - updataCheckedCrawlResultChange, currentWebsite, - updateCurrentWebsite, - previewIndex, hideWebsitePreview, } = useWebsiteCrawl() - const { - prefix, - setPrefix, - keywords, - setKeywords, - startAfter, - setStartAfter, - selectedFileList, - setSelectedFileList, - fileList: onlineDriveFileList, - setFileList, - } = useOnlineDrive() + // const { } = useOnlineDrive() const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling @@ -127,7 +95,7 @@ const CreateFormPipeline = () => { return const datasourceInfoList: Record[] = [] if (datasourceType === DatasourceType.localFile) { - const { id, name, type, size, extension, mime_type } = previewFile.current as File + const { id, name, type, size, extension, mime_type } = previewFileRef.current as File const documentInfo = { related_id: id, name, @@ -141,7 +109,7 @@ const CreateFormPipeline = () => { datasourceInfoList.push(documentInfo) } if (datasourceType === DatasourceType.onlineDocument) { - const { workspace_id, ...rest } = previewOnlineDocument.current + const { workspace_id, ...rest } = previewOnlineDocumentRef.current! const documentInfo = { workspace_id, page: rest, @@ -162,7 +130,7 @@ const CreateFormPipeline = () => { setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) }, }) - }, [datasource, datasourceType, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline]) + }, [datasource, datasourceType, pipelineId, previewFileRef, previewOnlineDocumentRef, previewWebsitePage, runPublishedPipeline]) const handleProcess = useCallback(async (data: Record) => { if (!datasource) @@ -230,14 +198,14 @@ const CreateFormPipeline = () => { }, [handlePreviewChunks, handleProcess]) const handlePreviewFileChange = useCallback((file: DocumentItem) => { - previewFile.current = file + previewFileRef.current = file onClickPreview() - }, [onClickPreview, previewFile]) + }, [onClickPreview, previewFileRef]) const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { - previewOnlineDocument.current = page + previewOnlineDocumentRef.current = page onClickPreview() - }, [onClickPreview, previewOnlineDocument]) + }, [onClickPreview, previewOnlineDocumentRef]) const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { previewWebsitePage.current = website @@ -272,59 +240,26 @@ const CreateFormPipeline = () => { /> {datasourceType === DatasourceType.localFile && ( )} {datasourceType === DatasourceType.onlineDocument && ( )} {datasourceType === DatasourceType.websiteCrawl && ( )} {datasourceType === DatasourceType.onlineDrive && ( )} {isShowVectorSpaceFull && ( @@ -362,15 +297,25 @@ const CreateFormPipeline = () => { currentStep === 1 && (
- {currentFile && } + {currentLocalFile && ( + + )} {currentDocument && ( + )} + {currentWebsite && ( + )} - {currentWebsite && }
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx index 02c230344e..d444b2eb47 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx @@ -1,12 +1,12 @@ 'use client' -import React from 'react' +import React, { useMemo } from 'react' import { useTranslation } from 'react-i18next' import Loading from './loading' import type { CustomFile as File } from '@/models/datasets' import { RiCloseLine } from '@remixicon/react' import { useFilePreview } from '@/service/use-common' import DocumentFileIcon from '../../../common/document-file-icon' -import { formatNumberAbbreviated } from '@/utils/format' +import { formatFileSize, formatNumberAbbreviated } from '@/utils/format' type FilePreviewProps = { file: File @@ -20,31 +20,19 @@ const FilePreview = ({ const { t } = useTranslation() const { data: fileData, isFetching } = useFilePreview(file.id || '') - const getFileName = (currentFile?: File) => { - if (!currentFile) + const fileName = useMemo(() => { + if (!file) return '' - const arr = currentFile.name.split('.') + const arr = file.name.split('.') return arr.slice(0, -1).join() - } - - const getFileSize = (fileSize: number) => { - if (!fileSize) - return fileSize - const units = ['', 'K', 'M', 'G', 'T', 'P'] - let index = 0 - while (fileSize >= 1024 && index < units.length) { - fileSize = fileSize / 1024 - index++ - } - return `${fileSize.toFixed(2)} ${units[index]}B` - } + }, [file]) return (
{t('datasetPipeline.addDocuments.stepOne.preview')}
-
{`${getFileName(file)}.${file.extension}`}
+
{`${fileName}.${file.extension || ''}`}
{file.extension} · - {getFileSize(file.size)} + {formatFileSize(file.size)} {fileData && ( <> · diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx index fa04f15fcb..ce7a5da24c 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx @@ -6,12 +6,12 @@ import { RiCloseLine, RiGlobalLine } from '@remixicon/react' import { formatNumberAbbreviated } from '@/utils/format' type WebsitePreviewProps = { - payload: CrawlResultItem + currentWebsite: CrawlResultItem hidePreview: () => void } const WebsitePreview = ({ - payload, + currentWebsite, hidePreview, }: WebsitePreviewProps) => { const { t } = useTranslation() @@ -21,13 +21,13 @@ const WebsitePreview = ({
{t('datasetPipeline.addDocuments.stepOne.preview')}
-
{payload.title}
+
{currentWebsite.title}
- {payload.source_url} + {currentWebsite.source_url} · · - {`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`} + {`${formatNumberAbbreviated(currentWebsite.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}
- {payload.markdown} + {currentWebsite.markdown}
) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts index 8a1a6539c4..75d3220432 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts @@ -5,12 +5,7 @@ import { useNodes } from 'reactflow' import { BlockEnum } from '@/app/components/workflow/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' import { useCallback, useMemo, useState } from 'react' -import type { CrawlResult } from '@/models/datasets' -import { type CrawlResultItem, CrawlStep, type FileItem } from '@/models/datasets' -import produce from 'immer' -import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' import type { OnlineDriveFile } from '@/models/pipeline' -import { OnlineDriveFileType } from '@/models/pipeline' export const useTestRunSteps = () => { const { t } = useTranslation() @@ -57,131 +52,27 @@ export const useDatasourceOptions = () => { data: node.data, }) }) - if (process.env.NODE_ENV === 'development') { - // todo: delete mock data - options.push({ - label: 'Google Drive', - value: '123456', - // @ts-expect-error mock data - data: { - datasource_parameters: {}, - datasource_configurations: {}, - type: BlockEnum.DataSource, - title: 'Google Drive', - plugin_id: 'langgenius/google-drive', - provider_type: 'online_drive', - provider_name: 'google_drive', - datasource_name: 'google-drive', - datasource_label: 'Google Drive', - selected: false, - }, - }) - } return options }, [datasourceNodes]) return options } -export const useLocalFile = () => { - const [fileList, setFileList] = useState([]) - - const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { - const newList = produce(list, (draft) => { - const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) - draft[targetIndex] = { - ...draft[targetIndex], - progress, - } - }) - setFileList(newList) - } - - const updateFileList = (preparedFiles: FileItem[]) => { - setFileList(preparedFiles) - } - - return { - fileList, - updateFile, - updateFileList, - } -} - -export const useOnlineDocuments = () => { - const [documentsData, setDocumentsData] = useState([]) - const [searchValue, setSearchValue] = useState('') - const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') - const [onlineDocuments, setOnlineDocuments] = useState([]) - - const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => { - const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { - next.pages.forEach((page) => { - prev[page.page_id] = { - ...page, - workspace_id: next.workspace_id, - } - }) - - return prev - }, {}) - return pagesMap - }, [documentsData]) - const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])] - const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) - - const updateOnlineDocuments = (value: NotionPage[]) => { - setOnlineDocuments(value) - } - - return { - documentsData, - setDocumentsData, - searchValue, - setSearchValue, - currentWorkspaceId, - setCurrentWorkspaceId, - PagesMapAndSelectedPagesId, - selectedPagesId, - setSelectedPagesId, - onlineDocuments, - updateOnlineDocuments, - } -} - -export const useWebsiteCrawl = () => { - const [websitePages, setWebsitePages] = useState([]) - const [crawlResult, setCrawlResult] = useState() - const [step, setStep] = useState(CrawlStep.init) - - return { - crawlResult, - setCrawlResult, - websitePages, - setWebsitePages, - step, - setStep, - } -} - export const useOnlineDrive = () => { const [prefix, setPrefix] = useState([]) const [keywords, setKeywords] = useState('') + const [bucket, setBucket] = useState('') const [startAfter, setStartAfter] = useState('') const [selectedFileList, setSelectedFileList] = useState([]) - const [fileList, setFileList] = useState([ - { - key: 'Bucket_1', - size: 1024, // unit bytes - type: OnlineDriveFileType.bucket, - }, - ]) + const [fileList, setFileList] = useState([]) return { prefix, setPrefix, keywords, setKeywords, + bucket, + setBucket, startAfter, setStartAfter, selectedFileList, diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index 148ba7e636..591f1d74a7 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -1,6 +1,6 @@ import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store' import { useCallback, useMemo, useState } from 'react' -import { useLocalFile, useOnlineDocuments, useOnlineDrive, useTestRunSteps, useWebsiteCrawl } from './hooks' +import { useTestRunSteps } from './hooks' import DataSourceOptions from './data-source-options' import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file' import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents' @@ -16,9 +16,14 @@ import CloseButton from './close-button' import Header from './header' import FooterTips from './footer-tips' import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider' +import { useDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store' const TestRunPanel = () => { const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel) + const fileList = useDataSourceStore(state => state.localFileList) + const onlineDocuments = useDataSourceStore(state => state.onlineDocuments) + const websitePages = useDataSourceStore(state => state.websitePages) + const selectedFileList = useDataSourceStore(state => state.selectedFileList) const [datasource, setDatasource] = useState() const { @@ -27,45 +32,6 @@ const TestRunPanel = () => { handleNextStep, handleBackStep, } = useTestRunSteps() - const { - fileList, - updateFile, - updateFileList, - } = useLocalFile() - const { - documentsData, - setDocumentsData, - searchValue, - setSearchValue, - currentWorkspaceId, - setCurrentWorkspaceId, - PagesMapAndSelectedPagesId, - selectedPagesId, - setSelectedPagesId, - onlineDocuments, - updateOnlineDocuments, - } = useOnlineDocuments() - const { - crawlResult, - setCrawlResult, - websitePages, - setWebsitePages, - step, - setStep, - } = useWebsiteCrawl() - const { - prefix, - setPrefix, - keywords, - setKeywords, - startAfter, - setStartAfter, - selectedFileList, - setSelectedFileList, - fileList: onlineDriveFileList, - setFileList, - } = useOnlineDrive() - const { handleRun } = useWorkflowRun() const datasourceType = datasource?.nodeData.provider_type @@ -77,12 +43,16 @@ const TestRunPanel = () => { return !onlineDocuments.length if (datasourceType === DatasourceType.websiteCrawl) return !websitePages.length + if (datasourceType === DatasourceType.onlineDrive) + return !selectedFileList.length return false - }, [datasource, datasourceType, fileList, onlineDocuments.length, websitePages.length]) + }, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileList.length, websitePages.length]) - const handleClose = () => { + const handleClose = useCallback(() => { setShowDebugAndPreviewPanel(false) - } + }, [setShowDebugAndPreviewPanel]) + + const { handleRun } = useWorkflowRun() const handleProcess = useCallback((data: Record) => { if (!datasource) @@ -137,57 +107,29 @@ const TestRunPanel = () => { /> {datasourceType === DatasourceType.localFile && ( )} {datasourceType === DatasourceType.onlineDocument && ( )} {datasourceType === DatasourceType.websiteCrawl && ( )} {datasourceType === DatasourceType.onlineDrive && ( )}