From 300541957345ccdeea6aa133e29f989f4df2b221 Mon Sep 17 00:00:00 2001 From: twwu Date: Wed, 28 May 2025 18:34:26 +0800 Subject: [PATCH] feat: implement document upload steps and enhance test run panel with new hooks and components --- .../documents/create-from-pipeline/hooks.ts | 121 ++++++++++++++++- .../documents/create-from-pipeline/index.tsx | 125 ++++++------------ .../panel/test-run/close-button.tsx | 22 +++ .../components/panel/test-run/header.tsx | 31 +++++ .../components/panel/test-run/hooks.ts | 76 ++++++++++- .../components/panel/test-run/index.tsx | 104 +++++---------- 6 files changed, 321 insertions(+), 158 deletions(-) create mode 100644 web/app/components/rag-pipeline/components/panel/test-run/close-button.tsx create mode 100644 web/app/components/rag-pipeline/components/panel/test-run/header.tsx diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 3fcdde5abf..da27a22cfa 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -1,13 +1,26 @@ import { useTranslation } from 'react-i18next' import { AddDocumentsStep } from './types' import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' -import { useMemo } from 'react' +import { useCallback, useMemo, useRef, useState } from 'react' import { BlockEnum, type Node } from '@/app/components/workflow/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' import type { DatasourceType } from '@/models/pipeline' +import type { CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets' +import produce from 'immer' +import type { NotionPage } from '@/models/common' export const useAddDocumentsSteps = () => { const { t } = useTranslation() + const [currentStep, setCurrentStep] = useState(1) + + const handleNextStep = useCallback(() => { + setCurrentStep(preStep => preStep + 1) + }, []) + + const handleBackStep = useCallback(() => { + setCurrentStep(preStep => preStep - 1) + }, []) + const steps = [ { label: t('datasetPipeline.addDocuments.steps.chooseDatasource'), @@ -22,7 +35,13 @@ export const useAddDocumentsSteps = () => { value: AddDocumentsStep.processingDocuments, }, ] - return steps + + return { + steps, + currentStep, + handleNextStep, + handleBackStep, + } } export const useDatasourceOptions = (pipelineNodes: Node[]) => { @@ -56,3 +75,101 @@ export const useDatasourceOptions = (pipelineNodes: Node[]) return { datasources, options } } + +export const useLocalFile = () => { + const [fileList, setFileList] = useState([]) + const [currentFile, setCurrentFile] = useState() + + const previewFile = useRef(fileList[0]?.file as DocumentItem) + + const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList]) + + const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { + const newList = produce(list, (draft) => { + const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) + draft[targetIndex] = { + ...draft[targetIndex], + progress, + } + }) + setFileList(newList) + } + + const updateFileList = useCallback((preparedFiles: FileItem[]) => { + setFileList(preparedFiles) + }, []) + + const updateCurrentFile = useCallback((file: File) => { + setCurrentFile(file) + }, []) + + const hideFilePreview = useCallback(() => { + setCurrentFile(undefined) + }, []) + + return { + fileList, + previewFile, + allFileLoaded, + updateFile, + updateFileList, + currentFile, + updateCurrentFile, + hideFilePreview, + } +} + +export const useNotionsPages = () => { + const [notionPages, setNotionPages] = useState([]) + const [currentNotionPage, setCurrentNotionPage] = useState() + + const previewNotionPage = useRef(notionPages[0]) + + const updateNotionPages = (value: NotionPage[]) => { + setNotionPages(value) + } + + const updateCurrentPage = useCallback((page: NotionPage) => { + setCurrentNotionPage(page) + }, []) + + const hideNotionPagePreview = useCallback(() => { + setCurrentNotionPage(undefined) + }, []) + + return { + notionPages, + previewNotionPage, + updateNotionPages, + currentNotionPage, + updateCurrentPage, + hideNotionPagePreview, + } +} + +export const useWebsiteCrawl = () => { + const [websitePages, setWebsitePages] = useState([]) + const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') + const [currentWebsite, setCurrentWebsite] = useState() + + const previewWebsitePage = useRef(websitePages[0]) + + const updateCurrentWebsite = useCallback((website: CrawlResultItem) => { + setCurrentWebsite(website) + }, []) + + const hideWebsitePreview = useCallback(() => { + setCurrentWebsite(undefined) + }, []) + + return { + websitePages, + websiteCrawlJobId, + previewWebsitePage, + setWebsitePages, + setWebsiteCrawlJobId, + currentWebsite, + updateCurrentWebsite, + hideWebsitePreview, + } +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 3e727b4ba7..f8b45dff8c 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -1,9 +1,8 @@ 'use client' import { useCallback, useMemo, useRef, useState } from 'react' import DataSourceOptions from './data-source-options' -import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse, FileItem } from '@/models/datasets' +import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets' import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file' -import produce from 'immer' import { useProviderContextSelector } from '@/context/provider-context' import type { NotionPage } from '@/models/common' import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion' @@ -26,108 +25,72 @@ import ChunkPreview from './preview/chunk-preview' import Processing from './processing' import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' +import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks' const TestRunPanel = () => { const { t } = useTranslation() - const [currentStep, setCurrentStep] = useState(1) - const [datasource, setDatasource] = useState() - const [fileList, setFiles] = useState([]) - const [notionPages, setNotionPages] = useState([]) - const [websitePages, setWebsitePages] = useState([]) - const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') - const [currentFile, setCurrentFile] = useState() - const [currentNotionPage, setCurrentNotionPage] = useState() - const [currentWebsite, setCurrentWebsite] = useState() - const [estimateData, setEstimateData] = useState(undefined) - const plan = useProviderContextSelector(state => state.plan) const enableBilling = useProviderContextSelector(state => state.enableBilling) const datasetId = useDatasetDetailContextWithSelector(s => s.dataset?.id) const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) const indexingType = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique) const retrievalMethod = useDatasetDetailContextWithSelector(s => s.dataset?.retrieval_model_dict.search_method) + const [datasource, setDatasource] = useState() + const [estimateData, setEstimateData] = useState(undefined) const isPreview = useRef(false) const formRef = useRef(null) - const previewFile = useRef(fileList[0].file as DocumentItem) - const previewNotionPage = useRef(notionPages[0]) - const previewWebsitePage = useRef(websitePages[0]) const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '') - const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) + const { + currentStep, + handleNextStep, + handleBackStep, + } = useAddDocumentsSteps() + const { + fileList, + previewFile, + allFileLoaded, + updateFile, + updateFileList, + currentFile, + updateCurrentFile, + hideFilePreview, + } = useLocalFile() + const { + notionPages, + previewNotionPage, + updateNotionPages, + currentNotionPage, + updateCurrentPage, + hideNotionPagePreview, + } = useNotionsPages() + const { + websitePages, + websiteCrawlJobId, + previewWebsitePage, + setWebsitePages, + setWebsiteCrawlJobId, + currentWebsite, + updateCurrentWebsite, + hideWebsitePreview, + } = useWebsiteCrawl() + const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling const notSupportBatchUpload = enableBilling && plan.type === 'sandbox' - const nextDisabled = useMemo(() => { - if (!fileList.length) - return true - if (fileList.some(file => !file.file.id)) - return true - return isShowVectorSpaceFull - }, [fileList, isShowVectorSpaceFull]) const nextBtnDisabled = useMemo(() => { if (!datasource) return true if (datasource.type === DatasourceType.localFile) - return nextDisabled + return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id) if (datasource.type === DatasourceType.onlineDocument) return isShowVectorSpaceFull || !notionPages.length if (datasource.type === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length return false - }, [datasource, nextDisabled, isShowVectorSpaceFull, notionPages.length, websitePages.length]) - - const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { - const newList = produce(list, (draft) => { - const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) - draft[targetIndex] = { - ...draft[targetIndex], - progress, - } - }) - setFiles(newList) - } - - const updateFileList = useCallback((preparedFiles: FileItem[]) => { - setFiles(preparedFiles) - }, []) - - const updateNotionPages = useCallback((value: NotionPage[]) => { - setNotionPages(value) - }, []) - - const updateCurrentFile = useCallback((file: File) => { - setCurrentFile(file) - }, []) - - const hideFilePreview = useCallback(() => { - setCurrentFile(undefined) - }, []) - - const updateCurrentPage = useCallback((page: NotionPage) => { - setCurrentNotionPage(page) - }, []) - - const hideNotionPagePreview = useCallback(() => { - setCurrentNotionPage(undefined) - }, []) - - const updateCurrentWebsite = useCallback((website: CrawlResultItem) => { - setCurrentWebsite(website) - }, []) - - const hideWebsitePreview = useCallback(() => { - setCurrentWebsite(undefined) - }, []) - - const handleNextStep = useCallback(() => { - setCurrentStep(preStep => preStep + 1) - }, []) - - const handleBackStep = useCallback(() => { - setCurrentStep(preStep => preStep - 1) - }, []) + }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length]) const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline() @@ -176,7 +139,7 @@ const TestRunPanel = () => { setEstimateData(res.data.outputs as FileIndexingEstimateResponse) }, }) - }, [datasource, pipelineId, runPublishedPipeline, websiteCrawlJobId]) + }, [datasource, pipelineId, previewFile, previewNotionPage, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId]) const handleProcess = useCallback(async (data: Record) => { if (!datasource) @@ -246,17 +209,17 @@ const TestRunPanel = () => { const handlePreviewFileChange = useCallback((file: DocumentItem) => { previewFile.current = file onClickPreview() - }, [onClickPreview]) + }, [onClickPreview, previewFile]) const handlePreviewNotionPageChange = useCallback((page: NotionPage) => { previewNotionPage.current = page onClickPreview() - }, [onClickPreview]) + }, [onClickPreview, previewNotionPage]) const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { previewWebsitePage.current = website onClickPreview() - }, [onClickPreview]) + }, [onClickPreview, previewWebsitePage]) if (isFetchingPipelineInfo) { return ( diff --git a/web/app/components/rag-pipeline/components/panel/test-run/close-button.tsx b/web/app/components/rag-pipeline/components/panel/test-run/close-button.tsx new file mode 100644 index 0000000000..cdca16323a --- /dev/null +++ b/web/app/components/rag-pipeline/components/panel/test-run/close-button.tsx @@ -0,0 +1,22 @@ +import React from 'react' +import { RiCloseLine } from '@remixicon/react' + +type CloseButtonProps = { + handleClose: () => void +} + +const CloseButton = ({ + handleClose, +}: CloseButtonProps) => { + return ( + + ) +} + +export default React.memo(CloseButton) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/header.tsx b/web/app/components/rag-pipeline/components/panel/test-run/header.tsx new file mode 100644 index 0000000000..f6f91f7a8f --- /dev/null +++ b/web/app/components/rag-pipeline/components/panel/test-run/header.tsx @@ -0,0 +1,31 @@ +import Tooltip from '@/app/components/base/tooltip' +import React from 'react' +import { useTranslation } from 'react-i18next' +import StepIndicator from './step-indicator' + +type HeaderProps = { + steps: { label: string; value: string }[] + currentStep: number +} + +const Header = ({ + steps, + currentStep, +}: HeaderProps) => { + const { t } = useTranslation() + + return ( +
+
+ {t('datasetPipeline.testRun.title')} + +
+ +
+ ) +} + +export default React.memo(Header) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts index 56f5352b01..661a021a6a 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts @@ -4,11 +4,24 @@ import { TestRunStep } from './types' import { useNodes } from 'reactflow' import { BlockEnum } from '@/app/components/workflow/types' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -import { useMemo } from 'react' +import { useCallback, useMemo, useState } from 'react' import type { DatasourceType } from '@/models/pipeline' +import type { CrawlResultItem, FileItem } from '@/models/datasets' +import produce from 'immer' +import type { NotionPage } from '@/models/common' export const useTestRunSteps = () => { const { t } = useTranslation() + const [currentStep, setCurrentStep] = useState(1) + + const handleNextStep = useCallback(() => { + setCurrentStep(preStep => preStep + 1) + }, []) + + const handleBackStep = useCallback(() => { + setCurrentStep(preStep => preStep - 1) + }, []) + const steps = [ { label: t('datasetPipeline.testRun.steps.dataSource'), @@ -19,7 +32,13 @@ export const useTestRunSteps = () => { value: TestRunStep.documentProcessing, }, ] - return steps + + return { + steps, + currentStep, + handleNextStep, + handleBackStep, + } } export const useDatasourceOptions = () => { @@ -54,3 +73,56 @@ export const useDatasourceOptions = () => { return { datasources, options } } + +export const useLocalFile = () => { + const [fileList, setFileList] = useState([]) + + const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList]) + + const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { + const newList = produce(list, (draft) => { + const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) + draft[targetIndex] = { + ...draft[targetIndex], + progress, + } + }) + setFileList(newList) + } + + const updateFileList = (preparedFiles: FileItem[]) => { + setFileList(preparedFiles) + } + + return { + fileList, + allFileLoaded, + updateFile, + updateFileList, + } +} + +export const useNotionPages = () => { + const [notionPages, setNotionPages] = useState([]) + + const updateNotionPages = (value: NotionPage[]) => { + setNotionPages(value) + } + + return { + notionPages, + updateNotionPages, + } +} + +export const useWebsiteCrawl = () => { + const [websitePages, setWebsitePages] = useState([]) + const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') + + return { + websitePages, + websiteCrawlJobId, + setWebsitePages, + setWebsiteCrawlJobId, + } +} diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index 17fe9a7334..078bce24d7 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -1,96 +1,69 @@ import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store' -import { RiCloseLine } from '@remixicon/react' import { useCallback, useMemo, useState } from 'react' -import StepIndicator from './step-indicator' -import { useTestRunSteps } from './hooks' +import { useLocalFile, useNotionPages, useTestRunSteps, useWebsiteCrawl } from './hooks' import DataSourceOptions from './data-source-options' -import type { CrawlResultItem, FileItem } from '@/models/datasets' import LocalFile from './data-source/local-file' -import produce from 'immer' import { useProviderContextSelector } from '@/context/provider-context' -import type { NotionPage } from '@/models/common' import Notion from './data-source/notion' import VectorSpaceFull from '@/app/components/billing/vector-space-full' import WebsiteCrawl from './data-source/website-crawl' import Actions from './data-source/actions' import DocumentProcessing from './document-processing' -import { useTranslation } from 'react-i18next' import { usePipelineRun } from '../../../hooks' import type { Datasource } from './types' import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' -import Tooltip from '@/app/components/base/tooltip' +import CloseButton from './close-button' +import Header from './header' const TestRunPanel = () => { - const { t } = useTranslation() const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel) - const [currentStep, setCurrentStep] = useState(1) - const [datasource, setDatasource] = useState() - const [fileList, setFiles] = useState([]) - const [notionPages, setNotionPages] = useState([]) - const [websitePages, setWebsitePages] = useState([]) - const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') - const plan = useProviderContextSelector(state => state.plan) const enableBilling = useProviderContextSelector(state => state.enableBilling) + const [datasource, setDatasource] = useState() - const steps = useTestRunSteps() + const { + steps, + currentStep, + handleNextStep, + handleBackStep, + } = useTestRunSteps() + const { + fileList, + allFileLoaded, + updateFile, + updateFileList, + } = useLocalFile() + const { + notionPages, + updateNotionPages, + } = useNotionPages() + const { + websitePages, + websiteCrawlJobId, + setWebsitePages, + setWebsiteCrawlJobId, + } = useWebsiteCrawl() + const { handleRun } = usePipelineRun() - const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling - const nextDisabled = useMemo(() => { - if (!fileList.length) - return true - if (fileList.some(file => !file.file.id)) - return true - return isShowVectorSpaceFull - }, [fileList, isShowVectorSpaceFull]) const nextBtnDisabled = useMemo(() => { if (!datasource) return true if (datasource.type === DatasourceType.localFile) - return nextDisabled + return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id) if (datasource.type === DatasourceType.onlineDocument) return isShowVectorSpaceFull || !notionPages.length if (datasource.type === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length return false - }, [datasource, nextDisabled, isShowVectorSpaceFull, notionPages.length, websitePages.length]) + }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length]) const handleClose = () => { setShowDebugAndPreviewPanel(false) } - const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => { - const newList = produce(list, (draft) => { - const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID) - draft[targetIndex] = { - ...draft[targetIndex], - progress, - } - }) - setFiles(newList) - } - - const updateFileList = (preparedFiles: FileItem[]) => { - setFiles(preparedFiles) - } - - const updateNotionPages = (value: NotionPage[]) => { - setNotionPages(value) - } - - const handleNextStep = useCallback(() => { - setCurrentStep(preStep => preStep + 1) - }, []) - - const handleBackStep = useCallback(() => { - setCurrentStep(preStep => preStep - 1) - }, []) - - const { handleRun } = usePipelineRun() - const handleProcess = useCallback((data: Record) => { if (!datasource) return @@ -136,23 +109,8 @@ const TestRunPanel = () => {
- -
-
- {t('datasetPipeline.testRun.title')} - -
- -
+ +
{ currentStep === 1 && (