feat: implement document upload steps and enhance test run panel with new hooks and components

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -1,13 +1,26 @@
 import { useTranslation } from 'react-i18next'
 import { AddDocumentsStep } from './types'
 import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
-import { useMemo } from 'react'
+import { useCallback, useMemo, useRef, useState } from 'react'
 import { BlockEnum, type Node } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
 import type { DatasourceType } from '@/models/pipeline'
+import type { CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets'
+import produce from 'immer'
+import type { NotionPage } from '@/models/common'
 
 export const useAddDocumentsSteps = () => {
   const { t } = useTranslation()
+  const [currentStep, setCurrentStep] = useState(1)
+
+  const handleNextStep = useCallback(() => {
+    setCurrentStep(preStep => preStep + 1)
+  }, [])
+
+  const handleBackStep = useCallback(() => {
+    setCurrentStep(preStep => preStep - 1)
+  }, [])
+
   const steps = [
     {
       label: t('datasetPipeline.addDocuments.steps.chooseDatasource'),
@@ -22,7 +35,13 @@ export const useAddDocumentsSteps = () => {
       value: AddDocumentsStep.processingDocuments,
     },
   ]
-  return steps
+
+  return {
+    steps,
+    currentStep,
+    handleNextStep,
+    handleBackStep,
+  }
 }
 
 export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
@@ -56,3 +75,101 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
 
   return { datasources, options }
 }
+
+export const useLocalFile = () => {
+  const [fileList, setFileList] = useState<FileItem[]>([])
+  const [currentFile, setCurrentFile] = useState<File | undefined>()
+
+  const previewFile = useRef<DocumentItem>(fileList[0]?.file as DocumentItem)
+
+  const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
+
+  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
+    const newList = produce(list, (draft) => {
+      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
+      draft[targetIndex] = {
+        ...draft[targetIndex],
+        progress,
+      }
+    })
+    setFileList(newList)
+  }
+
+  const updateFileList = useCallback((preparedFiles: FileItem[]) => {
+    setFileList(preparedFiles)
+  }, [])
+
+  const updateCurrentFile = useCallback((file: File) => {
+    setCurrentFile(file)
+  }, [])
+
+  const hideFilePreview = useCallback(() => {
+    setCurrentFile(undefined)
+  }, [])
+
+  return {
+    fileList,
+    previewFile,
+    allFileLoaded,
+    updateFile,
+    updateFileList,
+    currentFile,
+    updateCurrentFile,
+    hideFilePreview,
+  }
+}
+
+export const useNotionsPages = () => {
+  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
+  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
+
+  const previewNotionPage = useRef<NotionPage>(notionPages[0])
+
+  const updateNotionPages = (value: NotionPage[]) => {
+    setNotionPages(value)
+  }
+
+  const updateCurrentPage = useCallback((page: NotionPage) => {
+    setCurrentNotionPage(page)
+  }, [])
+
+  const hideNotionPagePreview = useCallback(() => {
+    setCurrentNotionPage(undefined)
+  }, [])
+
+  return {
+    notionPages,
+    previewNotionPage,
+    updateNotionPages,
+    currentNotionPage,
+    updateCurrentPage,
+    hideNotionPagePreview,
+  }
+}
+
+export const useWebsiteCrawl = () => {
+  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
+  const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
+  const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
+
+  const previewWebsitePage = useRef<CrawlResultItem>(websitePages[0])
+
+  const updateCurrentWebsite = useCallback((website: CrawlResultItem) => {
+    setCurrentWebsite(website)
+  }, [])
+
+  const hideWebsitePreview = useCallback(() => {
+    setCurrentWebsite(undefined)
+  }, [])
+
+  return {
+    websitePages,
+    websiteCrawlJobId,
+    previewWebsitePage,
+    setWebsitePages,
+    setWebsiteCrawlJobId,
+    currentWebsite,
+    updateCurrentWebsite,
+    hideWebsitePreview,
+  }
+}

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -1,9 +1,8 @@
 'use client'
 import { useCallback, useMemo, useRef, useState } from 'react'
 import DataSourceOptions from './data-source-options'
-import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse, FileItem } from '@/models/datasets'
+import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets'
 import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
-import produce from 'immer'
 import { useProviderContextSelector } from '@/context/provider-context'
 import type { NotionPage } from '@/models/common'
 import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
@@ -26,108 +25,72 @@ import ChunkPreview from './preview/chunk-preview'
 import Processing from './processing'
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
+import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks'
 
 const TestRunPanel = () => {
   const { t } = useTranslation()
-  const [currentStep, setCurrentStep] = useState(1)
-  const [datasource, setDatasource] = useState<Datasource>()
-  const [fileList, setFiles] = useState<FileItem[]>([])
-  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
-  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
-  const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
-  const [currentFile, setCurrentFile] = useState<File | undefined>()
-  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
-  const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
-  const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
-
   const plan = useProviderContextSelector(state => state.plan)
   const enableBilling = useProviderContextSelector(state => state.enableBilling)
   const datasetId = useDatasetDetailContextWithSelector(s => s.dataset?.id)
   const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
   const indexingType = useDatasetDetailContextWithSelector(s => s.dataset?.indexing_technique)
   const retrievalMethod = useDatasetDetailContextWithSelector(s => s.dataset?.retrieval_model_dict.search_method)
+  const [datasource, setDatasource] = useState<Datasource>()
+  const [estimateData, setEstimateData] = useState<FileIndexingEstimateResponse | undefined>(undefined)
 
   const isPreview = useRef(false)
   const formRef = useRef<any>(null)
-  const previewFile = useRef<DocumentItem>(fileList[0].file as DocumentItem)
-  const previewNotionPage = useRef<NotionPage>(notionPages[0])
-  const previewWebsitePage = useRef<CrawlResultItem>(websitePages[0])
 
   const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')
 
-  const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
+  const {
+    currentStep,
+    handleNextStep,
+    handleBackStep,
+  } = useAddDocumentsSteps()
+  const {
+    fileList,
+    previewFile,
+    allFileLoaded,
+    updateFile,
+    updateFileList,
+    currentFile,
+    updateCurrentFile,
+    hideFilePreview,
+  } = useLocalFile()
+  const {
+    notionPages,
+    previewNotionPage,
+    updateNotionPages,
+    currentNotionPage,
+    updateCurrentPage,
+    hideNotionPagePreview,
+  } = useNotionsPages()
+  const {
+    websitePages,
+    websiteCrawlJobId,
+    previewWebsitePage,
+    setWebsitePages,
+    setWebsiteCrawlJobId,
+    currentWebsite,
+    updateCurrentWebsite,
+    hideWebsitePreview,
+  } = useWebsiteCrawl()
+
   const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
   const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
   const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
-  const nextDisabled = useMemo(() => {
-    if (!fileList.length)
-      return true
-    if (fileList.some(file => !file.file.id))
-      return true
-    return isShowVectorSpaceFull
-  }, [fileList, isShowVectorSpaceFull])
 
   const nextBtnDisabled = useMemo(() => {
     if (!datasource) return true
     if (datasource.type === DatasourceType.localFile)
-      return nextDisabled
+      return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
     if (datasource.type === DatasourceType.onlineDocument)
       return isShowVectorSpaceFull || !notionPages.length
     if (datasource.type === DatasourceType.websiteCrawl)
       return isShowVectorSpaceFull || !websitePages.length
     return false
-  }, [datasource, nextDisabled, isShowVectorSpaceFull, notionPages.length, websitePages.length])
+  }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
-
-  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
-    const newList = produce(list, (draft) => {
-      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
-      draft[targetIndex] = {
-        ...draft[targetIndex],
-        progress,
-      }
-    })
-    setFiles(newList)
-  }
-
-  const updateFileList = useCallback((preparedFiles: FileItem[]) => {
-    setFiles(preparedFiles)
-  }, [])
-
-  const updateNotionPages = useCallback((value: NotionPage[]) => {
-    setNotionPages(value)
-  }, [])
-
-  const updateCurrentFile = useCallback((file: File) => {
-    setCurrentFile(file)
-  }, [])
-
-  const hideFilePreview = useCallback(() => {
-    setCurrentFile(undefined)
-  }, [])
-
-  const updateCurrentPage = useCallback((page: NotionPage) => {
-    setCurrentNotionPage(page)
-  }, [])
-
-  const hideNotionPagePreview = useCallback(() => {
-    setCurrentNotionPage(undefined)
-  }, [])
-
-  const updateCurrentWebsite = useCallback((website: CrawlResultItem) => {
-    setCurrentWebsite(website)
-  }, [])
-
-  const hideWebsitePreview = useCallback(() => {
-    setCurrentWebsite(undefined)
-  }, [])
-
-  const handleNextStep = useCallback(() => {
-    setCurrentStep(preStep => preStep + 1)
-  }, [])
-
-  const handleBackStep = useCallback(() => {
-    setCurrentStep(preStep => preStep - 1)
-  }, [])
 
   const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
 
@@ -176,7 +139,7 @@ const TestRunPanel = () => {
         setEstimateData(res.data.outputs as FileIndexingEstimateResponse)
       },
     })
-  }, [datasource, pipelineId, runPublishedPipeline, websiteCrawlJobId])
+  }, [datasource, pipelineId, previewFile, previewNotionPage, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
 
   const handleProcess = useCallback(async (data: Record<string, any>) => {
     if (!datasource)
@@ -246,17 +209,17 @@ const TestRunPanel = () => {
   const handlePreviewFileChange = useCallback((file: DocumentItem) => {
     previewFile.current = file
     onClickPreview()
-  }, [onClickPreview])
+  }, [onClickPreview, previewFile])
 
   const handlePreviewNotionPageChange = useCallback((page: NotionPage) => {
     previewNotionPage.current = page
     onClickPreview()
-  }, [onClickPreview])
+  }, [onClickPreview, previewNotionPage])
 
   const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
     previewWebsitePage.current = website
     onClickPreview()
-  }, [onClickPreview])
+  }, [onClickPreview, previewWebsitePage])
 
   if (isFetchingPipelineInfo) {
     return (

web/app/components/rag-pipeline/components/panel/test-run/close-button.tsx

@@ -0,0 +1,22 @@
+import React from 'react'
+import { RiCloseLine } from '@remixicon/react'
+
+type CloseButtonProps = {
+  handleClose: () => void
+}
+
+const CloseButton = ({
+  handleClose,
+}: CloseButtonProps) => {
+  return (
+    <button
+      type='button'
+      className='absolute right-2.5 top-2.5 flex size-8 items-center justify-center p-1.5'
+      onClick={handleClose}
+    >
+      <RiCloseLine className='size-4 text-text-tertiary' />
+    </button>
+  )
+}
+
+export default React.memo(CloseButton)

web/app/components/rag-pipeline/components/panel/test-run/header.tsx

@@ -0,0 +1,31 @@
+import Tooltip from '@/app/components/base/tooltip'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import StepIndicator from './step-indicator'
+
+type HeaderProps = {
+  steps: { label: string; value: string }[]
+  currentStep: number
+}
+
+const Header = ({
+  steps,
+  currentStep,
+}: HeaderProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex flex-col gap-y-0.5 px-3 pb-2 pt-3.5'>
+      <div className='flex items-center gap-x-1 pl-1 pr-8'>
+        <span className='system-md-semibold-uppercase text-text-primary'>{t('datasetPipeline.testRun.title')}</span>
+        <Tooltip
+          popupContent={t('datasetPipeline.testRun.tooltip')}
+          popupClassName='max-w-[240px]'
+        />
+      </div>
+      <StepIndicator steps={steps} currentStep={currentStep} />
+    </div>
+  )
+}
+
+export default React.memo(Header)

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -4,11 +4,24 @@ import { TestRunStep } from './types'
 import { useNodes } from 'reactflow'
 import { BlockEnum } from '@/app/components/workflow/types'
 import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
-import { useMemo } from 'react'
+import { useCallback, useMemo, useState } from 'react'
 import type { DatasourceType } from '@/models/pipeline'
+import type { CrawlResultItem, FileItem } from '@/models/datasets'
+import produce from 'immer'
+import type { NotionPage } from '@/models/common'
 
 export const useTestRunSteps = () => {
   const { t } = useTranslation()
+  const [currentStep, setCurrentStep] = useState(1)
+
+  const handleNextStep = useCallback(() => {
+    setCurrentStep(preStep => preStep + 1)
+  }, [])
+
+  const handleBackStep = useCallback(() => {
+    setCurrentStep(preStep => preStep - 1)
+  }, [])
+
   const steps = [
     {
       label: t('datasetPipeline.testRun.steps.dataSource'),
@@ -19,7 +32,13 @@ export const useTestRunSteps = () => {
       value: TestRunStep.documentProcessing,
     },
   ]
-  return steps
+
+  return {
+    steps,
+    currentStep,
+    handleNextStep,
+    handleBackStep,
+  }
 }
 
 export const useDatasourceOptions = () => {
@@ -54,3 +73,56 @@ export const useDatasourceOptions = () => {
 
   return { datasources, options }
 }
+
+export const useLocalFile = () => {
+  const [fileList, setFileList] = useState<FileItem[]>([])
+
+  const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
+
+  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
+    const newList = produce(list, (draft) => {
+      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
+      draft[targetIndex] = {
+        ...draft[targetIndex],
+        progress,
+      }
+    })
+    setFileList(newList)
+  }
+
+  const updateFileList = (preparedFiles: FileItem[]) => {
+    setFileList(preparedFiles)
+  }
+
+  return {
+    fileList,
+    allFileLoaded,
+    updateFile,
+    updateFileList,
+  }
+}
+
+export const useNotionPages = () => {
+  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
+
+  const updateNotionPages = (value: NotionPage[]) => {
+    setNotionPages(value)
+  }
+
+  return {
+    notionPages,
+    updateNotionPages,
+  }
+}
+
+export const useWebsiteCrawl = () => {
+  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
+  const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
+
+  return {
+    websitePages,
+    websiteCrawlJobId,
+    setWebsitePages,
+    setWebsiteCrawlJobId,
+  }
+}

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -1,96 +1,69 @@
 import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store'
-import { RiCloseLine } from '@remixicon/react'
 import { useCallback, useMemo, useState } from 'react'
-import StepIndicator from './step-indicator'
+import { useLocalFile, useNotionPages, useTestRunSteps, useWebsiteCrawl } from './hooks'
-import { useTestRunSteps } from './hooks'
 import DataSourceOptions from './data-source-options'
-import type { CrawlResultItem, FileItem } from '@/models/datasets'
 import LocalFile from './data-source/local-file'
-import produce from 'immer'
 import { useProviderContextSelector } from '@/context/provider-context'
-import type { NotionPage } from '@/models/common'
 import Notion from './data-source/notion'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import WebsiteCrawl from './data-source/website-crawl'
 import Actions from './data-source/actions'
 import DocumentProcessing from './document-processing'
-import { useTranslation } from 'react-i18next'
 import { usePipelineRun } from '../../../hooks'
 import type { Datasource } from './types'
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
-import Tooltip from '@/app/components/base/tooltip'
+import CloseButton from './close-button'
+import Header from './header'
 
 const TestRunPanel = () => {
-  const { t } = useTranslation()
   const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
-  const [currentStep, setCurrentStep] = useState(1)
-  const [datasource, setDatasource] = useState<Datasource>()
-  const [fileList, setFiles] = useState<FileItem[]>([])
-  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
-  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
-  const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
-
   const plan = useProviderContextSelector(state => state.plan)
   const enableBilling = useProviderContextSelector(state => state.enableBilling)
+  const [datasource, setDatasource] = useState<Datasource>()
 
-  const steps = useTestRunSteps()
+  const {
+    steps,
+    currentStep,
+    handleNextStep,
+    handleBackStep,
+  } = useTestRunSteps()
+  const {
+    fileList,
+    allFileLoaded,
+    updateFile,
+    updateFileList,
+  } = useLocalFile()
+  const {
+    notionPages,
+    updateNotionPages,
+  } = useNotionPages()
+  const {
+    websitePages,
+    websiteCrawlJobId,
+    setWebsitePages,
+    setWebsiteCrawlJobId,
+  } = useWebsiteCrawl()
+  const { handleRun } = usePipelineRun()
 
-  const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
   const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
   const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
-  const nextDisabled = useMemo(() => {
-    if (!fileList.length)
-      return true
-    if (fileList.some(file => !file.file.id))
-      return true
-    return isShowVectorSpaceFull
-  }, [fileList, isShowVectorSpaceFull])
 
   const nextBtnDisabled = useMemo(() => {
     if (!datasource) return true
     if (datasource.type === DatasourceType.localFile)
-      return nextDisabled
+      return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
     if (datasource.type === DatasourceType.onlineDocument)
       return isShowVectorSpaceFull || !notionPages.length
     if (datasource.type === DatasourceType.websiteCrawl)
       return isShowVectorSpaceFull || !websitePages.length
     return false
-  }, [datasource, nextDisabled, isShowVectorSpaceFull, notionPages.length, websitePages.length])
+  }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
 
   const handleClose = () => {
     setShowDebugAndPreviewPanel(false)
   }
 
-  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
-    const newList = produce(list, (draft) => {
-      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
-      draft[targetIndex] = {
-        ...draft[targetIndex],
-        progress,
-      }
-    })
-    setFiles(newList)
-  }
-
-  const updateFileList = (preparedFiles: FileItem[]) => {
-    setFiles(preparedFiles)
-  }
-
-  const updateNotionPages = (value: NotionPage[]) => {
-    setNotionPages(value)
-  }
-
-  const handleNextStep = useCallback(() => {
-    setCurrentStep(preStep => preStep + 1)
-  }, [])
-
-  const handleBackStep = useCallback(() => {
-    setCurrentStep(preStep => preStep - 1)
-  }, [])
-
-  const { handleRun } = usePipelineRun()
-
   const handleProcess = useCallback((data: Record<string, any>) => {
     if (!datasource)
       return
@@ -136,23 +109,8 @@ const TestRunPanel = () => {
     <div
       className='relative flex h-full w-[480px] flex-col rounded-l-2xl border-y-[0.5px] border-l-[0.5px] border-components-panel-border bg-components-panel-bg shadow-xl shadow-shadow-shadow-1'
     >
-      <button
+      <CloseButton handleClose={handleClose} />
-        type='button'
+      <Header steps={steps} currentStep={currentStep} />
-        className='absolute right-2.5 top-2.5 flex size-8 items-center justify-center p-1.5'
-        onClick={handleClose}
-      >
-        <RiCloseLine className='size-4 text-text-tertiary' />
-      </button>
-      <div className='flex flex-col gap-y-0.5 px-3 pb-2 pt-3.5'>
-        <div className='flex items-center gap-x-1 pl-1 pr-8'>
-          <span className='system-md-semibold-uppercase text-text-primary'>{t('datasetPipeline.testRun.title')}</span>
-          <Tooltip
-            popupContent={t('datasetPipeline.testRun.tooltip')}
-            popupClassName='max-w-[240px]'
-          />
-        </div>
-        <StepIndicator steps={steps} currentStep={currentStep} />
-      </div>
       <div className='grow overflow-y-auto'>
         {
           currentStep === 1 && (