refactor: add create-from-pipeline page and associated components for document processing

2025-05-21 16:37:02 +08:00 · 2025-05-21 16:37:02 +08:00 · b18519b824
parent 9aef4b6d6b
commit b18519b824
20 changed files with 414 additions and 157 deletions
--- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/documents/create-from-pipeline/page.tsx
+++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/documents/create-from-pipeline/page.tsx
@ -0,0 +1,10 @@
+import React from 'react'
+import CreateFromPipeline from '@/app/components/datasets/documents/create-from-pipeline'
+
+const CreateFromPipelinePage = async () => {
+  return (
+    <CreateFromPipeline />
+  )
+}
+
+export default CreateFromPipelinePage
--- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout-main.tsx
+++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout-main.tsx
@ -109,7 +109,7 @@ const DatasetDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
    params: { datasetId },
  } = props
  const pathname = usePathname()
-  const hideSideBar = /documents\/create$/.test(pathname)
+  const hideSideBar = pathname.endsWith('documents/create') || pathname.endsWith('documents/create-from-pipeline')
  const { t } = useTranslation()
  const { isCurrentWorkspaceDatasetOperator } = useAppContext()

--- a/web/app/(commonLayout)/datasets/Datasets.tsx
+++ b/web/app/(commonLayout)/datasets/Datasets.tsx
@ -1,96 +0,0 @@
-'use client'
-
-import { useCallback, useEffect, useRef } from 'react'
-import useSWRInfinite from 'swr/infinite'
-import { debounce } from 'lodash-es'
-import NewDatasetCard from './NewDatasetCard'
-import DatasetCard from '@/app/components/datasets/list/dataset-card'
-import type { DataSetListResponse, FetchDatasetsParams } from '@/models/datasets'
-import { fetchDatasets } from '@/service/datasets'
-import { useAppContext } from '@/context/app-context'
-import { useTranslation } from 'react-i18next'
-
-const getKey = (
-  pageIndex: number,
-  previousPageData: DataSetListResponse,
-  tags: string[],
-  keyword: string,
-  includeAll: boolean,
-) => {
-  if (!pageIndex || previousPageData.has_more) {
-    const params: FetchDatasetsParams = {
-      url: 'datasets',
-      params: {
-        page: pageIndex + 1,
-        limit: 30,
-        include_all: includeAll,
-      },
-    }
-    if (tags.length)
-      params.params.tag_ids = tags
-    if (keyword)
-      params.params.keyword = keyword
-    return params
-  }
-  return null
-}
-
-type Props = {
-  containerRef: React.RefObject<HTMLDivElement>
-  tags: string[]
-  keywords: string
-  includeAll: boolean
-}
-
-const Datasets = ({
-  containerRef,
-  tags,
-  keywords,
-  includeAll,
-}: Props) => {
-  const { t } = useTranslation()
-  const { isCurrentWorkspaceEditor } = useAppContext()
-  const { data, isLoading, setSize, mutate } = useSWRInfinite(
-    (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords, includeAll),
-    fetchDatasets,
-    { revalidateFirstPage: false, revalidateAll: true },
-  )
-  const loadingStateRef = useRef(false)
-  const anchorRef = useRef<HTMLAnchorElement>(null)
-
-  useEffect(() => {
-    loadingStateRef.current = isLoading
-  }, [isLoading, t])
-
-  const onScroll = useCallback(
-    debounce(() => {
-      if (!loadingStateRef.current && containerRef.current && anchorRef.current) {
-        const { scrollTop, clientHeight } = containerRef.current
-        const anchorOffset = anchorRef.current.offsetTop
-        if (anchorOffset - scrollTop - clientHeight < 100)
-          setSize(size => size + 1)
-      }
-    }, 50),
-    [setSize],
-  )
-
-  useEffect(() => {
-    const currentContainer = containerRef.current
-    currentContainer?.addEventListener('scroll', onScroll)
-    return () => {
-      currentContainer?.removeEventListener('scroll', onScroll)
-      onScroll.cancel()
-    }
-  }, [onScroll])
-
-  return (
-    <nav className='grid shrink-0 grow grid-cols-1 content-start gap-4 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4'>
-      {isCurrentWorkspaceEditor && <NewDatasetCard ref={anchorRef} />}
-      {data?.map(({ data: datasets }) => datasets.map(dataset => (
-        <DatasetCard key={dataset.id} dataset={dataset} onSuccess={mutate} />),
-      ))}
-    </nav>
-  )
-}
-
-export default Datasets
--- a/web/app/(commonLayout)/datasets/NewDatasetCard.tsx
+++ b/web/app/(commonLayout)/datasets/NewDatasetCard.tsx
@ -1,42 +0,0 @@
-'use client'
-import { useTranslation } from 'react-i18next'
-import { basePath } from '@/utils/var'
-import {
-  RiAddLine,
-  RiArrowRightLine,
-} from '@remixicon/react'
-
-const CreateAppCard = (
-  {
-    ref,
-    ..._
-  },
-) => {
-  const { t } = useTranslation()
-
-  return (
-    <div className='bg-background-default-dimm flex min-h-[160px] flex-col rounded-xl border-[0.5px]
-      border-components-panel-border transition-all duration-200 ease-in-out'
-    >
-      <a ref={ref} className='group flex grow cursor-pointer items-start p-4' href={`${basePath}/datasets/create`}>
-        <div className='flex items-center gap-3'>
-          <div className='flex h-10 w-10 items-center justify-center rounded-lg border border-dashed border-divider-regular bg-background-default-lighter
-            p-2 group-hover:border-solid group-hover:border-effects-highlight group-hover:bg-background-default-dodge'
-          >
-            <RiAddLine className='h-4 w-4 text-text-tertiary group-hover:text-text-accent'/>
-          </div>
-          <div className='system-md-semibold text-text-secondary group-hover:text-text-accent'>{t('dataset.createDataset')}</div>
-        </div>
-      </a>
-      <div className='system-xs-regular p-4 pt-0 text-text-tertiary'>{t('dataset.createDatasetIntro')}</div>
-      <a className='group flex cursor-pointer items-center gap-1 rounded-b-xl border-t-[0.5px] border-divider-subtle p-4' href={`${basePath}/datasets/connect`}>
-        <div className='system-xs-medium text-text-tertiary group-hover:text-text-accent'>{t('dataset.connectDataset')}</div>
-        <RiArrowRightLine className='h-3.5 w-3.5 text-text-tertiary group-hover:text-text-accent' />
-      </a>
-    </div>
-  )
-}
-
-CreateAppCard.displayName = 'CreateAppCard'
-
-export default CreateAppCard
--- a/web/app/components/datasets/create-from-pipeline/index.tsx
+++ b/web/app/components/datasets/create-from-pipeline/index.tsx
@ -7,8 +7,7 @@ import Effect from '../../base/effect'
 const CreateFromPipeline = () => {
  return (
    <div
-      className='relative flex flex-col rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
-      style={{ height: 'calc(100vh - 56px)' }}
+      className='relative flex h-[calc(100vh-56px)] flex-col rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
    >
      <Effect className='left-8 top-[-34px] opacity-20' />
      <Header />
--- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
+++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts
@ -0,0 +1,21 @@
+import { useTranslation } from 'react-i18next'
+import { AddDocumentsStep } from './types'
+
+export const useAddDocumentsSteps = () => {
+  const { t } = useTranslation()
+  const steps = [
+    {
+      label: t('datasetPipeline.addDocuments.steps.chooseDatasource'),
+      value: AddDocumentsStep.dataSource,
+    },
+    {
+      label: t('datasetPipeline.addDocuments.steps.ProcessDocuments'),
+      value: AddDocumentsStep.processDocuments,
+    },
+    {
+      label: t('datasetPipeline.addDocuments.steps.ProcessingDocuments'),
+      value: AddDocumentsStep.processingDocuments,
+    },
+  ]
+  return steps
+}
--- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx
@ -0,0 +1,217 @@
+'use client'
+import { useCallback, useMemo, useState } from 'react'
+// import StepIndicator from './step-indicator'
+// import { useTestRunSteps } from './hooks'
+// import DataSourceOptions from './data-source-options'
+import type { CrawlResultItem, FileItem } from '@/models/datasets'
+import { DataSourceType } from '@/models/datasets'
+// import LocalFile from './data-source/local-file'
+import produce from 'immer'
+import { useProviderContextSelector } from '@/context/provider-context'
+import { DataSourceProvider, type NotionPage } from '@/models/common'
+// import Notion from './data-source/notion'
+import VectorSpaceFull from '@/app/components/billing/vector-space-full'
+// import Firecrawl from './data-source/website/firecrawl'
+// import JinaReader from './data-source/website/jina-reader'
+// import WaterCrawl from './data-source/website/water-crawl'
+// import Actions from './data-source/actions'
+// import DocumentProcessing from './document-processing'
+import { useTranslation } from 'react-i18next'
+import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
+import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
+import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
+import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl'
+import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader'
+import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl'
+import Actions from '@/app/components/rag-pipeline/components/panel/test-run/data-source/actions'
+import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing'
+import LeftHeader from './left-header'
+// import { usePipelineRun } from '../../../hooks'
+// import type { Datasource } from './types'
+
+const TestRunPanel = () => {
+  const { t } = useTranslation()
+  const [currentStep, setCurrentStep] = useState(1)
+  const [datasource, setDatasource] = useState<Datasource>()
+  const [fileList, setFiles] = useState<FileItem[]>([])
+  const [notionPages, setNotionPages] = useState<NotionPage[]>([])
+  const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
+  const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
+
+  const plan = useProviderContextSelector(state => state.plan)
+  const enableBilling = useProviderContextSelector(state => state.enableBilling)
+
+  // const steps = useTestRunSteps()
+
+  const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
+  const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
+  const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
+  const notSupportBatchUpload = enableBilling && plan.type === 'sandbox'
+  const nextDisabled = useMemo(() => {
+    if (!fileList.length)
+      return true
+    if (fileList.some(file => !file.file.id))
+      return true
+    return isShowVectorSpaceFull
+  }, [fileList, isShowVectorSpaceFull])
+
+  const nextBtnDisabled = useMemo(() => {
+    if (!datasource) return true
+    if (datasource.type === DataSourceType.FILE)
+      return nextDisabled
+    if (datasource.type === DataSourceType.NOTION)
+      return isShowVectorSpaceFull || !notionPages.length
+    if (datasource.type === DataSourceProvider.fireCrawl
+      || datasource.type === DataSourceProvider.jinaReader
+      || datasource.type === DataSourceProvider.waterCrawl)
+      return isShowVectorSpaceFull || !websitePages.length
+    return false
+  }, [datasource, nextDisabled, isShowVectorSpaceFull, notionPages.length, websitePages.length])
+
+  const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
+    const newList = produce(list, (draft) => {
+      const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
+      draft[targetIndex] = {
+        ...draft[targetIndex],
+        progress,
+      }
+    })
+    setFiles(newList)
+  }
+
+  const updateFileList = (preparedFiles: FileItem[]) => {
+    setFiles(preparedFiles)
+  }
+
+  const updateNotionPages = (value: NotionPage[]) => {
+    setNotionPages(value)
+  }
+
+  const handleNextStep = useCallback(() => {
+    setCurrentStep(preStep => preStep + 1)
+  }, [])
+
+  const handleBackStep = useCallback(() => {
+    setCurrentStep(preStep => preStep - 1)
+  }, [])
+
+  // const { handleRun } = usePipelineRun()
+
+  const handleProcess = useCallback((data: Record<string, any>) => {
+    if (!datasource)
+      return
+    const datasourceInfo: Record<string, any> = {}
+    let datasource_type = ''
+    if (datasource.type === DataSourceType.FILE) {
+      datasource_type = 'local_file'
+      datasourceInfo.fileId = fileList.map(file => file.fileID)
+    }
+    if (datasource.type === DataSourceType.NOTION) {
+      datasource_type = 'online_document'
+      datasourceInfo.workspaceId = notionPages[0].workspace_id
+      datasourceInfo.page = notionPages.map((page) => {
+        const { workspace_id, ...rest } = page
+        return rest
+      })
+    }
+    if (datasource.type === DataSourceProvider.fireCrawl
+      || datasource.type === DataSourceProvider.jinaReader
+      || datasource.type === DataSourceProvider.waterCrawl) {
+      datasource_type = 'website_crawl'
+      datasourceInfo.jobId = websiteCrawlJobId
+      datasourceInfo.result = websitePages
+    }
+    // handleRun({
+    //   inputs: data,
+    //   datasource_type,
+    //   datasource_info: datasourceInfo,
+    // })
+  }, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages])
+
+  return (
+    <div
+      className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
+    >
+      <div className='flex flex-1 flex-col px-14'>
+        <LeftHeader
+          title={t('datasetPipeline.addDocuments.title')}
+          currentStep={currentStep}
+        />
+        <div className='grow overflow-y-auto'>
+          {
+            currentStep === 1 && (
+              <>
+                <div className='flex flex-col gap-y-4 px-4 py-2'>
+                  {/* <DataSourceOptions
+                  datasourceNodeId={datasource?.nodeId || ''}
+                  onSelect={setDatasource}
+                /> */}
+                  {datasource?.type === DataSourceType.FILE && (
+                    <LocalFile
+                      files={fileList}
+                      updateFile={updateFile}
+                      updateFileList={updateFileList}
+                      notSupportBatchUpload={notSupportBatchUpload}
+                    />
+                  )}
+                  {datasource?.type === DataSourceType.NOTION && (
+                    <Notion
+                      nodeId={datasource?.nodeId || ''}
+                      notionPages={notionPages}
+                      updateNotionPages={updateNotionPages}
+                    />
+                  )}
+                  {datasource?.type === DataSourceProvider.fireCrawl && (
+                    <FireCrawl
+                      nodeId={datasource?.nodeId || ''}
+                      variables={datasource?.variables}
+                      checkedCrawlResult={websitePages}
+                      onCheckedCrawlResultChange={setWebsitePages}
+                      onJobIdChange={setWebsiteCrawlJobId}
+                    />
+                  )}
+                  {datasource?.type === DataSourceProvider.jinaReader && (
+                    <JinaReader
+                      nodeId={datasource?.nodeId || ''}
+                      variables={datasource?.variables}
+                      checkedCrawlResult={websitePages}
+                      onCheckedCrawlResultChange={setWebsitePages}
+                      onJobIdChange={setWebsiteCrawlJobId}
+                    />
+                  )}
+                  {datasource?.type === DataSourceProvider.waterCrawl && (
+                    <WaterCrawl
+                      nodeId={datasource?.nodeId || ''}
+                      variables={datasource?.variables}
+                      checkedCrawlResult={websitePages}
+                      onCheckedCrawlResultChange={setWebsitePages}
+                      onJobIdChange={setWebsiteCrawlJobId}
+                    />
+                  )}
+                  {isShowVectorSpaceFull && (
+                    <VectorSpaceFull />
+                  )}
+                </div>
+                <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
+              </>
+            )
+          }
+          {
+            currentStep === 2 && (
+              <DocumentProcessing
+                dataSourceNodeId={datasource?.nodeId || ''}
+                onProcess={handleProcess}
+                onBack={handleBackStep}
+              />
+            )
+          }
+        </div>
+      </div>
+      {/* Preview */}
+      <div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'>
+      </div>
+    </div>
+  )
+}
+
+export default TestRunPanel
--- a/web/app/components/datasets/documents/create-from-pipeline/left-header.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/left-header.tsx
@ -0,0 +1,46 @@
+import React from 'react'
+import { RiArrowLeftLine } from '@remixicon/react'
+import Button from '@/app/components/base/button'
+import { useParams } from 'next/navigation'
+import Effect from '@/app/components/base/effect'
+import { useAddDocumentsSteps } from './hooks'
+import StepIndicator from './step-indicator'
+
+type LeftHeaderProps = {
+  title: string
+  currentStep: number
+}
+
+const LeftHeader = ({
+  title,
+  currentStep,
+}: LeftHeaderProps) => {
+  const { datasetId } = useParams()
+  const steps = useAddDocumentsSteps()
+
+  return (
+    <div className='relative flex flex-col gap-y-0.5 pb-2 pt-4'>
+      <div className='flex items-center gap-x-2'>
+        <span className='system-2xs-semibold-uppercase bg-pipeline-add-documents-title-bg bg-clip-text text-transparent'>
+          {title}
+        </span>
+        <span className='system-2xs-regular text-divider-regular'>/</span>
+        <StepIndicator steps={steps} currentStep={currentStep} />
+      </div>
+      <div className='system-md-semibold text-text-primary'>
+        {steps[currentStep - 1]?.label}
+      </div>
+      <a
+        className='absolute -left-11 top-3.5'
+        href={`/datasets/${datasetId}/documents`}
+      >
+        <Button variant='secondary-accent' className='size-9 rounded-full p-0'>
+          <RiArrowLeftLine className='size-5 ' />
+        </Button>
+      </a>
+      <Effect className='left-8 top-[-34px] opacity-20' />
+    </div>
+  )
+}
+
+export default React.memo(LeftHeader)
--- a/web/app/components/datasets/documents/create-from-pipeline/step-indicator.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/step-indicator.tsx
@ -0,0 +1,33 @@
+import cn from '@/utils/classnames'
+import React from 'react'
+
+type Step = {
+  label: string
+  value: string
+}
+
+type StepIndicatorProps = {
+  currentStep: number
+  steps: Step[]
+}
+
+const StepIndicator = ({
+  currentStep,
+  steps,
+}: StepIndicatorProps) => {
+  return (
+    <div className='flex gap-x-1'>
+      {steps.map((step, index) => {
+        const isActive = index === currentStep - 1
+        return (
+          <div
+            key={step.value}
+            className={cn('h-1 w-1 rounded-lg bg-divider-solid', isActive && 'w-2 bg-state-accent-solid')}
+          />
+        )
+      })}
+    </div>
+  )
+}
+
+export default React.memo(StepIndicator)
--- a/web/app/components/datasets/documents/create-from-pipeline/types.tsx
+++ b/web/app/components/datasets/documents/create-from-pipeline/types.tsx
@ -0,0 +1,5 @@
+export enum AddDocumentsStep {
+  dataSource = 'dataSource',
+  processDocuments = 'processDocuments',
+  processingDocuments = 'processingDocuments',
+}
--- a/web/app/components/datasets/documents/detail/completed/segment-card/index.tsx
+++ b/web/app/components/datasets/documents/detail/completed/segment-card/index.tsx
@ -1,7 +1,7 @@
 import React, { type FC, useCallback, useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { RiDeleteBinLine, RiEditLine } from '@remixicon/react'
-import { StatusItem } from '../../../list'
+import StatusItem from '../../../status-item'
 import { useDocumentContext } from '../../index'
 import ChildSegmentList from '../child-segment-list'
 import Tag from '../common/tag'
@ -228,15 +228,15 @@ const SegmentCard: FC<ISegmentCardProps> = ({
      }
      {
        isParagraphMode && child_chunks.length > 0
-          && <ChildSegmentList
-            parentChunkId={id}
-            childChunks={child_chunks}
-            enabled={enabled}
-            onDelete={onDeleteChildChunk!}
-            handleAddNewChildChunk={handleAddNewChildChunk}
-            onClickSlice={onClickSlice}
-            focused={focused.segmentContent}
-          />
+        && <ChildSegmentList
+          parentChunkId={id}
+          childChunks={child_chunks}
+          enabled={enabled}
+          onDelete={onDeleteChildChunk!}
+          handleAddNewChildChunk={handleAddNewChildChunk}
+          onClickSlice={onClickSlice}
+          focused={focused.segmentContent}
+        />
      }
      {showModal
        && <Confirm
--- a/web/app/components/datasets/documents/detail/index.tsx
+++ b/web/app/components/datasets/documents/detail/index.tsx
@ -5,7 +5,8 @@ import { createContext, useContext, useContextSelector } from 'use-context-selec
 import { useTranslation } from 'react-i18next'
 import { useRouter } from 'next/navigation'
 import { RiArrowLeftLine, RiLayoutLeft2Line, RiLayoutRight2Line } from '@remixicon/react'
-import { OperationAction, StatusItem } from '../list'
+import Operations from '../operations'
+import StatusItem from '../status-item'
 import DocumentPicker from '../../common/document-picker'
 import Completed from './completed'
 import Embedding from './embedding'
@ -228,7 +229,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
              datasetId={datasetId}
              onUpdate={handleOperate}
            />
-            <OperationAction
+            <Operations
              scene='detail'
              embeddingAvailable={embeddingAvailable}
              detail={{
--- a/web/app/components/datasets/documents/index.tsx
+++ b/web/app/components/datasets/documents/index.tsx
@ -15,7 +15,7 @@ import Button from '@/app/components/base/button'
 import Input from '@/app/components/base/input'
 import { get } from '@/service/base'
 import { createDocument } from '@/service/datasets'
-import { useDatasetDetailContext } from '@/context/dataset-detail'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import { NotionPageSelectorModal } from '@/app/components/base/notion-page-selector'
 import type { NotionPage } from '@/models/common'
 import type { CreateDocumentReq } from '@/models/datasets'
@ -93,7 +93,7 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
  const [currPage, setCurrPage] = React.useState<number>(0)
  const [limit, setLimit] = useState<number>(DEFAULT_LIMIT)
  const router = useRouter()
-  const { dataset } = useDatasetDetailContext()
+  const dataset = useDatasetDetailContextWithSelector(s => s.dataset)
  const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false)
  const [timerCanRun, setTimerCanRun] = useState(true)
  const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION
@ -172,6 +172,11 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
  const total = documentsRes?.total || 0

  const routeToDocCreate = () => {
+    // if dataset is create from pipeline, redirect to create from pipeline page
+    if (dataset?.pipeline_id) {
+      router.push(`/datasets/${datasetId}/documents/create-from-pipeline`)
+      return
+    }
    if (isDataSourceNotion) {
      setNotionPageSelectorModalVisible(true)
      return
@ -267,7 +272,7 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
                ? 'https://docs.dify.ai/zh-hans/guides/knowledge-base/integrate-knowledge-within-application'
                : 'https://docs.dify.ai/en/guides/knowledge-base/integrate-knowledge-within-application'
            }
-            >
+          >
            <span>{t('datasetDocuments.list.learnMore')}</span>
            <RiExternalLinkLine className='h-3 w-3' />
          </a>
--- a/web/i18n/en-US/dataset-pipeline.ts
+++ b/web/i18n/en-US/dataset-pipeline.ts
@ -70,6 +70,15 @@ const translation = {
    addInputField: 'Add Input Field',
    editInputField: 'Edit Input Field',
  },
+  addDocuments: {
+    title: 'Add Documents',
+    steps: {
+      chooseDatasource: 'Choose a Data Source',
+      processDocuments: 'Process Documents',
+      processingDocuments: 'Processing Documents',
+    },
+    backToDataSource: 'Data Source',
+  },
 }

 export default translation
--- a/web/i18n/zh-Hans/dataset-pipeline.ts
+++ b/web/i18n/zh-Hans/dataset-pipeline.ts
@ -70,6 +70,15 @@ const translation = {
    addInputField: '添加输入字段',
    editInputField: '编辑输入字段',
  },
+  addDocuments: {
+    title: '添加文档',
+    steps: {
+      chooseDatasource: '选择数据源',
+      processDocuments: '处理文档',
+      processingDocuments: '正在处理文档',
+    },
+    backToDataSource: '数据源',
+  },
 }

 export default translation
--- a/web/models/pipeline.ts
+++ b/web/models/pipeline.ts
@ -1,4 +1,4 @@
-import type { Edge, Node, SupportUploadFileTypes } from '@/app/components/workflow/types'
+import type { Edge, EnvironmentVariable, Node, SupportUploadFileTypes } from '@/app/components/workflow/types'
 import type { DSLImportMode, DSLImportStatus } from './app'
 import type { ChunkingMode, DatasetPermission, IconInfo } from './datasets'
 import type { Dependency } from '@/app/components/plugins/types'
@ -143,3 +143,30 @@ export type PipelineDatasourceNodeRunRequest = {
 }

 export type PipelineDatasourceNodeRunResponse = Record<string, any>
+
+export type PublishedPipelineInfoResponse = {
+  id: string
+  graph: {
+    nodes: Node[]
+    edges: Edge[]
+    viewport: Viewport
+  }
+  created_at: number
+  created_by: {
+    id: string
+    name: string
+    email: string
+  }
+  hash: string
+  updated_at: number
+  updated_by: {
+    id: string
+    name: string
+    email: string
+  },
+  environment_variables?: EnvironmentVariable[]
+  rag_pipeline_variables?: RAGPipelineVariables
+  version: string
+  marked_name: string
+  marked_comment: string
+}
--- a/web/service/use-pipeline.ts
+++ b/web/service/use-pipeline.ts
@ -15,6 +15,7 @@ import type {
  PipelineTemplateByIdResponse,
  PipelineTemplateListParams,
  PipelineTemplateListResponse,
+  PublishedPipelineInfoResponse,
  UpdateTemplateInfoRequest,
  UpdateTemplateInfoResponse,
 } from '@/models/pipeline'
@ -156,3 +157,12 @@ export const useDataSourceList = (enabled?: boolean) => {
    retry: false,
  })
 }
+
+export const usePublishedPipelineInfo = (pipelineId: string) => {
+  return useQuery<PublishedPipelineInfoResponse>({
+    queryKey: [NAME_SPACE, 'published-pipeline', pipelineId],
+    queryFn: () => {
+      return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`)
+    },
+  })
+}
--- a/web/tailwind-common-config.ts
+++ b/web/tailwind-common-config.ts
@ -124,6 +124,7 @@ const config = {
        'tag-selector-mask-bg': 'var(--color-tag-selector-mask-bg)',
        'tag-selector-mask-hover-bg': 'var(--color-tag-selector-mask-hover-bg)',
        'pipeline-template-card-hover-bg': 'var(--color-pipeline-template-card-hover-bg)',
+        'pipeline-add-documents-title-bg': 'var(--color-pipeline-add-documents-title-bg)',
      },
      animation: {
        'spin-slow': 'spin 2s linear infinite',
--- a/web/themes/manual-dark.css
+++ b/web/themes/manual-dark.css
@ -67,4 +67,5 @@ html[data-theme="dark"] {
    --color-tag-selector-mask-bg: linear-gradient(90deg, rgba(34, 34, 37, 0) 0%, rgba(34, 34, 37, 1) 100%);
    --color-tag-selector-mask-hover-bg: linear-gradient(90deg, rgba(39, 39, 43, 0) 0%, rgba(39, 39, 43, 1) 100%);
    --color-pipeline-template-card-hover-bg: linear-gradient(0deg, rgba(58, 58, 64, 1) 60.27%, rgba(58, 58, 64, 0) 100%);
+    --color-pipeline-add-documents-title-bg: linear-gradient(92deg, rgba(54, 191, 250, 1) 0%, rgba(41, 109, 255, 1) 97.78%);
 }
--- a/web/themes/manual-light.css
+++ b/web/themes/manual-light.css
@ -67,4 +67,5 @@ html[data-theme="light"] {
    --color-tag-selector-mask-bg: linear-gradient(90deg, rgba(252, 252, 253, 0) 0%, rgba(252, 252, 253, 1) 100%);
    --color-tag-selector-mask-hover-bg: linear-gradient(90deg, rgba(255, 255, 255, 0) 0%, rgba(255, 255, 255, 1) 100%);
    --color-pipeline-template-card-hover-bg: linear-gradient(0deg, rgba(249, 250, 251, 1) 60.27%, rgba(249, 250, 251, 0) 100%);
+	--color-pipeline-add-documents-title-bg: linear-gradient(92deg, rgba(11, 165, 236, 0.95) 0%, rgba(21, 90, 239, 0.95) 97.78%);
 }