refactor: refactor preview components

web/app/components/base/form/form-scenarios/demo/index.tsx

@@ -24,7 +24,7 @@ const DemoForm = () => {
     },
   })
 
-const name = useStore(form.store, state => state.values.name)
+  const name = useStore(form.store, state => state.values.name)
 
   return (
     <form
@@ -59,7 +59,7 @@ const name = useStore(form.store, state => state.values.name)
         )
       }
       <form.AppForm>
-        <form.SubmitButton>Submit</form.SubmitButton>
+        <form.Actions />
       </form.AppForm>
     </form>
   )

web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx

@@ -0,0 +1,62 @@
+import { useCallback, useEffect } from 'react'
+import { useDatasourceOptions } from '../hooks'
+import OptionCard from './option-card'
+import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge'
+import { Notion } from '@/app/components/base/icons/src/public/common'
+import { Jina } from '@/app/components/base/icons/src/public/llm'
+import { DataSourceType } from '@/models/datasets'
+import { DataSourceProvider } from '@/models/common'
+import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
+import type { Node } from '@/app/components/workflow/types'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+
+type DataSourceOptionsProps = {
+  pipelineNodes: Node<DataSourceNodeType>[]
+  datasourceNodeId: string
+  onSelect: (option: Datasource) => void
+}
+
+const DATA_SOURCE_ICONS = {
+  [DataSourceType.FILE]: File as React.FC<React.SVGProps<SVGSVGElement>>,
+  [DataSourceType.NOTION]: Notion as React.FC<React.SVGProps<SVGSVGElement>>,
+  [DataSourceProvider.fireCrawl]: '🔥',
+  [DataSourceProvider.jinaReader]: Jina as React.FC<React.SVGProps<SVGSVGElement>>,
+  [DataSourceProvider.waterCrawl]: Watercrawl as React.FC<React.SVGProps<SVGSVGElement>>,
+}
+
+const DataSourceOptions = ({
+  pipelineNodes,
+  datasourceNodeId,
+  onSelect,
+}: DataSourceOptionsProps) => {
+  const { datasources, options } = useDatasourceOptions(pipelineNodes)
+
+  const handelSelect = useCallback((value: string) => {
+    const selectedOption = datasources.find(option => option.nodeId === value)
+    if (!selectedOption)
+      return
+    onSelect(selectedOption)
+  }, [datasources, onSelect])
+
+  useEffect(() => {
+    if (options.length > 0)
+      handelSelect(options[0].value)
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
+  return (
+    <div className='grid w-full grid-cols-4 gap-1'>
+      {options.map(option => (
+        <OptionCard
+          key={option.value}
+          label={option.label}
+          selected={datasourceNodeId === option.value}
+          Icon={DATA_SOURCE_ICONS[option.type as keyof typeof DATA_SOURCE_ICONS]}
+          onClick={handelSelect.bind(null, option.value)}
+        />
+      ))}
+    </div>
+  )
+}
+
+export default DataSourceOptions

web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx

@@ -0,0 +1,41 @@
+import React from 'react'
+import cn from '@/utils/classnames'
+
+type OptionCardProps = {
+  label: string
+  Icon: React.FC<React.SVGProps<SVGSVGElement>> | string
+  selected: boolean
+  onClick?: () => void
+}
+
+const OptionCard = ({
+  label,
+  Icon,
+  selected,
+  onClick,
+}: OptionCardProps) => {
+  return (
+    <div
+      className={cn(
+        'flex items-center gap-2 rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg p-3 shadow-shadow-shadow-3',
+        selected
+          ? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg shadow-xs ring-[0.5px] ring-inset ring-components-option-card-option-selected-border'
+          : 'hover:bg-components-option-card-bg-hover hover:border-components-option-card-option-border-hover hover:shadow-xs',
+      )}
+      onClick={onClick}
+    >
+      <div className='flex size-8 items-center justify-center rounded-lg border-[0.5px] border-components-panel-border bg-background-default-dodge p-1.5'>
+        {
+          typeof Icon === 'string'
+            ? <div className='text-[18px] leading-[18px]'>{Icon}</div>
+            : <Icon className='size-5' />
+        }
+      </div>
+      <div className={cn('system-sm-medium text-text-secondary', selected && 'text-primary')}>
+        {label}
+      </div>
+    </div>
+  )
+}
+
+export default React.memo(OptionCard)

web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx

@@ -0,0 +1,44 @@
+import React from 'react'
+import Button from '@/app/components/base/button'
+import { useTranslation } from 'react-i18next'
+import { useParams } from 'next/navigation'
+import { RiArrowRightLine } from '@remixicon/react'
+
+type ActionsProps = {
+  disabled?: boolean
+  handleNextStep: () => void
+}
+
+const Actions = ({
+  disabled,
+  handleNextStep,
+}: ActionsProps) => {
+  const { t } = useTranslation()
+  const { datasetId } = useParams()
+
+  return (
+    <div className='flex justify-end gap-x-2'>
+      <a
+        href={`/datasets/${datasetId}/documents`}
+      >
+        <Button
+          variant='ghost'
+          className='px-3 py-2'
+        >
+          {t('common.operation.cancel')}
+        </Button>
+      </a>
+      <Button
+        disabled={disabled}
+        variant='primary'
+        onClick={handleNextStep}
+        className='gap-x-0.5'
+      >
+        <span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
+        <RiArrowRightLine className='size-4' />
+      </Button>
+    </div>
+  )
+}
+
+export default React.memo(Actions)

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -1,5 +1,11 @@
 import { useTranslation } from 'react-i18next'
 import { AddDocumentsStep } from './types'
+import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
+import { useMemo } from 'react'
+import { BlockEnum, type Node } from '@/app/components/workflow/types'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import { DataSourceType } from '@/models/datasets'
+import { DataSourceProvider } from '@/models/common'
 
 export const useAddDocumentsSteps = () => {
   const { t } = useTranslation()
@@ -19,3 +25,79 @@ export const useAddDocumentsSteps = () => {
   ]
   return steps
 }
+
+export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
+  const { t } = useTranslation()
+  const datasources: Datasource[] = useMemo(() => {
+    const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
+    return datasourceNodes.map((node) => {
+      let type: DataSourceType | DataSourceProvider = DataSourceType.FILE
+      switch (node.data.tool_name) {
+        case 'file_upload':
+          type = DataSourceType.FILE
+          break
+        case 'search_notion':
+          type = DataSourceType.NOTION
+          break
+        case 'firecrawl':
+          type = DataSourceProvider.fireCrawl
+          break
+        case 'jina_reader':
+          type = DataSourceProvider.jinaReader
+          break
+        case 'water_crawl':
+          type = DataSourceProvider.waterCrawl
+          break
+      }
+      return {
+        nodeId: node.id,
+        type,
+        variables: node.data.variables,
+      }
+    })
+  }, [pipelineNodes])
+
+  const options = useMemo(() => {
+    const options: DataSourceOption[] = []
+    datasources.forEach((source) => {
+      if (source.type === DataSourceType.FILE) {
+        options.push({
+          label: t('datasetPipeline.testRun.dataSource.localFiles'),
+          value: source.nodeId,
+          type: DataSourceType.FILE,
+        })
+      }
+      if (source.type === DataSourceType.NOTION) {
+        options.push({
+          label: 'Notion',
+          value: source.nodeId,
+          type: DataSourceType.NOTION,
+        })
+      }
+      if (source.type === DataSourceProvider.fireCrawl) {
+        options.push({
+          label: 'Firecrawl',
+          value: source.nodeId,
+          type: DataSourceProvider.fireCrawl,
+        })
+      }
+      if (source.type === DataSourceProvider.jinaReader) {
+        options.push({
+          label: 'Jina Reader',
+          value: source.nodeId,
+          type: DataSourceProvider.jinaReader,
+        })
+      }
+      if (source.type === DataSourceProvider.waterCrawl) {
+        options.push({
+          label: 'Water Crawl',
+          value: source.nodeId,
+          type: DataSourceProvider.waterCrawl,
+        })
+      }
+    })
+    return options
+  }, [datasources, t])
+
+  return { datasources, options }
+}

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -1,33 +1,30 @@
 'use client'
 import { useCallback, useMemo, useState } from 'react'
-// import StepIndicator from './step-indicator'
+import DataSourceOptions from './data-source-options'
-// import { useTestRunSteps } from './hooks'
+import type { CrawlResultItem, CustomFile as File, FileItem } from '@/models/datasets'
-// import DataSourceOptions from './data-source-options'
-import type { CrawlResultItem, FileItem } from '@/models/datasets'
 import { DataSourceType } from '@/models/datasets'
-// import LocalFile from './data-source/local-file'
+import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
 import produce from 'immer'
 import { useProviderContextSelector } from '@/context/provider-context'
 import { DataSourceProvider, type NotionPage } from '@/models/common'
-// import Notion from './data-source/notion'
-import VectorSpaceFull from '@/app/components/billing/vector-space-full'
-// import Firecrawl from './data-source/website/firecrawl'
-// import JinaReader from './data-source/website/jina-reader'
-// import WaterCrawl from './data-source/website/water-crawl'
-// import Actions from './data-source/actions'
-// import DocumentProcessing from './document-processing'
-import { useTranslation } from 'react-i18next'
-import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
-import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
 import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
+import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl'
 import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader'
 import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl'
-import Actions from '@/app/components/rag-pipeline/components/panel/test-run/data-source/actions'
+import Actions from './data-source/actions'
 import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing'
+import { useTranslation } from 'react-i18next'
+import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
 import LeftHeader from './left-header'
-// import { usePipelineRun } from '../../../hooks'
+import { usePublishedPipelineInfo } from '@/service/use-pipeline'
-// import type { Datasource } from './types'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import Loading from '@/app/components/base/loading'
+import type { Node } from '@/app/components/workflow/types'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import FilePreview from './preview/file-preview'
+import NotionPagePreview from './preview/notion-page-preview'
+import WebsitePreview from './preview/web-preview'
 
 const TestRunPanel = () => {
   const { t } = useTranslation()
@@ -37,11 +34,15 @@ const TestRunPanel = () => {
   const [notionPages, setNotionPages] = useState<NotionPage[]>([])
   const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
   const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
+  const [currentFile, setCurrentFile] = useState<File | undefined>()
+  const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
+  const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
 
   const plan = useProviderContextSelector(state => state.plan)
   const enableBilling = useProviderContextSelector(state => state.enableBilling)
+  const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
 
-  // const steps = useTestRunSteps()
+  const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')
 
   const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
   const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
@@ -79,13 +80,37 @@ const TestRunPanel = () => {
     setFiles(newList)
   }
 
-  const updateFileList = (preparedFiles: FileItem[]) => {
+  const updateFileList = useCallback((preparedFiles: FileItem[]) => {
     setFiles(preparedFiles)
-  }
+  }, [])
 
-  const updateNotionPages = (value: NotionPage[]) => {
+  const updateNotionPages = useCallback((value: NotionPage[]) => {
     setNotionPages(value)
-  }
+  }, [])
+
+  const updateCurrentFile = useCallback((file: File) => {
+    setCurrentFile(file)
+  }, [])
+
+  const hideFilePreview = useCallback(() => {
+    setCurrentFile(undefined)
+  }, [])
+
+  const updateCurrentPage = useCallback((page: NotionPage) => {
+    setCurrentNotionPage(page)
+  }, [])
+
+  const hideNotionPagePreview = useCallback(() => {
+    setCurrentNotionPage(undefined)
+  }, [])
+
+  const updateCurrentWebsite = useCallback((website: CrawlResultItem) => {
+    setCurrentWebsite(website)
+  }, [])
+
+  const hideWebsitePreview = useCallback(() => {
+    setCurrentWebsite(undefined)
+  }, [])
 
   const handleNextStep = useCallback(() => {
     setCurrentStep(preStep => preStep + 1)
@@ -95,8 +120,6 @@ const TestRunPanel = () => {
     setCurrentStep(preStep => preStep - 1)
   }, [])
 
-  // const { handleRun } = usePipelineRun()
-
   const handleProcess = useCallback((data: Record<string, any>) => {
     if (!datasource)
       return
@@ -121,13 +144,16 @@ const TestRunPanel = () => {
       datasourceInfo.jobId = websiteCrawlJobId
       datasourceInfo.result = websitePages
     }
-    // handleRun({
+    // todo: Run Pipeline
-    //   inputs: data,
+    console.log('datasource_type', datasource_type)
-    //   datasource_type,
-    //   datasource_info: datasourceInfo,
-    // })
   }, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages])
 
+  if (isFetchingPipelineInfo) {
+    return (
+      <Loading type='app' />
+    )
+  }
+
   return (
     <div
       className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
@@ -140,60 +166,65 @@ const TestRunPanel = () => {
         <div className='grow overflow-y-auto'>
           {
             currentStep === 1 && (
-              <>
+              <div className='flex flex-col gap-y-5 pt-4'>
-                <div className='flex flex-col gap-y-4 px-4 py-2'>
+                <DataSourceOptions
-                  {/* <DataSourceOptions
                   datasourceNodeId={datasource?.nodeId || ''}
                   onSelect={setDatasource}
-                /> */}
+                  pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
-                  {datasource?.type === DataSourceType.FILE && (
+                />
-                    <LocalFile
+                {datasource?.type === DataSourceType.FILE && (
-                      files={fileList}
+                  <LocalFile
-                      updateFile={updateFile}
+                    files={fileList}
-                      updateFileList={updateFileList}
+                    updateFile={updateFile}
-                      notSupportBatchUpload={notSupportBatchUpload}
+                    updateFileList={updateFileList}
-                    />
+                    onPreview={updateCurrentFile}
-                  )}
+                    notSupportBatchUpload={notSupportBatchUpload}
-                  {datasource?.type === DataSourceType.NOTION && (
+                  />
-                    <Notion
+                )}
-                      nodeId={datasource?.nodeId || ''}
+                {datasource?.type === DataSourceType.NOTION && (
-                      notionPages={notionPages}
+                  <Notion
-                      updateNotionPages={updateNotionPages}
+                    nodeId={datasource?.nodeId || ''}
-                    />
+                    notionPages={notionPages}
-                  )}
+                    updateNotionPages={updateNotionPages}
-                  {datasource?.type === DataSourceProvider.fireCrawl && (
+                    canPreview
-                    <FireCrawl
+                    onPreview={updateCurrentPage}
-                      nodeId={datasource?.nodeId || ''}
+                  />
-                      variables={datasource?.variables}
+                )}
-                      checkedCrawlResult={websitePages}
+                {datasource?.type === DataSourceProvider.fireCrawl && (
-                      onCheckedCrawlResultChange={setWebsitePages}
+                  <FireCrawl
-                      onJobIdChange={setWebsiteCrawlJobId}
+                    nodeId={datasource?.nodeId || ''}
-                    />
+                    variables={datasource?.variables}
-                  )}
+                    checkedCrawlResult={websitePages}
-                  {datasource?.type === DataSourceProvider.jinaReader && (
+                    onCheckedCrawlResultChange={setWebsitePages}
-                    <JinaReader
+                    onJobIdChange={setWebsiteCrawlJobId}
-                      nodeId={datasource?.nodeId || ''}
+                    onPreview={updateCurrentWebsite}
-                      variables={datasource?.variables}
+                  />
-                      checkedCrawlResult={websitePages}
+                )}
-                      onCheckedCrawlResultChange={setWebsitePages}
+                {datasource?.type === DataSourceProvider.jinaReader && (
-                      onJobIdChange={setWebsiteCrawlJobId}
+                  <JinaReader
-                    />
+                    nodeId={datasource?.nodeId || ''}
-                  )}
+                    variables={datasource?.variables}
-                  {datasource?.type === DataSourceProvider.waterCrawl && (
+                    checkedCrawlResult={websitePages}
-                    <WaterCrawl
+                    onCheckedCrawlResultChange={setWebsitePages}
-                      nodeId={datasource?.nodeId || ''}
+                    onJobIdChange={setWebsiteCrawlJobId}
-                      variables={datasource?.variables}
+                    onPreview={updateCurrentWebsite}
-                      checkedCrawlResult={websitePages}
+                  />
-                      onCheckedCrawlResultChange={setWebsitePages}
+                )}
-                      onJobIdChange={setWebsiteCrawlJobId}
+                {datasource?.type === DataSourceProvider.waterCrawl && (
-                    />
+                  <WaterCrawl
-                  )}
+                    nodeId={datasource?.nodeId || ''}
-                  {isShowVectorSpaceFull && (
+                    variables={datasource?.variables}
-                    <VectorSpaceFull />
+                    checkedCrawlResult={websitePages}
-                  )}
+                    onCheckedCrawlResultChange={setWebsitePages}
-                </div>
+                    onJobIdChange={setWebsiteCrawlJobId}
+                    onPreview={updateCurrentWebsite}
+                  />
+                )}
+                {isShowVectorSpaceFull && (
+                  <VectorSpaceFull />
+                )}
                 <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
-              </>
+              </div>
             )
           }
           {
@@ -209,6 +240,15 @@ const TestRunPanel = () => {
       </div>
       {/* Preview */}
       <div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'>
+        {
+          currentStep === 1 && (
+            <>
+              {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
+              {currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
+              {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
+            </>
+          )
+        }
       </div>
     </div>
   )

web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx

@@ -0,0 +1,78 @@
+'use client'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import Loading from './loading'
+import type { CustomFile as File } from '@/models/datasets'
+import { RiCloseLine } from '@remixicon/react'
+import { useFilePreview } from '@/service/use-common'
+import DocumentFileIcon from '../../../common/document-file-icon'
+import { formatNumberAbbreviated } from '@/utils/format'
+
+type FilePreviewProps = {
+  file: File
+  hidePreview: () => void
+}
+
+const FilePreview = ({
+  file,
+  hidePreview,
+}: FilePreviewProps) => {
+  const { t } = useTranslation()
+  const { data: fileData, isFetching } = useFilePreview(file.id || '')
+
+  const getFileName = (currentFile?: File) => {
+    if (!currentFile)
+      return ''
+    const arr = currentFile.name.split('.')
+    return arr.slice(0, -1).join()
+  }
+
+  const getFileSize = (size: number) => {
+    if (size / 1024 < 10)
+      return `${(size / 1024).toFixed(1)} KB`
+
+    return `${(size / 1024 / 1024).toFixed(1)} MB`
+  }
+
+  return (
+    <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
+      <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
+        <div className='flex grow flex-col gap-y-1'>
+          <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
+          <div className='title-md-semi-bold text-tex-primary'>{`${getFileName(file)}.${file.extension}`}</div>
+          <div className='system-xs-medium flex gap-x-1  text-text-tertiary'>
+            <DocumentFileIcon
+              className='size-6 shrink-0'
+              name={file.name}
+              extension={file.extension}
+            />
+            <span className='uppercase'>{file.extension}</span>
+            <span>·</span>
+            <span>{getFileSize(file.size)}</span>
+            {fileData && (
+              <>
+                <span>·</span>
+                <span>{`${formatNumberAbbreviated(fileData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
+              </>
+            )}
+          </div>
+        </div>
+        <button
+          type='button'
+          className='flex h-8 w-8 shrink-0 items-center justify-center'
+          onClick={hidePreview}
+        >
+          <RiCloseLine className='size-[18px]' />
+        </button>
+      </div>
+      <div className='px-6 py-5'>
+        {isFetching && <Loading />}
+        {!isFetching && fileData && (
+          <div className='body-md-regular overflow-hidden text-text-secondary'>{fileData.content}</div>
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default FilePreview

web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx

@@ -0,0 +1,52 @@
+import React from 'react'
+import { SkeletonContainer, SkeletonRectangle } from '@/app/components/base/skeleton'
+
+const Loading = () => {
+  return (
+    <div className='flex h-full flex-col gap-y-12 bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg px-6 py-5'>
+      <SkeletonContainer className='w-full gap-0'>
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-3/5' />
+      </SkeletonContainer>
+      <SkeletonContainer className='w-full gap-0'>
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-[70%]' />
+      </SkeletonContainer>
+      <SkeletonContainer className='w-full gap-0'>
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-[56%]' />
+      </SkeletonContainer>
+      <SkeletonContainer className='w-full gap-0'>
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-3/5' />
+      </SkeletonContainer>
+      <SkeletonContainer className='w-full gap-0'>
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-3/5' />
+      </SkeletonContainer>
+      <SkeletonContainer className='w-full gap-0'>
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-full' />
+        <SkeletonRectangle className='my-1.5 w-1/2' />
+      </SkeletonContainer>
+    </div>
+  )
+}
+
+export default React.memo(Loading)

web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx

@@ -0,0 +1,65 @@
+'use client'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import type { NotionPage } from '@/models/common'
+import { usePreviewNotionPage } from '@/service/knowledge/use-dataset'
+import { RiCloseLine } from '@remixicon/react'
+import { formatNumberAbbreviated } from '@/utils/format'
+import Loading from './loading'
+import { Notion } from '@/app/components/base/icons/src/public/common'
+
+type NotionPagePreviewProps = {
+  currentPage: NotionPage
+  hidePreview: () => void
+}
+
+const NotionPagePreview = ({
+  currentPage,
+  hidePreview,
+}: NotionPagePreviewProps) => {
+  const { t } = useTranslation()
+
+  const { data: notionPageData, isFetching } = usePreviewNotionPage({
+    workspaceID: currentPage.workspace_id,
+    pageID: currentPage.page_id,
+    pageType: currentPage.type,
+  })
+
+  return (
+    <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
+      <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
+        <div className='flex grow flex-col gap-y-1'>
+          <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
+          <div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div>
+          <div className='system-xs-medium flex gap-x-1  text-text-tertiary'>
+            <Notion className='size-3.5' />
+            <span>·</span>
+            <span>Notion Page</span>
+            <span>·</span>
+            {notionPageData && (
+              <>
+                <span>·</span>
+                <span>{`${formatNumberAbbreviated(notionPageData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
+              </>
+            )}
+          </div>
+        </div>
+        <button
+          type='button'
+          className='flex h-8 w-8 shrink-0 items-center justify-center'
+          onClick={hidePreview}
+        >
+          <RiCloseLine className='size-[18px]' />
+        </button>
+      </div>
+      <div className='px-6 py-5'>
+        {isFetching && <Loading />}
+        {!isFetching && notionPageData && (
+          <div className='body-md-regular overflow-hidden text-text-secondary'>{notionPageData.content}</div>
+        )}
+      </div>
+    </div>
+  )
+}
+
+export default NotionPagePreview

web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx

@@ -0,0 +1,48 @@
+'use client'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import type { CrawlResultItem } from '@/models/datasets'
+import { RiCloseLine, RiGlobalLine } from '@remixicon/react'
+import { formatNumberAbbreviated } from '@/utils/format'
+
+type WebsitePreviewProps = {
+  payload: CrawlResultItem
+  hidePreview: () => void
+}
+
+const WebsitePreview = ({
+  payload,
+  hidePreview,
+}: WebsitePreviewProps) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
+      <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
+        <div className='flex grow flex-col gap-y-1'>
+          <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
+          <div className='title-md-semi-bold text-tex-primary'>{payload.title}</div>
+          <div className='system-xs-medium flex gap-x-1  text-text-tertiary'>
+            <RiGlobalLine className='size-3.5' />
+            <span className='uppercase' title={payload.source_url}>{payload.source_url}</span>
+            <span>·</span>
+            <span>·</span>
+            <span>{`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
+          </div>
+        </div>
+        <button
+          type='button'
+          className='flex h-8 w-8 shrink-0 items-center justify-center'
+          onClick={hidePreview}
+        >
+          <RiCloseLine className='size-[18px]' />
+        </button>
+      </div>
+      <div className='px-6 py-5'>
+        <div className='body-md-regular overflow-hidden text-text-secondary'>{payload.markdown}</div>
+      </div>
+    </div>
+  )
+}
+
+export default WebsitePreview

web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx

@@ -23,6 +23,7 @@ type IFileUploaderProps = {
   prepareFileList: (files: FileItem[]) => void
   onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
   onFileListUpdate?: (files: FileItem[]) => void
+  onPreview?: (file: File) => void
   notSupportBatchUpload?: boolean
 }
 
@@ -31,6 +32,7 @@ const FileUploader = ({
   prepareFileList,
   onFileUpdate,
   onFileListUpdate,
+  onPreview,
   notSupportBatchUpload,
 }: IFileUploaderProps) => {
   const { t } = useTranslation()
@@ -284,6 +286,7 @@ const FileUploader = ({
             return (
               <div
                 key={`${fileItem.fileID}-${index}`}
+                onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
                 className={cn(
                   'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
                   isError && 'border-state-destructive-border bg-state-destructive-hover',

web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx

@@ -1,10 +1,11 @@
-import type { FileItem } from '@/models/datasets'
+import type { CustomFile as File, FileItem } from '@/models/datasets'
 import FileUploader from './file-uploader'
 
 type LocalFileProps = {
   files: FileItem[]
   updateFileList: (files: FileItem[]) => void
   updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
+  onPreview?: (file: File) => void
   notSupportBatchUpload: boolean
 }
 
@@ -12,6 +13,7 @@ const LocalFile = ({
   files,
   updateFileList,
   updateFile,
+  onPreview,
   notSupportBatchUpload,
 }: LocalFileProps) => {
   return (
@@ -20,6 +22,7 @@ const LocalFile = ({
       prepareFileList={updateFileList}
       onFileListUpdate={updateFileList}
       onFileUpdate={updateFile}
+      onPreview={onPreview}
       notSupportBatchUpload={notSupportBatchUpload}
     />
   )

web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx

@@ -5,20 +5,27 @@ type NotionProps = {
   nodeId: string
   notionPages: NotionPage[]
   updateNotionPages: (value: NotionPage[]) => void
+  canPreview?: boolean
+  onPreview?: (selectedPage: NotionPage) => void
+  isInPipeline?: boolean
 }
 
 const Notion = ({
   nodeId,
   notionPages,
   updateNotionPages,
+  canPreview = false,
+  onPreview,
+  isInPipeline = false,
 }: NotionProps) => {
   return (
     <NotionPageSelector
       nodeId={nodeId}
       value={notionPages.map(page => page.page_id)}
       onSelect={updateNotionPages}
-      canPreview={false}
+      canPreview={canPreview}
-      isInPipeline
+      onPreview={onPreview}
+      isInPipeline={isInPipeline}
     />
   )
 }

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx

@@ -3,23 +3,33 @@ import React, { useCallback } from 'react'
 import cn from '@/utils/classnames'
 import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
 import Checkbox from '@/app/components/base/checkbox'
+import Button from '@/app/components/base/button'
+import { useTranslation } from 'react-i18next'
 
 type CrawledResultItemProps = {
   payload: CrawlResultItemType
   isChecked: boolean
   onCheckChange: (checked: boolean) => void
+  isPreview: boolean
+  showPreview: boolean
+  onPreview: () => void
 }
 
 const CrawledResultItem = ({
   payload,
   isChecked,
   onCheckChange,
+  isPreview,
+  onPreview,
+  showPreview,
 }: CrawledResultItemProps) => {
+  const { t } = useTranslation()
+
   const handleCheckChange = useCallback(() => {
     onCheckChange(!isChecked)
   }, [isChecked, onCheckChange])
   return (
-    <div className={cn('group flex cursor-pointer gap-x-2 rounded-lg p-2 hover:bg-state-base-hover')}>
+    <div className={cn('flex cursor-pointer gap-x-2 rounded-lg p-2', isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover')}>
       <Checkbox
         className='shrink-0'
         checked={isChecked}
@@ -39,6 +49,13 @@ const CrawledResultItem = ({
           {payload.source_url}
         </div>
       </div>
+      {showPreview && <Button
+        size='small'
+        onClick={onPreview}
+        className='system-xs-medium-uppercase right-0 top-0 hidden px-1.5 group-hover:absolute group-hover:block'
+      >
+        {t('datasetCreation.stepOne.website.preview')}
+      </Button>}
     </div>
   )
 }

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx

@@ -1,5 +1,5 @@
 'use client'
-import React, { useCallback } from 'react'
+import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import cn from '@/utils/classnames'
 import type { CrawlResultItem } from '@/models/datasets'
@@ -13,6 +13,7 @@ type CrawledResultProps = {
   list: CrawlResultItem[]
   checkedList: CrawlResultItem[]
   onSelectedChange: (selected: CrawlResultItem[]) => void
+  onPreview?: (payload: CrawlResultItem) => void
   usedTime: number
 }
 
@@ -22,8 +23,10 @@ const CrawledResult = ({
   checkedList,
   onSelectedChange,
   usedTime,
+  onPreview,
 }: CrawledResultProps) => {
   const { t } = useTranslation()
+  const [previewIndex, setPreviewIndex] = useState<number>(-1)
 
   const isCheckAll = checkedList.length === list.length
 
@@ -45,6 +48,12 @@ const CrawledResult = ({
     }
   }, [checkedList, onSelectedChange])
 
+  const handlePreview = useCallback((index: number) => {
+    if (!onPreview) return
+    setPreviewIndex(index)
+    onPreview(list[index])
+  }, [list, onPreview])
+
   return (
     <div className={cn('flex flex-col gap-y-2', className)}>
       <div className='system-sm-medium pt-2 text-text-primary'>
@@ -61,12 +70,15 @@ const CrawledResult = ({
           />
         </div>
         <div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
-          {list.map(item => (
+          {list.map((item, index) => (
             <CrawledResultItem
               key={item.source_url}
               payload={item}
               isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
               onCheckChange={handleItemCheckChange(item)}
+              isPreview={index === previewIndex}
+              onPreview={handlePreview.bind(null, index)}
+              showPreview={!!onPreview}
             />
           ))}
         </div>

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx

@@ -22,6 +22,7 @@ type CrawlerProps = {
   datasourceProvider: DataSourceProvider
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
+  onPreview?: (payload: CrawlResultItem) => void
 }
 
 enum Step {
@@ -37,6 +38,7 @@ const Crawler = ({
   datasourceProvider,
   onCheckedCrawlResultChange,
   onJobIdChange,
+  onPreview,
 }: CrawlerProps) => {
   const { t } = useTranslation()
   const [step, setStep] = useState<Step>(Step.init)
@@ -123,6 +125,7 @@ const Crawler = ({
               checkedList={checkedCrawlResult}
               onSelectedChange={onCheckedCrawlResultChange}
               usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
+              onPreview={onPreview}
             />
           )}
         </div>

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx

@@ -11,6 +11,7 @@ type FireCrawlProps = {
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
+  onPreview?: (payload: CrawlResultItem) => void
 }
 
 const FireCrawl = ({
@@ -19,6 +20,7 @@ const FireCrawl = ({
   checkedCrawlResult,
   onCheckedCrawlResultChange,
   onJobIdChange,
+  onPreview,
 }: FireCrawlProps) => {
   return (
     <Crawler
@@ -28,6 +30,7 @@ const FireCrawl = ({
       datasourceProvider={DataSourceProvider.fireCrawl}
       onCheckedCrawlResultChange={onCheckedCrawlResultChange}
       onJobIdChange={onJobIdChange}
+      onPreview={onPreview}
     />
   )
 }

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx

@@ -11,6 +11,7 @@ type JinaReaderProps = {
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
+  onPreview?: (payload: CrawlResultItem) => void
 }
 
 const JinaReader = ({
@@ -19,6 +20,7 @@ const JinaReader = ({
   checkedCrawlResult,
   onCheckedCrawlResultChange,
   onJobIdChange,
+  onPreview,
 }: JinaReaderProps) => {
   return (
     <Crawler
@@ -28,6 +30,7 @@ const JinaReader = ({
       datasourceProvider={DataSourceProvider.jinaReader}
       onCheckedCrawlResultChange={onCheckedCrawlResultChange}
       onJobIdChange={onJobIdChange}
+      onPreview={onPreview}
     />
   )
 }

web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx

@@ -11,6 +11,7 @@ type WaterCrawlProps = {
   checkedCrawlResult: CrawlResultItem[]
   onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
   onJobIdChange: (jobId: string) => void
+  onPreview?: (payload: CrawlResultItem) => void
 }
 
 const WaterCrawl = ({
@@ -19,6 +20,7 @@ const WaterCrawl = ({
   checkedCrawlResult,
   onCheckedCrawlResultChange,
   onJobIdChange,
+  onPreview,
 }: WaterCrawlProps) => {
   return (
     <Crawler
@@ -28,6 +30,7 @@ const WaterCrawl = ({
       datasourceProvider={DataSourceProvider.jinaReader}
       onCheckedCrawlResultChange={onCheckedCrawlResultChange}
       onJobIdChange={onJobIdChange}
+      onPreview={onPreview}
     />
   )
 }

web/i18n/en-US/dataset-pipeline.ts

@@ -78,6 +78,10 @@ const translation = {
       processingDocuments: 'Processing Documents',
     },
     backToDataSource: 'Data Source',
+    stepOne: {
+      preview: 'Preview',
+    },
+    characters: 'characters',
   },
 }
 

web/i18n/zh-Hans/dataset-pipeline.ts

@@ -78,6 +78,10 @@ const translation = {
       processingDocuments: '正在处理文档',
     },
     backToDataSource: '数据源',
+    stepOne: {
+      preview: '预览',
+    },
+    characters: '字符',
   },
 }
 

web/service/knowledge/use-dataset.ts

@@ -44,3 +44,22 @@ export const useDatasetRelatedApps = (datasetId: string) => {
     queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`),
   })
 }
+
+type NotionPagePreviewRequest = {
+  workspaceID: string
+  pageID: string
+  pageType: string
+}
+
+type NotionPagePreviewResponse = {
+  content: string
+}
+
+export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => {
+  const { workspaceID, pageID, pageType } = params
+  return useQuery({
+    queryKey: [NAME_SPACE, 'preview-notion-page'],
+    queryFn: () => get<NotionPagePreviewResponse>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`),
+    enabled: !!workspaceID && !!pageID && !!pageType,
+  })
+}

web/service/use-common.ts

@@ -60,3 +60,15 @@ export const useMembers = () => {
     }),
   })
 }
+
+type FilePreviewResponse = {
+  content: string
+}
+
+export const useFilePreview = (fileID: string) => {
+  return useQuery<FilePreviewResponse>({
+    queryKey: [NAME_SPACE, 'file-preview', fileID],
+    queryFn: () => get<FilePreviewResponse>(`/files/${fileID}/preview`),
+    enabled: !!fileID,
+  })
+}

web/service/use-pipeline.ts

@@ -164,5 +164,6 @@ export const usePublishedPipelineInfo = (pipelineId: string) => {
     queryFn: () => {
       return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`)
     },
+    enabled: !!pipelineId,
   })
 }

web/utils/format.ts

@@ -56,3 +56,35 @@ export const downloadFile = ({ data, fileName }: { data: Blob; fileName: string
   a.remove()
   window.URL.revokeObjectURL(url)
 }
+
+/**
+ * Formats a number into a readable string using "k", "M", or "B" suffix.
+ * @example
+ * 950     => "950"
+ * 1200    => "1.2k"
+ * 1500000 => "1.5M"
+ * 2000000000 => "2B"
+ *
+ * @param {number} num - The number to format
+ * @returns {string} - The formatted number string
+ */
+export const formatNumberAbbreviated = (num: number) => {
+  // If less than 1000, return as-is
+  if (num < 1000) return num.toString()
+
+  // Define thresholds and suffixes
+  const units = [
+    { value: 1e9, symbol: 'B' },
+    { value: 1e6, symbol: 'M' },
+    { value: 1e3, symbol: 'k' },
+  ]
+
+  for (let i = 0; i < units.length; i++) {
+    if (num >= units[i].value) {
+      const formatted = (num / units[i].value).toFixed(1)
+      return formatted.endsWith('.0')
+        ? `${Number.parseInt(formatted)}${units[i].symbol}`
+        : `${formatted}${units[i].symbol}`
+    }
+  }
+}