opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: refactor preview components

This commit is contained in:
twwu 2025-05-22 14:49:40 +08:00
parent 69a60101fe
commit faf6b9ea03
25 changed files with 727 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
web/app/components/base/form/form-scenarios/demo/index.tsx
View File

@ -24,7 +24,7 @@ const DemoForm = () => {
},
})
const name = useStore(form.store, state => state.values.name)
const name = useStore(form.store, state => state.values.name)
return (
<form
@ -59,7 +59,7 @@ const name = useStore(form.store, state => state.values.name)
)
}
<form.AppForm>
<form.SubmitButton>Submit</form.SubmitButton>
<form.Actions />
</form.AppForm>
</form>
)

62
web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx Normal file
View File

@ -0,0 +1,62 @@
import { useCallback, useEffect } from 'react'
import { useDatasourceOptions } from '../hooks'
import OptionCard from './option-card'
import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge'
import { Notion } from '@/app/components/base/icons/src/public/common'
import { Jina } from '@/app/components/base/icons/src/public/llm'
import { DataSourceType } from '@/models/datasets'
import { DataSourceProvider } from '@/models/common'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
type DataSourceOptionsProps = {
pipelineNodes: Node<DataSourceNodeType>[]
datasourceNodeId: string
onSelect: (option: Datasource) => void
}
const DATA_SOURCE_ICONS = {
[DataSourceType.FILE]: File as React.FC<React.SVGProps<SVGSVGElement>>,
[DataSourceType.NOTION]: Notion as React.FC<React.SVGProps<SVGSVGElement>>,
[DataSourceProvider.fireCrawl]: '🔥',
[DataSourceProvider.jinaReader]: Jina as React.FC<React.SVGProps<SVGSVGElement>>,
[DataSourceProvider.waterCrawl]: Watercrawl as React.FC<React.SVGProps<SVGSVGElement>>,
}
const DataSourceOptions = ({
pipelineNodes,
datasourceNodeId,
onSelect,
}: DataSourceOptionsProps) => {
const { datasources, options } = useDatasourceOptions(pipelineNodes)
const handelSelect = useCallback((value: string) => {
const selectedOption = datasources.find(option => option.nodeId === value)
if (!selectedOption)
return
onSelect(selectedOption)
}, [datasources, onSelect])
useEffect(() => {
if (options.length > 0)
handelSelect(options[0].value)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
return (
<div className='grid w-full grid-cols-4 gap-1'>
{options.map(option => (
<OptionCard
key={option.value}
label={option.label}
selected={datasourceNodeId === option.value}
Icon={DATA_SOURCE_ICONS[option.type as keyof typeof DATA_SOURCE_ICONS]}
onClick={handelSelect.bind(null, option.value)}
/>
))}
</div>
)
}
export default DataSourceOptions

41
web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx Normal file
View File

@ -0,0 +1,41 @@
import React from 'react'
import cn from '@/utils/classnames'
type OptionCardProps = {
label: string
Icon: React.FC<React.SVGProps<SVGSVGElement>> | string
selected: boolean
onClick?: () => void
}
const OptionCard = ({
label,
Icon,
selected,
onClick,
}: OptionCardProps) => {
return (
<div
className={cn(
'flex items-center gap-2 rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg p-3 shadow-shadow-shadow-3',
selected
? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg shadow-xs ring-[0.5px] ring-inset ring-components-option-card-option-selected-border'
: 'hover:bg-components-option-card-bg-hover hover:border-components-option-card-option-border-hover hover:shadow-xs',
)}
onClick={onClick}
>
<div className='flex size-8 items-center justify-center rounded-lg border-[0.5px] border-components-panel-border bg-background-default-dodge p-1.5'>
{
typeof Icon === 'string'
? <div className='text-[18px] leading-[18px]'>{Icon}</div>
: <Icon className='size-5' />
}
</div>
<div className={cn('system-sm-medium text-text-secondary', selected && 'text-primary')}>
{label}
</div>
</div>
)
}
export default React.memo(OptionCard)

44
web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx Normal file
View File

@ -0,0 +1,44 @@
import React from 'react'
import Button from '@/app/components/base/button'
import { useTranslation } from 'react-i18next'
import { useParams } from 'next/navigation'
import { RiArrowRightLine } from '@remixicon/react'
type ActionsProps = {
disabled?: boolean
handleNextStep: () => void
}
const Actions = ({
disabled,
handleNextStep,
}: ActionsProps) => {
const { t } = useTranslation()
const { datasetId } = useParams()
return (
<div className='flex justify-end gap-x-2'>
<a
href={`/datasets/${datasetId}/documents`}
>
<Button
variant='ghost'
className='px-3 py-2'
>
{t('common.operation.cancel')}
</Button>
</a>
<Button
disabled={disabled}
variant='primary'
onClick={handleNextStep}
className='gap-x-0.5'
>
<span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
<RiArrowRightLine className='size-4' />
</Button>
</div>
)
}
export default React.memo(Actions)

82
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -1,5 +1,11 @@
import { useTranslation } from 'react-i18next'
import { AddDocumentsStep } from './types'
import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import { useMemo } from 'react'
import { BlockEnum, type Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { DataSourceType } from '@/models/datasets'
import { DataSourceProvider } from '@/models/common'
export const useAddDocumentsSteps = () => {
const { t } = useTranslation()
@ -19,3 +25,79 @@ export const useAddDocumentsSteps = () => {
]
return steps
}
export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
const { t } = useTranslation()
const datasources: Datasource[] = useMemo(() => {
const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
return datasourceNodes.map((node) => {
let type: DataSourceType | DataSourceProvider = DataSourceType.FILE
switch (node.data.tool_name) {
case 'file_upload':
type = DataSourceType.FILE
break
case 'search_notion':
type = DataSourceType.NOTION
break
case 'firecrawl':
type = DataSourceProvider.fireCrawl
break
case 'jina_reader':
type = DataSourceProvider.jinaReader
break
case 'water_crawl':
type = DataSourceProvider.waterCrawl
break
}
return {
nodeId: node.id,
type,
variables: node.data.variables,
}
})
}, [pipelineNodes])
const options = useMemo(() => {
const options: DataSourceOption[] = []
datasources.forEach((source) => {
if (source.type === DataSourceType.FILE) {
options.push({
label: t('datasetPipeline.testRun.dataSource.localFiles'),
value: source.nodeId,
type: DataSourceType.FILE,
})
}
if (source.type === DataSourceType.NOTION) {
options.push({
label: 'Notion',
value: source.nodeId,
type: DataSourceType.NOTION,
})
}
if (source.type === DataSourceProvider.fireCrawl) {
options.push({
label: 'Firecrawl',
value: source.nodeId,
type: DataSourceProvider.fireCrawl,
})
}
if (source.type === DataSourceProvider.jinaReader) {
options.push({
label: 'Jina Reader',
value: source.nodeId,
type: DataSourceProvider.jinaReader,
})
}
if (source.type === DataSourceProvider.waterCrawl) {
options.push({
label: 'Water Crawl',
value: source.nodeId,
type: DataSourceProvider.waterCrawl,
})
}
})
return options
}, [datasources, t])
return { datasources, options }
}

202
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -1,33 +1,30 @@
'use client'
import { useCallback, useMemo, useState } from 'react'
// import StepIndicator from './step-indicator'
// import { useTestRunSteps } from './hooks'
// import DataSourceOptions from './data-source-options'
import type { CrawlResultItem, FileItem } from '@/models/datasets'
import DataSourceOptions from './data-source-options'
import type { CrawlResultItem, CustomFile as File, FileItem } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets'
// import LocalFile from './data-source/local-file'
import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
import produce from 'immer'
import { useProviderContextSelector } from '@/context/provider-context'
import { DataSourceProvider, type NotionPage } from '@/models/common'
// import Notion from './data-source/notion'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
// import Firecrawl from './data-source/website/firecrawl'
// import JinaReader from './data-source/website/jina-reader'
// import WaterCrawl from './data-source/website/water-crawl'
// import Actions from './data-source/actions'
// import DocumentProcessing from './document-processing'
import { useTranslation } from 'react-i18next'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl'
import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader'
import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl'
import Actions from '@/app/components/rag-pipeline/components/panel/test-run/data-source/actions'
import Actions from './data-source/actions'
import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing'
import { useTranslation } from 'react-i18next'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import LeftHeader from './left-header'
// import { usePipelineRun } from '../../../hooks'
// import type { Datasource } from './types'
import { usePublishedPipelineInfo } from '@/service/use-pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import Loading from '@/app/components/base/loading'
import type { Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import FilePreview from './preview/file-preview'
import NotionPagePreview from './preview/notion-page-preview'
import WebsitePreview from './preview/web-preview'
const TestRunPanel = () => {
const { t } = useTranslation()
@ -37,11 +34,15 @@ const TestRunPanel = () => {
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
const [currentFile, setCurrentFile] = useState<File | undefined>()
const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()
const plan = useProviderContextSelector(state => state.plan)
const enableBilling = useProviderContextSelector(state => state.enableBilling)
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
// const steps = useTestRunSteps()
const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')
const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
@ -79,13 +80,37 @@ const TestRunPanel = () => {
setFiles(newList)
}
const updateFileList = (preparedFiles: FileItem[]) => {
const updateFileList = useCallback((preparedFiles: FileItem[]) => {
setFiles(preparedFiles)
}
}, [])
const updateNotionPages = (value: NotionPage[]) => {
const updateNotionPages = useCallback((value: NotionPage[]) => {
setNotionPages(value)
}
}, [])
const updateCurrentFile = useCallback((file: File) => {
setCurrentFile(file)
}, [])
const hideFilePreview = useCallback(() => {
setCurrentFile(undefined)
}, [])
const updateCurrentPage = useCallback((page: NotionPage) => {
setCurrentNotionPage(page)
}, [])
const hideNotionPagePreview = useCallback(() => {
setCurrentNotionPage(undefined)
}, [])
const updateCurrentWebsite = useCallback((website: CrawlResultItem) => {
setCurrentWebsite(website)
}, [])
const hideWebsitePreview = useCallback(() => {
setCurrentWebsite(undefined)
}, [])
const handleNextStep = useCallback(() => {
setCurrentStep(preStep => preStep + 1)
@ -95,8 +120,6 @@ const TestRunPanel = () => {
setCurrentStep(preStep => preStep - 1)
}, [])
// const { handleRun } = usePipelineRun()
const handleProcess = useCallback((data: Record<string, any>) => {
if (!datasource)
return
@ -121,13 +144,16 @@ const TestRunPanel = () => {
datasourceInfo.jobId = websiteCrawlJobId
datasourceInfo.result = websitePages
}
// handleRun({
// inputs: data,
// datasource_type,
// datasource_info: datasourceInfo,
// })
// todo: Run Pipeline
console.log('datasource_type', datasource_type)
}, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages])
if (isFetchingPipelineInfo) {
return (
<Loading type='app' />
)
}
return (
<div
className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
@ -140,60 +166,65 @@ const TestRunPanel = () => {
<div className='grow overflow-y-auto'>
{
currentStep === 1 && (
<>
<div className='flex flex-col gap-y-4 px-4 py-2'>
{/* <DataSourceOptions
<div className='flex flex-col gap-y-5 pt-4'>
<DataSourceOptions
datasourceNodeId={datasource?.nodeId || ''}
onSelect={setDatasource}
/> */}
{datasource?.type === DataSourceType.FILE && (
<LocalFile
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
notSupportBatchUpload={notSupportBatchUpload}
/>
)}
{datasource?.type === DataSourceType.NOTION && (
<Notion
nodeId={datasource?.nodeId || ''}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
/>
)}
{datasource?.type === DataSourceProvider.fireCrawl && (
<FireCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
/>
)}
{datasource?.type === DataSourceProvider.jinaReader && (
<JinaReader
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
/>
)}
{datasource?.type === DataSourceProvider.waterCrawl && (
<WaterCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
/>
)}
{isShowVectorSpaceFull && (
<VectorSpaceFull />
)}
</div>
pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
/>
{datasource?.type === DataSourceType.FILE && (
<LocalFile
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
onPreview={updateCurrentFile}
notSupportBatchUpload={notSupportBatchUpload}
/>
)}
{datasource?.type === DataSourceType.NOTION && (
<Notion
nodeId={datasource?.nodeId || ''}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
canPreview
onPreview={updateCurrentPage}
/>
)}
{datasource?.type === DataSourceProvider.fireCrawl && (
<FireCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>
)}
{datasource?.type === DataSourceProvider.jinaReader && (
<JinaReader
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>
)}
{datasource?.type === DataSourceProvider.waterCrawl && (
<WaterCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>
)}
{isShowVectorSpaceFull && (
<VectorSpaceFull />
)}
<Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
</>
</div>
)
}
{
@ -209,6 +240,15 @@ const TestRunPanel = () => {
</div>
{/* Preview */}
<div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'>
{
currentStep === 1 && (
<>
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
</>
)
}
</div>
</div>
)

78
web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx Normal file
View File

@ -0,0 +1,78 @@
'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import Loading from './loading'
import type { CustomFile as File } from '@/models/datasets'
import { RiCloseLine } from '@remixicon/react'
import { useFilePreview } from '@/service/use-common'
import DocumentFileIcon from '../../../common/document-file-icon'
import { formatNumberAbbreviated } from '@/utils/format'
type FilePreviewProps = {
file: File
hidePreview: () => void
}
const FilePreview = ({
file,
hidePreview,
}: FilePreviewProps) => {
const { t } = useTranslation()
const { data: fileData, isFetching } = useFilePreview(file.id || '')
const getFileName = (currentFile?: File) => {
if (!currentFile)
return ''
const arr = currentFile.name.split('.')
return arr.slice(0, -1).join()
}
const getFileSize = (size: number) => {
if (size / 1024 < 10)
return `${(size / 1024).toFixed(1)} KB`
return `${(size / 1024 / 1024).toFixed(1)} MB`
}
return (
<div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
<div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
<div className='flex grow flex-col gap-y-1'>
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
<div className='title-md-semi-bold text-tex-primary'>{`${getFileName(file)}.${file.extension}`}</div>
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
<DocumentFileIcon
className='size-6 shrink-0'
name={file.name}
extension={file.extension}
/>
<span className='uppercase'>{file.extension}</span>
<span>·</span>
<span>{getFileSize(file.size)}</span>
{fileData && (
<>
<span>·</span>
<span>{`${formatNumberAbbreviated(fileData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
</>
)}
</div>
</div>
<button
type='button'
className='flex h-8 w-8 shrink-0 items-center justify-center'
onClick={hidePreview}
>
<RiCloseLine className='size-[18px]' />
</button>
</div>
<div className='px-6 py-5'>
{isFetching && <Loading />}
{!isFetching && fileData && (
<div className='body-md-regular overflow-hidden text-text-secondary'>{fileData.content}</div>
)}
</div>
</div>
)
}
export default FilePreview

52
web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx Normal file
View File

@ -0,0 +1,52 @@
import React from 'react'
import { SkeletonContainer, SkeletonRectangle } from '@/app/components/base/skeleton'
const Loading = () => {
return (
<div className='flex h-full flex-col gap-y-12 bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg px-6 py-5'>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-3/5' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-[70%]' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-[56%]' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-3/5' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-3/5' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-1/2' />
</SkeletonContainer>
</div>
)
}
export default React.memo(Loading)

65
web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx Normal file
View File

@ -0,0 +1,65 @@
'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import type { NotionPage } from '@/models/common'
import { usePreviewNotionPage } from '@/service/knowledge/use-dataset'
import { RiCloseLine } from '@remixicon/react'
import { formatNumberAbbreviated } from '@/utils/format'
import Loading from './loading'
import { Notion } from '@/app/components/base/icons/src/public/common'
type NotionPagePreviewProps = {
currentPage: NotionPage
hidePreview: () => void
}
const NotionPagePreview = ({
currentPage,
hidePreview,
}: NotionPagePreviewProps) => {
const { t } = useTranslation()
const { data: notionPageData, isFetching } = usePreviewNotionPage({
workspaceID: currentPage.workspace_id,
pageID: currentPage.page_id,
pageType: currentPage.type,
})
return (
<div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
<div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
<div className='flex grow flex-col gap-y-1'>
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
<div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div>
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
<Notion className='size-3.5' />
<span>·</span>
<span>Notion Page</span>
<span>·</span>
{notionPageData && (
<>
<span>·</span>
<span>{`${formatNumberAbbreviated(notionPageData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
</>
)}
</div>
</div>
<button
type='button'
className='flex h-8 w-8 shrink-0 items-center justify-center'
onClick={hidePreview}
>
<RiCloseLine className='size-[18px]' />
</button>
</div>
<div className='px-6 py-5'>
{isFetching && <Loading />}
{!isFetching && notionPageData && (
<div className='body-md-regular overflow-hidden text-text-secondary'>{notionPageData.content}</div>
)}
</div>
</div>
)
}
export default NotionPagePreview

48
web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx Normal file
View File

@ -0,0 +1,48 @@
'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import type { CrawlResultItem } from '@/models/datasets'
import { RiCloseLine, RiGlobalLine } from '@remixicon/react'
import { formatNumberAbbreviated } from '@/utils/format'
type WebsitePreviewProps = {
payload: CrawlResultItem
hidePreview: () => void
}
const WebsitePreview = ({
payload,
hidePreview,
}: WebsitePreviewProps) => {
const { t } = useTranslation()
return (
<div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
<div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
<div className='flex grow flex-col gap-y-1'>
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
<div className='title-md-semi-bold text-tex-primary'>{payload.title}</div>
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
<RiGlobalLine className='size-3.5' />
<span className='uppercase' title={payload.source_url}>{payload.source_url}</span>
<span>·</span>
<span>·</span>
<span>{`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
</div>
</div>
<button
type='button'
className='flex h-8 w-8 shrink-0 items-center justify-center'
onClick={hidePreview}
>
<RiCloseLine className='size-[18px]' />
</button>
</div>
<div className='px-6 py-5'>
<div className='body-md-regular overflow-hidden text-text-secondary'>{payload.markdown}</div>
</div>
</div>
)
}
export default WebsitePreview

3
web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx
View File

@ -23,6 +23,7 @@ type IFileUploaderProps = {
prepareFileList: (files: FileItem[]) => void
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
onFileListUpdate?: (files: FileItem[]) => void
onPreview?: (file: File) => void
notSupportBatchUpload?: boolean
}
@ -31,6 +32,7 @@ const FileUploader = ({
prepareFileList,
onFileUpdate,
onFileListUpdate,
onPreview,
notSupportBatchUpload,
}: IFileUploaderProps) => {
const { t } = useTranslation()
@ -284,6 +286,7 @@ const FileUploader = ({
return (
<div
key={`${fileItem.fileID}-${index}`}
onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
className={cn(
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
isError && 'border-state-destructive-border bg-state-destructive-hover',

5
web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx
View File

@ -1,10 +1,11 @@
import type { FileItem } from '@/models/datasets'
import type { CustomFile as File, FileItem } from '@/models/datasets'
import FileUploader from './file-uploader'
type LocalFileProps = {
files: FileItem[]
updateFileList: (files: FileItem[]) => void
updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
onPreview?: (file: File) => void
notSupportBatchUpload: boolean
}
@ -12,6 +13,7 @@ const LocalFile = ({
files,
updateFileList,
updateFile,
onPreview,
notSupportBatchUpload,
}: LocalFileProps) => {
return (
@ -20,6 +22,7 @@ const LocalFile = ({
prepareFileList={updateFileList}
onFileListUpdate={updateFileList}
onFileUpdate={updateFile}
onPreview={onPreview}
notSupportBatchUpload={notSupportBatchUpload}
/>
)

11
web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx
View File

@ -5,20 +5,27 @@ type NotionProps = {
nodeId: string
notionPages: NotionPage[]
updateNotionPages: (value: NotionPage[]) => void
canPreview?: boolean
onPreview?: (selectedPage: NotionPage) => void
isInPipeline?: boolean
}
const Notion = ({
nodeId,
notionPages,
updateNotionPages,
canPreview = false,
onPreview,
isInPipeline = false,
}: NotionProps) => {
return (
<NotionPageSelector
nodeId={nodeId}
value={notionPages.map(page => page.page_id)}
onSelect={updateNotionPages}
canPreview={false}
isInPipeline
canPreview={canPreview}
onPreview={onPreview}
isInPipeline={isInPipeline}
/>
)
}

19
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx
View File

@ -3,23 +3,33 @@ import React, { useCallback } from 'react'
import cn from '@/utils/classnames'
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
import Checkbox from '@/app/components/base/checkbox'
import Button from '@/app/components/base/button'
import { useTranslation } from 'react-i18next'
type CrawledResultItemProps = {
payload: CrawlResultItemType
isChecked: boolean
onCheckChange: (checked: boolean) => void
isPreview: boolean
showPreview: boolean
onPreview: () => void
}
const CrawledResultItem = ({
payload,
isChecked,
onCheckChange,
isPreview,
onPreview,
showPreview,
}: CrawledResultItemProps) => {
const { t } = useTranslation()
const handleCheckChange = useCallback(() => {
onCheckChange(!isChecked)
}, [isChecked, onCheckChange])
return (
<div className={cn('group flex cursor-pointer gap-x-2 rounded-lg p-2 hover:bg-state-base-hover')}>
<div className={cn('flex cursor-pointer gap-x-2 rounded-lg p-2', isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover')}>
<Checkbox
className='shrink-0'
checked={isChecked}
@ -39,6 +49,13 @@ const CrawledResultItem = ({
{payload.source_url}
</div>
</div>
{showPreview && <Button
size='small'
onClick={onPreview}
className='system-xs-medium-uppercase right-0 top-0 hidden px-1.5 group-hover:absolute group-hover:block'
>
{t('datasetCreation.stepOne.website.preview')}
</Button>}
</div>
)
}

16
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx
View File

@ -1,5 +1,5 @@
'use client'
import React, { useCallback } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames'
import type { CrawlResultItem } from '@/models/datasets'
@ -13,6 +13,7 @@ type CrawledResultProps = {
list: CrawlResultItem[]
checkedList: CrawlResultItem[]
onSelectedChange: (selected: CrawlResultItem[]) => void
onPreview?: (payload: CrawlResultItem) => void
usedTime: number
}
@ -22,8 +23,10 @@ const CrawledResult = ({
checkedList,
onSelectedChange,
usedTime,
onPreview,
}: CrawledResultProps) => {
const { t } = useTranslation()
const [previewIndex, setPreviewIndex] = useState<number>(-1)
const isCheckAll = checkedList.length === list.length
@ -45,6 +48,12 @@ const CrawledResult = ({
}
}, [checkedList, onSelectedChange])
const handlePreview = useCallback((index: number) => {
if (!onPreview) return
setPreviewIndex(index)
onPreview(list[index])
}, [list, onPreview])
return (
<div className={cn('flex flex-col gap-y-2', className)}>
<div className='system-sm-medium pt-2 text-text-primary'>
@ -61,12 +70,15 @@ const CrawledResult = ({
/>
</div>
<div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
{list.map(item => (
{list.map((item, index) => (
<CrawledResultItem
key={item.source_url}
payload={item}
isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
onCheckChange={handleItemCheckChange(item)}
isPreview={index === previewIndex}
onPreview={handlePreview.bind(null, index)}
showPreview={!!onPreview}
/>
))}
</div>

3
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx
View File

@ -22,6 +22,7 @@ type CrawlerProps = {
datasourceProvider: DataSourceProvider
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
}
enum Step {
@ -37,6 +38,7 @@ const Crawler = ({
datasourceProvider,
onCheckedCrawlResultChange,
onJobIdChange,
onPreview,
}: CrawlerProps) => {
const { t } = useTranslation()
const [step, setStep] = useState<Step>(Step.init)
@ -123,6 +125,7 @@ const Crawler = ({
checkedList={checkedCrawlResult}
onSelectedChange={onCheckedCrawlResultChange}
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
onPreview={onPreview}
/>
)}
</div>

3
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx
View File

@ -11,6 +11,7 @@ type FireCrawlProps = {
checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
}
const FireCrawl = ({
@ -19,6 +20,7 @@ const FireCrawl = ({
checkedCrawlResult,
onCheckedCrawlResultChange,
onJobIdChange,
onPreview,
}: FireCrawlProps) => {
return (
<Crawler
@ -28,6 +30,7 @@ const FireCrawl = ({
datasourceProvider={DataSourceProvider.fireCrawl}
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange}
onPreview={onPreview}
/>
)
}

3
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx
View File

@ -11,6 +11,7 @@ type JinaReaderProps = {
checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
}
const JinaReader = ({
@ -19,6 +20,7 @@ const JinaReader = ({
checkedCrawlResult,
onCheckedCrawlResultChange,
onJobIdChange,
onPreview,
}: JinaReaderProps) => {
return (
<Crawler
@ -28,6 +30,7 @@ const JinaReader = ({
datasourceProvider={DataSourceProvider.jinaReader}
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange}
onPreview={onPreview}
/>
)
}

3
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx
View File

@ -11,6 +11,7 @@ type WaterCrawlProps = {
checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
}
const WaterCrawl = ({
@ -19,6 +20,7 @@ const WaterCrawl = ({
checkedCrawlResult,
onCheckedCrawlResultChange,
onJobIdChange,
onPreview,
}: WaterCrawlProps) => {
return (
<Crawler
@ -28,6 +30,7 @@ const WaterCrawl = ({
datasourceProvider={DataSourceProvider.jinaReader}
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange}
onPreview={onPreview}
/>
)
}

4
web/i18n/en-US/dataset-pipeline.ts
View File

@ -78,6 +78,10 @@ const translation = {
processingDocuments: 'Processing Documents',
},
backToDataSource: 'Data Source',
stepOne: {
preview: 'Preview',
},
characters: 'characters',
},
}

4
web/i18n/zh-Hans/dataset-pipeline.ts
View File

@ -78,6 +78,10 @@ const translation = {
processingDocuments: '正在处理文档',
},
backToDataSource: '数据源',
stepOne: {
preview: '预览',
},
characters: '字符',
},
}

19
web/service/knowledge/use-dataset.ts
View File

@ -44,3 +44,22 @@ export const useDatasetRelatedApps = (datasetId: string) => {
queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`),
})
}
type NotionPagePreviewRequest = {
workspaceID: string
pageID: string
pageType: string
}
type NotionPagePreviewResponse = {
content: string
}
export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => {
const { workspaceID, pageID, pageType } = params
return useQuery({
queryKey: [NAME_SPACE, 'preview-notion-page'],
queryFn: () => get<NotionPagePreviewResponse>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`),
enabled: !!workspaceID && !!pageID && !!pageType,
})
}

12
web/service/use-common.ts
View File

@ -60,3 +60,15 @@ export const useMembers = () => {
}),
})
}
type FilePreviewResponse = {
content: string
}
export const useFilePreview = (fileID: string) => {
return useQuery<FilePreviewResponse>({
queryKey: [NAME_SPACE, 'file-preview', fileID],
queryFn: () => get<FilePreviewResponse>(`/files/${fileID}/preview`),
enabled: !!fileID,
})
}

1
web/service/use-pipeline.ts
View File

@ -164,5 +164,6 @@ export const usePublishedPipelineInfo = (pipelineId: string) => {
queryFn: () => {
return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`)
},
enabled: !!pipelineId,
})
}

32
web/utils/format.ts
View File

@ -56,3 +56,35 @@ export const downloadFile = ({ data, fileName }: { data: Blob; fileName: string
a.remove()
window.URL.revokeObjectURL(url)
}
/**
* Formats a number into a readable string using "k", "M", or "B" suffix.
* @example
* 950 => "950"
* 1200 => "1.2k"
* 1500000 => "1.5M"
* 2000000000 => "2B"
*
* @param {number} num - The number to format
* @returns {string} - The formatted number string
*/
export const formatNumberAbbreviated = (num: number) => {
// If less than 1000, return as-is
if (num < 1000) return num.toString()
// Define thresholds and suffixes
const units = [
{ value: 1e9, symbol: 'B' },
{ value: 1e6, symbol: 'M' },
{ value: 1e3, symbol: 'k' },
]
for (let i = 0; i < units.length; i++) {
if (num >= units[i].value) {
const formatted = (num / units[i].value).toFixed(1)
return formatted.endsWith('.0')
? `${Number.parseInt(formatted)}${units[i].symbol}`
: `${formatted}${units[i].symbol}`
}
}
}