diff --git a/web/app/components/base/form/form-scenarios/demo/index.tsx b/web/app/components/base/form/form-scenarios/demo/index.tsx index f08edee41e..02f0e9dcea 100644 --- a/web/app/components/base/form/form-scenarios/demo/index.tsx +++ b/web/app/components/base/form/form-scenarios/demo/index.tsx @@ -24,7 +24,7 @@ const DemoForm = () => { }, }) -const name = useStore(form.store, state => state.values.name) + const name = useStore(form.store, state => state.values.name) return (
state.values.name) ) } - Submit +
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx new file mode 100644 index 0000000000..1421cefad8 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx @@ -0,0 +1,62 @@ +import { useCallback, useEffect } from 'react' +import { useDatasourceOptions } from '../hooks' +import OptionCard from './option-card' +import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge' +import { Notion } from '@/app/components/base/icons/src/public/common' +import { Jina } from '@/app/components/base/icons/src/public/llm' +import { DataSourceType } from '@/models/datasets' +import { DataSourceProvider } from '@/models/common' +import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import type { Node } from '@/app/components/workflow/types' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' + +type DataSourceOptionsProps = { + pipelineNodes: Node[] + datasourceNodeId: string + onSelect: (option: Datasource) => void +} + +const DATA_SOURCE_ICONS = { + [DataSourceType.FILE]: File as React.FC>, + [DataSourceType.NOTION]: Notion as React.FC>, + [DataSourceProvider.fireCrawl]: '🔥', + [DataSourceProvider.jinaReader]: Jina as React.FC>, + [DataSourceProvider.waterCrawl]: Watercrawl as React.FC>, +} + +const DataSourceOptions = ({ + pipelineNodes, + datasourceNodeId, + onSelect, +}: DataSourceOptionsProps) => { + const { datasources, options } = useDatasourceOptions(pipelineNodes) + + const handelSelect = useCallback((value: string) => { + const selectedOption = datasources.find(option => option.nodeId === value) + if (!selectedOption) + return + onSelect(selectedOption) + }, [datasources, onSelect]) + + useEffect(() => { + if (options.length > 0) + handelSelect(options[0].value) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + return ( +
+ {options.map(option => ( + + ))} +
+ ) +} + +export default DataSourceOptions diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx new file mode 100644 index 0000000000..4e54853282 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx @@ -0,0 +1,41 @@ +import React from 'react' +import cn from '@/utils/classnames' + +type OptionCardProps = { + label: string + Icon: React.FC> | string + selected: boolean + onClick?: () => void +} + +const OptionCard = ({ + label, + Icon, + selected, + onClick, +}: OptionCardProps) => { + return ( +
+
+ { + typeof Icon === 'string' + ?
{Icon}
+ : + } +
+
+ {label} +
+
+ ) +} + +export default React.memo(OptionCard) diff --git a/web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx b/web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx new file mode 100644 index 0000000000..7e3ff3df3f --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx @@ -0,0 +1,44 @@ +import React from 'react' +import Button from '@/app/components/base/button' +import { useTranslation } from 'react-i18next' +import { useParams } from 'next/navigation' +import { RiArrowRightLine } from '@remixicon/react' + +type ActionsProps = { + disabled?: boolean + handleNextStep: () => void +} + +const Actions = ({ + disabled, + handleNextStep, +}: ActionsProps) => { + const { t } = useTranslation() + const { datasetId } = useParams() + + return ( +
+ + + + +
+ ) +} + +export default React.memo(Actions) diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 1837466f79..429a6eb880 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -1,5 +1,11 @@ import { useTranslation } from 'react-i18next' import { AddDocumentsStep } from './types' +import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' +import { useMemo } from 'react' +import { BlockEnum, type Node } from '@/app/components/workflow/types' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import { DataSourceType } from '@/models/datasets' +import { DataSourceProvider } from '@/models/common' export const useAddDocumentsSteps = () => { const { t } = useTranslation() @@ -19,3 +25,79 @@ export const useAddDocumentsSteps = () => { ] return steps } + +export const useDatasourceOptions = (pipelineNodes: Node[]) => { + const { t } = useTranslation() + const datasources: Datasource[] = useMemo(() => { + const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource) + return datasourceNodes.map((node) => { + let type: DataSourceType | DataSourceProvider = DataSourceType.FILE + switch (node.data.tool_name) { + case 'file_upload': + type = DataSourceType.FILE + break + case 'search_notion': + type = DataSourceType.NOTION + break + case 'firecrawl': + type = DataSourceProvider.fireCrawl + break + case 'jina_reader': + type = DataSourceProvider.jinaReader + break + case 'water_crawl': + type = DataSourceProvider.waterCrawl + break + } + return { + nodeId: node.id, + type, + variables: node.data.variables, + } + }) + }, [pipelineNodes]) + + const options = useMemo(() => { + const options: DataSourceOption[] = [] + datasources.forEach((source) => { + if (source.type === DataSourceType.FILE) { + options.push({ + label: t('datasetPipeline.testRun.dataSource.localFiles'), + value: source.nodeId, + type: DataSourceType.FILE, + }) + } + if (source.type === DataSourceType.NOTION) { + options.push({ + label: 'Notion', + value: source.nodeId, + type: DataSourceType.NOTION, + }) + } + if (source.type === DataSourceProvider.fireCrawl) { + options.push({ + label: 'Firecrawl', + value: source.nodeId, + type: DataSourceProvider.fireCrawl, + }) + } + if (source.type === DataSourceProvider.jinaReader) { + options.push({ + label: 'Jina Reader', + value: source.nodeId, + type: DataSourceProvider.jinaReader, + }) + } + if (source.type === DataSourceProvider.waterCrawl) { + options.push({ + label: 'Water Crawl', + value: source.nodeId, + type: DataSourceProvider.waterCrawl, + }) + } + }) + return options + }, [datasources, t]) + + return { datasources, options } +} diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 7736b07eda..19edb8d586 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -1,33 +1,30 @@ 'use client' import { useCallback, useMemo, useState } from 'react' -// import StepIndicator from './step-indicator' -// import { useTestRunSteps } from './hooks' -// import DataSourceOptions from './data-source-options' -import type { CrawlResultItem, FileItem } from '@/models/datasets' +import DataSourceOptions from './data-source-options' +import type { CrawlResultItem, CustomFile as File, FileItem } from '@/models/datasets' import { DataSourceType } from '@/models/datasets' -// import LocalFile from './data-source/local-file' +import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file' import produce from 'immer' import { useProviderContextSelector } from '@/context/provider-context' import { DataSourceProvider, type NotionPage } from '@/models/common' -// import Notion from './data-source/notion' -import VectorSpaceFull from '@/app/components/billing/vector-space-full' -// import Firecrawl from './data-source/website/firecrawl' -// import JinaReader from './data-source/website/jina-reader' -// import WaterCrawl from './data-source/website/water-crawl' -// import Actions from './data-source/actions' -// import DocumentProcessing from './document-processing' -import { useTranslation } from 'react-i18next' -import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' -import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file' import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion' +import VectorSpaceFull from '@/app/components/billing/vector-space-full' import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl' import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader' import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl' -import Actions from '@/app/components/rag-pipeline/components/panel/test-run/data-source/actions' +import Actions from './data-source/actions' import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing' +import { useTranslation } from 'react-i18next' +import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' import LeftHeader from './left-header' -// import { usePipelineRun } from '../../../hooks' -// import type { Datasource } from './types' +import { usePublishedPipelineInfo } from '@/service/use-pipeline' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import Loading from '@/app/components/base/loading' +import type { Node } from '@/app/components/workflow/types' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' +import FilePreview from './preview/file-preview' +import NotionPagePreview from './preview/notion-page-preview' +import WebsitePreview from './preview/web-preview' const TestRunPanel = () => { const { t } = useTranslation() @@ -37,11 +34,15 @@ const TestRunPanel = () => { const [notionPages, setNotionPages] = useState([]) const [websitePages, setWebsitePages] = useState([]) const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') + const [currentFile, setCurrentFile] = useState() + const [currentNotionPage, setCurrentNotionPage] = useState() + const [currentWebsite, setCurrentWebsite] = useState() const plan = useProviderContextSelector(state => state.plan) const enableBilling = useProviderContextSelector(state => state.enableBilling) + const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) - // const steps = useTestRunSteps() + const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '') const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace @@ -79,13 +80,37 @@ const TestRunPanel = () => { setFiles(newList) } - const updateFileList = (preparedFiles: FileItem[]) => { + const updateFileList = useCallback((preparedFiles: FileItem[]) => { setFiles(preparedFiles) - } + }, []) - const updateNotionPages = (value: NotionPage[]) => { + const updateNotionPages = useCallback((value: NotionPage[]) => { setNotionPages(value) - } + }, []) + + const updateCurrentFile = useCallback((file: File) => { + setCurrentFile(file) + }, []) + + const hideFilePreview = useCallback(() => { + setCurrentFile(undefined) + }, []) + + const updateCurrentPage = useCallback((page: NotionPage) => { + setCurrentNotionPage(page) + }, []) + + const hideNotionPagePreview = useCallback(() => { + setCurrentNotionPage(undefined) + }, []) + + const updateCurrentWebsite = useCallback((website: CrawlResultItem) => { + setCurrentWebsite(website) + }, []) + + const hideWebsitePreview = useCallback(() => { + setCurrentWebsite(undefined) + }, []) const handleNextStep = useCallback(() => { setCurrentStep(preStep => preStep + 1) @@ -95,8 +120,6 @@ const TestRunPanel = () => { setCurrentStep(preStep => preStep - 1) }, []) - // const { handleRun } = usePipelineRun() - const handleProcess = useCallback((data: Record) => { if (!datasource) return @@ -121,13 +144,16 @@ const TestRunPanel = () => { datasourceInfo.jobId = websiteCrawlJobId datasourceInfo.result = websitePages } - // handleRun({ - // inputs: data, - // datasource_type, - // datasource_info: datasourceInfo, - // }) + // todo: Run Pipeline + console.log('datasource_type', datasource_type) }, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages]) + if (isFetchingPipelineInfo) { + return ( + + ) + } + return (
{
{ currentStep === 1 && ( - <> -
- {/* + */} - {datasource?.type === DataSourceType.FILE && ( - - )} - {datasource?.type === DataSourceType.NOTION && ( - - )} - {datasource?.type === DataSourceProvider.fireCrawl && ( - - )} - {datasource?.type === DataSourceProvider.jinaReader && ( - - )} - {datasource?.type === DataSourceProvider.waterCrawl && ( - - )} - {isShowVectorSpaceFull && ( - - )} -
+ pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node[]} + /> + {datasource?.type === DataSourceType.FILE && ( + + )} + {datasource?.type === DataSourceType.NOTION && ( + + )} + {datasource?.type === DataSourceProvider.fireCrawl && ( + + )} + {datasource?.type === DataSourceProvider.jinaReader && ( + + )} + {datasource?.type === DataSourceProvider.waterCrawl && ( + + )} + {isShowVectorSpaceFull && ( + + )} - +
) } { @@ -209,6 +240,15 @@ const TestRunPanel = () => {
{/* Preview */}
+ { + currentStep === 1 && ( + <> + {currentFile && } + {currentNotionPage && } + {currentWebsite && } + + ) + }
) diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx new file mode 100644 index 0000000000..ab6b0d8e12 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx @@ -0,0 +1,78 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import Loading from './loading' +import type { CustomFile as File } from '@/models/datasets' +import { RiCloseLine } from '@remixicon/react' +import { useFilePreview } from '@/service/use-common' +import DocumentFileIcon from '../../../common/document-file-icon' +import { formatNumberAbbreviated } from '@/utils/format' + +type FilePreviewProps = { + file: File + hidePreview: () => void +} + +const FilePreview = ({ + file, + hidePreview, +}: FilePreviewProps) => { + const { t } = useTranslation() + const { data: fileData, isFetching } = useFilePreview(file.id || '') + + const getFileName = (currentFile?: File) => { + if (!currentFile) + return '' + const arr = currentFile.name.split('.') + return arr.slice(0, -1).join() + } + + const getFileSize = (size: number) => { + if (size / 1024 < 10) + return `${(size / 1024).toFixed(1)} KB` + + return `${(size / 1024 / 1024).toFixed(1)} MB` + } + + return ( +
+
+
+
{t('datasetPipeline.addDocuments.stepOne.preview')}
+
{`${getFileName(file)}.${file.extension}`}
+
+ + {file.extension} + · + {getFileSize(file.size)} + {fileData && ( + <> + · + {`${formatNumberAbbreviated(fileData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`} + + )} +
+
+ +
+
+ {isFetching && } + {!isFetching && fileData && ( +
{fileData.content}
+ )} +
+
+ ) +} + +export default FilePreview diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx new file mode 100644 index 0000000000..3fd7723ae3 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx @@ -0,0 +1,52 @@ +import React from 'react' +import { SkeletonContainer, SkeletonRectangle } from '@/app/components/base/skeleton' + +const Loading = () => { + return ( +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ ) +} + +export default React.memo(Loading) diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx new file mode 100644 index 0000000000..711057a295 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx @@ -0,0 +1,65 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import type { NotionPage } from '@/models/common' +import { usePreviewNotionPage } from '@/service/knowledge/use-dataset' +import { RiCloseLine } from '@remixicon/react' +import { formatNumberAbbreviated } from '@/utils/format' +import Loading from './loading' +import { Notion } from '@/app/components/base/icons/src/public/common' + +type NotionPagePreviewProps = { + currentPage: NotionPage + hidePreview: () => void +} + +const NotionPagePreview = ({ + currentPage, + hidePreview, +}: NotionPagePreviewProps) => { + const { t } = useTranslation() + + const { data: notionPageData, isFetching } = usePreviewNotionPage({ + workspaceID: currentPage.workspace_id, + pageID: currentPage.page_id, + pageType: currentPage.type, + }) + + return ( +
+
+
+
{t('datasetPipeline.addDocuments.stepOne.preview')}
+
{currentPage?.page_name}
+
+ + · + Notion Page + · + {notionPageData && ( + <> + · + {`${formatNumberAbbreviated(notionPageData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`} + + )} +
+
+ +
+
+ {isFetching && } + {!isFetching && notionPageData && ( +
{notionPageData.content}
+ )} +
+
+ ) +} + +export default NotionPagePreview diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx new file mode 100644 index 0000000000..0b9ee6a5a0 --- /dev/null +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx @@ -0,0 +1,48 @@ +'use client' +import React from 'react' +import { useTranslation } from 'react-i18next' +import type { CrawlResultItem } from '@/models/datasets' +import { RiCloseLine, RiGlobalLine } from '@remixicon/react' +import { formatNumberAbbreviated } from '@/utils/format' + +type WebsitePreviewProps = { + payload: CrawlResultItem + hidePreview: () => void +} + +const WebsitePreview = ({ + payload, + hidePreview, +}: WebsitePreviewProps) => { + const { t } = useTranslation() + + return ( +
+
+
+
{t('datasetPipeline.addDocuments.stepOne.preview')}
+
{payload.title}
+
+ + {payload.source_url} + · + · + {`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`} +
+
+ +
+
+
{payload.markdown}
+
+
+ ) +} + +export default WebsitePreview diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx index 2f800b1370..cfa4495c2c 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx @@ -23,6 +23,7 @@ type IFileUploaderProps = { prepareFileList: (files: FileItem[]) => void onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void onFileListUpdate?: (files: FileItem[]) => void + onPreview?: (file: File) => void notSupportBatchUpload?: boolean } @@ -31,6 +32,7 @@ const FileUploader = ({ prepareFileList, onFileUpdate, onFileListUpdate, + onPreview, notSupportBatchUpload, }: IFileUploaderProps) => { const { t } = useTranslation() @@ -284,6 +286,7 @@ const FileUploader = ({ return (
fileItem.file?.id && onPreview?.(fileItem.file)} className={cn( 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', isError && 'border-state-destructive-border bg-state-destructive-hover', diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx index a0677ae183..65adf409f1 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx @@ -1,10 +1,11 @@ -import type { FileItem } from '@/models/datasets' +import type { CustomFile as File, FileItem } from '@/models/datasets' import FileUploader from './file-uploader' type LocalFileProps = { files: FileItem[] updateFileList: (files: FileItem[]) => void updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void + onPreview?: (file: File) => void notSupportBatchUpload: boolean } @@ -12,6 +13,7 @@ const LocalFile = ({ files, updateFileList, updateFile, + onPreview, notSupportBatchUpload, }: LocalFileProps) => { return ( @@ -20,6 +22,7 @@ const LocalFile = ({ prepareFileList={updateFileList} onFileListUpdate={updateFileList} onFileUpdate={updateFile} + onPreview={onPreview} notSupportBatchUpload={notSupportBatchUpload} /> ) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx index bbc87cc650..cfa2efe564 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx @@ -5,20 +5,27 @@ type NotionProps = { nodeId: string notionPages: NotionPage[] updateNotionPages: (value: NotionPage[]) => void + canPreview?: boolean + onPreview?: (selectedPage: NotionPage) => void + isInPipeline?: boolean } const Notion = ({ nodeId, notionPages, updateNotionPages, + canPreview = false, + onPreview, + isInPipeline = false, }: NotionProps) => { return ( page.page_id)} onSelect={updateNotionPages} - canPreview={false} - isInPipeline + canPreview={canPreview} + onPreview={onPreview} + isInPipeline={isInPipeline} /> ) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx index 7d7dbf33b9..58b7d1c5ac 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx @@ -3,23 +3,33 @@ import React, { useCallback } from 'react' import cn from '@/utils/classnames' import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' import Checkbox from '@/app/components/base/checkbox' +import Button from '@/app/components/base/button' +import { useTranslation } from 'react-i18next' type CrawledResultItemProps = { payload: CrawlResultItemType isChecked: boolean onCheckChange: (checked: boolean) => void + isPreview: boolean + showPreview: boolean + onPreview: () => void } const CrawledResultItem = ({ payload, isChecked, onCheckChange, + isPreview, + onPreview, + showPreview, }: CrawledResultItemProps) => { + const { t } = useTranslation() + const handleCheckChange = useCallback(() => { onCheckChange(!isChecked) }, [isChecked, onCheckChange]) return ( -
+
+ {showPreview && }
) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx index 483291d436..29737b0cdd 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx @@ -1,5 +1,5 @@ 'use client' -import React, { useCallback } from 'react' +import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import cn from '@/utils/classnames' import type { CrawlResultItem } from '@/models/datasets' @@ -13,6 +13,7 @@ type CrawledResultProps = { list: CrawlResultItem[] checkedList: CrawlResultItem[] onSelectedChange: (selected: CrawlResultItem[]) => void + onPreview?: (payload: CrawlResultItem) => void usedTime: number } @@ -22,8 +23,10 @@ const CrawledResult = ({ checkedList, onSelectedChange, usedTime, + onPreview, }: CrawledResultProps) => { const { t } = useTranslation() + const [previewIndex, setPreviewIndex] = useState(-1) const isCheckAll = checkedList.length === list.length @@ -45,6 +48,12 @@ const CrawledResult = ({ } }, [checkedList, onSelectedChange]) + const handlePreview = useCallback((index: number) => { + if (!onPreview) return + setPreviewIndex(index) + onPreview(list[index]) + }, [list, onPreview]) + return (
@@ -61,12 +70,15 @@ const CrawledResult = ({ />
- {list.map(item => ( + {list.map((item, index) => ( checkedItem.source_url === item.source_url)} onCheckChange={handleItemCheckChange(item)} + isPreview={index === previewIndex} + onPreview={handlePreview.bind(null, index)} + showPreview={!!onPreview} /> ))}
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx index a82272726b..54a556ab7b 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx @@ -22,6 +22,7 @@ type CrawlerProps = { datasourceProvider: DataSourceProvider onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void + onPreview?: (payload: CrawlResultItem) => void } enum Step { @@ -37,6 +38,7 @@ const Crawler = ({ datasourceProvider, onCheckedCrawlResultChange, onJobIdChange, + onPreview, }: CrawlerProps) => { const { t } = useTranslation() const [step, setStep] = useState(Step.init) @@ -123,6 +125,7 @@ const Crawler = ({ checkedList={checkedCrawlResult} onSelectedChange={onCheckedCrawlResultChange} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} + onPreview={onPreview} /> )}
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx index c6bbcf851e..438c37e392 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx @@ -11,6 +11,7 @@ type FireCrawlProps = { checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void + onPreview?: (payload: CrawlResultItem) => void } const FireCrawl = ({ @@ -19,6 +20,7 @@ const FireCrawl = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + onPreview, }: FireCrawlProps) => { return ( ) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx index 19cf3862a1..dbe6e1c0f7 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx @@ -11,6 +11,7 @@ type JinaReaderProps = { checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void + onPreview?: (payload: CrawlResultItem) => void } const JinaReader = ({ @@ -19,6 +20,7 @@ const JinaReader = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + onPreview, }: JinaReaderProps) => { return ( ) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx index beb586c4dd..e17733c9d9 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx @@ -11,6 +11,7 @@ type WaterCrawlProps = { checkedCrawlResult: CrawlResultItem[] onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onJobIdChange: (jobId: string) => void + onPreview?: (payload: CrawlResultItem) => void } const WaterCrawl = ({ @@ -19,6 +20,7 @@ const WaterCrawl = ({ checkedCrawlResult, onCheckedCrawlResultChange, onJobIdChange, + onPreview, }: WaterCrawlProps) => { return ( ) } diff --git a/web/i18n/en-US/dataset-pipeline.ts b/web/i18n/en-US/dataset-pipeline.ts index 3fc6b336d1..645eeeeb63 100644 --- a/web/i18n/en-US/dataset-pipeline.ts +++ b/web/i18n/en-US/dataset-pipeline.ts @@ -78,6 +78,10 @@ const translation = { processingDocuments: 'Processing Documents', }, backToDataSource: 'Data Source', + stepOne: { + preview: 'Preview', + }, + characters: 'characters', }, } diff --git a/web/i18n/zh-Hans/dataset-pipeline.ts b/web/i18n/zh-Hans/dataset-pipeline.ts index 670d97a18f..d3e9ab420e 100644 --- a/web/i18n/zh-Hans/dataset-pipeline.ts +++ b/web/i18n/zh-Hans/dataset-pipeline.ts @@ -78,6 +78,10 @@ const translation = { processingDocuments: '正在处理文档', }, backToDataSource: '数据源', + stepOne: { + preview: '预览', + }, + characters: '字符', }, } diff --git a/web/service/knowledge/use-dataset.ts b/web/service/knowledge/use-dataset.ts index 5d1e30b225..ff88a748ed 100644 --- a/web/service/knowledge/use-dataset.ts +++ b/web/service/knowledge/use-dataset.ts @@ -44,3 +44,22 @@ export const useDatasetRelatedApps = (datasetId: string) => { queryFn: () => get(`/datasets/${datasetId}/related-apps`), }) } + +type NotionPagePreviewRequest = { + workspaceID: string + pageID: string + pageType: string +} + +type NotionPagePreviewResponse = { + content: string +} + +export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => { + const { workspaceID, pageID, pageType } = params + return useQuery({ + queryKey: [NAME_SPACE, 'preview-notion-page'], + queryFn: () => get(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`), + enabled: !!workspaceID && !!pageID && !!pageType, + }) +} diff --git a/web/service/use-common.ts b/web/service/use-common.ts index df2f5e623b..167cf4e486 100644 --- a/web/service/use-common.ts +++ b/web/service/use-common.ts @@ -60,3 +60,15 @@ export const useMembers = () => { }), }) } + +type FilePreviewResponse = { + content: string +} + +export const useFilePreview = (fileID: string) => { + return useQuery({ + queryKey: [NAME_SPACE, 'file-preview', fileID], + queryFn: () => get(`/files/${fileID}/preview`), + enabled: !!fileID, + }) +} diff --git a/web/service/use-pipeline.ts b/web/service/use-pipeline.ts index 9b1215ab8a..d9b7afc2b0 100644 --- a/web/service/use-pipeline.ts +++ b/web/service/use-pipeline.ts @@ -164,5 +164,6 @@ export const usePublishedPipelineInfo = (pipelineId: string) => { queryFn: () => { return get(`/rag/pipelines/${pipelineId}/workflows/publish`) }, + enabled: !!pipelineId, }) } diff --git a/web/utils/format.ts b/web/utils/format.ts index 720c8f6762..572398405b 100644 --- a/web/utils/format.ts +++ b/web/utils/format.ts @@ -56,3 +56,35 @@ export const downloadFile = ({ data, fileName }: { data: Blob; fileName: string a.remove() window.URL.revokeObjectURL(url) } + +/** + * Formats a number into a readable string using "k", "M", or "B" suffix. + * @example + * 950 => "950" + * 1200 => "1.2k" + * 1500000 => "1.5M" + * 2000000000 => "2B" + * + * @param {number} num - The number to format + * @returns {string} - The formatted number string + */ +export const formatNumberAbbreviated = (num: number) => { + // If less than 1000, return as-is + if (num < 1000) return num.toString() + + // Define thresholds and suffixes + const units = [ + { value: 1e9, symbol: 'B' }, + { value: 1e6, symbol: 'M' }, + { value: 1e3, symbol: 'k' }, + ] + + for (let i = 0; i < units.length; i++) { + if (num >= units[i].value) { + const formatted = (num / units[i].value).toFixed(1) + return formatted.endsWith('.0') + ? `${Number.parseInt(formatted)}${units[i].symbol}` + : `${formatted}${units[i].symbol}` + } + } +}