From cbf0864edcdbd13af7ef2763da6357ad5c9622d8 Mon Sep 17 00:00:00 2001 From: twwu Date: Fri, 6 Jun 2025 10:08:19 +0800 Subject: [PATCH] refactor: refactor online documents handling and update related components --- .../documents/create-from-pipeline/hooks.ts | 28 +++++------ .../documents/create-from-pipeline/index.tsx | 46 +++++++++---------- .../preview/chunk-preview.tsx | 22 ++++----- .../{notion => online-documents}/index.tsx | 24 +++++----- .../online-document-selector.tsx} | 36 +++++++-------- .../components/panel/test-run/hooks.ts | 12 ++--- .../components/panel/test-run/index.tsx | 24 +++++----- 7 files changed, 96 insertions(+), 96 deletions(-) rename web/app/components/rag-pipeline/components/panel/test-run/data-source/{notion => online-documents}/index.tsx (53%) rename web/app/components/rag-pipeline/components/panel/test-run/data-source/{notion/notion-page-selector.tsx => online-documents/online-document-selector.tsx} (82%) diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 9eca3c4b4c..65a0f33786 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -119,31 +119,31 @@ export const useLocalFile = () => { } } -export const useNotionsPages = () => { - const [notionPages, setNotionPages] = useState([]) - const [currentNotionPage, setCurrentNotionPage] = useState() +export const useOnlineDocuments = () => { + const [onlineDocuments, setOnlineDocuments] = useState([]) + const [currentDocuments, setCurrentDocuments] = useState() - const previewNotionPage = useRef(notionPages[0]) + const previewOnlineDocument = useRef(onlineDocuments[0]) - const updateNotionPages = (value: NotionPage[]) => { - setNotionPages(value) + const updateOnlineDocuments = (value: NotionPage[]) => { + setOnlineDocuments(value) } const updateCurrentPage = useCallback((page: NotionPage) => { - setCurrentNotionPage(page) + setCurrentDocuments(page) }, []) - const hideNotionPagePreview = useCallback(() => { - setCurrentNotionPage(undefined) + const hideOnlineDocumentPreview = useCallback(() => { + setCurrentDocuments(undefined) }, []) return { - notionPages, - previewNotionPage, - updateNotionPages, - currentNotionPage, + onlineDocuments, + previewOnlineDocument, + updateOnlineDocuments, + currentDocuments, updateCurrentPage, - hideNotionPagePreview, + hideOnlineDocumentPreview, } } diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index b216b577e0..a293697790 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -5,7 +5,7 @@ import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEst import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file' import { useProviderContextSelector } from '@/context/provider-context' import type { NotionPage } from '@/models/common' -import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion' +import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents' import VectorSpaceFull from '@/app/components/billing/vector-space-full' import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl' import Actions from './data-source/actions' @@ -26,7 +26,7 @@ import Processing from './processing' import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline' import { DatasourceType } from '@/models/pipeline' import { TransferMethod } from '@/types/app' -import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks' +import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks' const CreateFormPipeline = () => { const { t } = useTranslation() @@ -63,13 +63,13 @@ const CreateFormPipeline = () => { hideFilePreview, } = useLocalFile() const { - notionPages, - previewNotionPage, - updateNotionPages, - currentNotionPage, + onlineDocuments, + previewOnlineDocument, + updateOnlineDocuments, + currentDocuments, updateCurrentPage, - hideNotionPagePreview, - } = useNotionsPages() + hideOnlineDocumentPreview, + } = useOnlineDocuments() const { websitePages, websiteCrawlJobId, @@ -90,11 +90,11 @@ const CreateFormPipeline = () => { if (datasource.type === DatasourceType.localFile) return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id) if (datasource.type === DatasourceType.onlineDocument) - return isShowVectorSpaceFull || !notionPages.length + return isShowVectorSpaceFull || !onlineDocuments.length if (datasource.type === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length return false - }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length]) + }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length]) const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline() @@ -117,7 +117,7 @@ const CreateFormPipeline = () => { datasourceInfoList.push(documentInfo) } if (datasource.type === DatasourceType.onlineDocument) { - const { workspace_id, ...rest } = previewNotionPage.current + const { workspace_id, ...rest } = previewOnlineDocument.current const documentInfo = { workspace_id, page: rest, @@ -143,7 +143,7 @@ const CreateFormPipeline = () => { setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs) }, }) - }, [datasource, pipelineId, previewFile, previewNotionPage, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId]) + }, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId]) const handleProcess = useCallback(async (data: Record) => { if (!datasource) @@ -166,7 +166,7 @@ const CreateFormPipeline = () => { }) } if (datasource.type === DatasourceType.onlineDocument) { - notionPages.forEach((page) => { + onlineDocuments.forEach((page) => { const { workspace_id, ...rest } = page const documentInfo = { workspace_id, @@ -196,7 +196,7 @@ const CreateFormPipeline = () => { handleNextStep() }, }) - }, [datasource, fileList, handleNextStep, notionPages, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages]) + }, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages]) const onClickProcess = useCallback(() => { isPreview.current = false @@ -217,10 +217,10 @@ const CreateFormPipeline = () => { onClickPreview() }, [onClickPreview, previewFile]) - const handlePreviewNotionPageChange = useCallback((page: NotionPage) => { - previewNotionPage.current = page + const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => { + previewOnlineDocument.current = page onClickPreview() - }, [onClickPreview, previewNotionPage]) + }, [onClickPreview, previewOnlineDocument]) const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => { previewWebsitePage.current = website @@ -263,15 +263,15 @@ const CreateFormPipeline = () => { /> )} {datasource?.type === DatasourceType.onlineDocument && ( - @@ -327,7 +327,7 @@ const CreateFormPipeline = () => { currentStep === 1 && (
{currentFile && } - {currentNotionPage && } + {currentDocuments && } {currentWebsite && }
) @@ -339,14 +339,14 @@ const CreateFormPipeline = () => { file.file)} - notionPages={notionPages} + onlineDocuments={onlineDocuments} websitePages={websitePages} isIdle={isIdle && isPreview.current} isPending={isPending && isPreview.current} estimateData={estimateData} onPreview={onClickPreview} handlePreviewFileChange={handlePreviewFileChange} - handlePreviewNotionPageChange={handlePreviewNotionPageChange} + handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange} handlePreviewWebsitePageChange={handlePreviewWebsiteChange} /> )} diff --git a/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx b/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx index 61779bd57b..cbc3882061 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx @@ -20,35 +20,35 @@ import { DatasourceType } from '@/models/pipeline' type ChunkPreviewProps = { datasource: Datasource files: CustomFile[] - notionPages: NotionPage[] + onlineDocuments: NotionPage[] websitePages: CrawlResultItem[] isIdle: boolean isPending: boolean estimateData: FileIndexingEstimateResponse | undefined onPreview: () => void handlePreviewFileChange: (file: DocumentItem) => void - handlePreviewNotionPageChange: (page: NotionPage) => void + handlePreviewOnlineDocumentChange: (page: NotionPage) => void handlePreviewWebsitePageChange: (page: CrawlResultItem) => void } const ChunkPreview = ({ datasource, files, - notionPages, + onlineDocuments, websitePages, isIdle, isPending, estimateData, onPreview, handlePreviewFileChange, - handlePreviewNotionPageChange, + handlePreviewOnlineDocumentChange, handlePreviewWebsitePageChange, }: ChunkPreviewProps) => { const { t } = useTranslation() const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form) const [previewFile, setPreviewFile] = useState(files[0] as DocumentItem) - const [previewNotionPage, setPreviewNotionPage] = useState(notionPages[0]) + const [previewOnlineDocument, setPreviewOnlineDocument] = useState(onlineDocuments[0]) const [previewWebsitePage, setPreviewWebsitePage] = useState(websitePages[0]) const dataSourceType = datasource?.type @@ -72,20 +72,20 @@ const ChunkPreview = ({ {dataSourceType === DatasourceType.onlineDocument && ({ + onlineDocuments.map(page => ({ id: page.page_id, name: page.page_name, extension: 'md', })) } onChange={(selected) => { - const selectedPage = notionPages.find(page => page.page_id === selected.id) - setPreviewNotionPage(selectedPage!) - handlePreviewNotionPageChange(selectedPage!) + const selectedPage = onlineDocuments.find(page => page.page_id === selected.id) + setPreviewOnlineDocument(selectedPage!) + handlePreviewOnlineDocumentChange(selectedPage!) }} value={{ - id: previewNotionPage?.page_id || '', - name: previewNotionPage?.page_name || '', + id: previewOnlineDocument?.page_id || '', + name: previewOnlineDocument?.page_name || '', extension: 'md', }} /> diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx similarity index 53% rename from web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx rename to web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx index 31c175d0c4..1a4abf86e1 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx @@ -1,35 +1,35 @@ import type { NotionPage } from '@/models/common' -import NotionPageSelector from './notion-page-selector' +import OnlineDocumentSelector from './online-document-selector' -type NotionProps = { +type OnlineDocumentsProps = { nodeId: string headerInfo: { title: string docTitle: string docLink: string } - notionPages: NotionPage[] - updateNotionPages: (value: NotionPage[]) => void + onlineDocuments: NotionPage[] + updateOnlineDocuments: (value: NotionPage[]) => void canPreview?: boolean onPreview?: (selectedPage: NotionPage) => void isInPipeline?: boolean } -const Notion = ({ +const OnlineDocuments = ({ nodeId, headerInfo, - notionPages, - updateNotionPages, + onlineDocuments, + updateOnlineDocuments, canPreview = false, onPreview, isInPipeline = false, -}: NotionProps) => { +}: OnlineDocumentsProps) => { return ( - page.page_id)} - onSelect={updateNotionPages} + value={onlineDocuments.map(page => page.page_id)} + onSelect={updateOnlineDocuments} canPreview={canPreview} onPreview={onPreview} isInPipeline={isInPipeline} @@ -37,4 +37,4 @@ const Notion = ({ ) } -export default Notion +export default OnlineDocuments diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx similarity index 82% rename from web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx rename to web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx index 672d7eb9d2..e8d8ceb700 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx @@ -8,7 +8,7 @@ import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { useDraftDatasourceNodeRun, usePublishedDatasourceNodeRun } from '@/service/use-pipeline' import { DatasourceType } from '@/models/pipeline' -type NotionPageSelectorProps = { +type OnlineDocumentSelectorProps = { value?: string[] onSelect: (selectedPages: NotionPage[]) => void canPreview?: boolean @@ -23,7 +23,7 @@ type NotionPageSelectorProps = { } } -const NotionPageSelector = ({ +const OnlineDocumentSelector = ({ value, onSelect, canPreview, @@ -32,42 +32,42 @@ const NotionPageSelector = ({ isInPipeline = false, nodeId, headerInfo, -}: NotionPageSelectorProps) => { +}: OnlineDocumentSelectorProps) => { const pipeline_id = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) - const [notionData, setNotionData] = useState([]) + const [documentsData, setDocumentsData] = useState([]) const [searchValue, setSearchValue] = useState('') const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') const useDatasourceNodeRun = useRef(!isInPipeline ? usePublishedDatasourceNodeRun : useDraftDatasourceNodeRun) - const { mutateAsync: getNotionPages } = useDatasourceNodeRun.current() + const { mutateAsync: crawlOnlineDocuments } = useDatasourceNodeRun.current() - const getNotionData = useCallback(async () => { + const getOnlineDocuments = useCallback(async () => { if (pipeline_id) { - await getNotionPages({ + await crawlOnlineDocuments({ pipeline_id, node_id: nodeId, inputs: {}, datasource_type: DatasourceType.onlineDocument, }, { - onSuccess(notionData) { - setNotionData(notionData as DataSourceNotionWorkspace[]) + onSuccess(documentsData) { + setDocumentsData(documentsData as DataSourceNotionWorkspace[]) }, }) } - }, [getNotionPages, nodeId, pipeline_id]) + }, [crawlOnlineDocuments, nodeId, pipeline_id]) useEffect(() => { - getNotionData() + getOnlineDocuments() // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - const firstWorkspaceId = notionData[0]?.workspace_id - const currentWorkspace = notionData.find(workspace => workspace.workspace_id === currentWorkspaceId) + const firstWorkspaceId = documentsData[0]?.workspace_id + const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId) const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { const selectedPagesId = new Set() const boundPagesId = new Set() - const pagesMap = notionData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { + const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { next.pages.forEach((page) => { if (page.is_bound) { selectedPagesId.add(page.page_id) @@ -82,7 +82,7 @@ const NotionPageSelector = ({ return prev }, {}) return [pagesMap, selectedPagesId, boundPagesId] - }, [notionData]) + }, [documentsData]) const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])] const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) @@ -107,7 +107,7 @@ const NotionPageSelector = ({ setCurrentWorkspaceId(firstWorkspaceId) }, [firstWorkspaceId]) - if (!notionData?.length) + if (!documentsData?.length) return null return ( @@ -121,7 +121,7 @@ const NotionPageSelector = ({
@@ -148,4 +148,4 @@ const NotionPageSelector = ({ ) } -export default NotionPageSelector +export default OnlineDocumentSelector diff --git a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts index 4a247a9599..20dca3b4bc 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts @@ -101,16 +101,16 @@ export const useLocalFile = () => { } } -export const useNotionPages = () => { - const [notionPages, setNotionPages] = useState([]) +export const useOnlineDocuments = () => { + const [onlineDocuments, setOnlineDocuments] = useState([]) - const updateNotionPages = (value: NotionPage[]) => { - setNotionPages(value) + const updateOnlineDocuments = (value: NotionPage[]) => { + setOnlineDocuments(value) } return { - notionPages, - updateNotionPages, + onlineDocuments, + updateOnlineDocuments, } } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index 9eb0e06b4c..b924200922 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -1,10 +1,10 @@ import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store' import { useCallback, useMemo, useState } from 'react' -import { useLocalFile, useNotionPages, useTestRunSteps, useWebsiteCrawl } from './hooks' +import { useLocalFile, useOnlineDocuments, useTestRunSteps, useWebsiteCrawl } from './hooks' import DataSourceOptions from './data-source-options' import LocalFile from './data-source/local-file' import { useProviderContextSelector } from '@/context/provider-context' -import Notion from './data-source/notion' +import OnlineDocuments from './data-source/online-documents' import VectorSpaceFull from '@/app/components/billing/vector-space-full' import WebsiteCrawl from './data-source/website-crawl' import Actions from './data-source/actions' @@ -35,9 +35,9 @@ const TestRunPanel = () => { updateFileList, } = useLocalFile() const { - notionPages, - updateNotionPages, - } = useNotionPages() + onlineDocuments, + updateOnlineDocuments, + } = useOnlineDocuments() const { websitePages, websiteCrawlJobId, @@ -54,11 +54,11 @@ const TestRunPanel = () => { if (datasource.type === DatasourceType.localFile) return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id) if (datasource.type === DatasourceType.onlineDocument) - return isShowVectorSpaceFull || !notionPages.length + return isShowVectorSpaceFull || !onlineDocuments.length if (datasource.type === DatasourceType.websiteCrawl) return isShowVectorSpaceFull || !websitePages.length return false - }, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length]) + }, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length]) const handleClose = () => { setShowDebugAndPreviewPanel(false) @@ -83,7 +83,7 @@ const TestRunPanel = () => { datasourceInfoList.push(documentInfo) } if (datasource.type === DatasourceType.onlineDocument) { - const { workspace_id, ...rest } = notionPages[0] + const { workspace_id, ...rest } = onlineDocuments[0] const documentInfo = { workspace_id, page: rest, @@ -103,7 +103,7 @@ const TestRunPanel = () => { datasource_type: datasource.type, datasource_info_list: datasourceInfoList, }) - }, [datasource, fileList, handleRun, notionPages, websiteCrawlJobId, websitePages]) + }, [datasource, fileList, handleRun, onlineDocuments, websiteCrawlJobId, websitePages]) return (
{ /> )} {datasource?.type === DatasourceType.onlineDocument && ( - )}