diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx index 6b843b499e..bbc87cc650 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx @@ -1,43 +1,25 @@ -import { useDataSources } from '@/service/use-common' -import { useCallback, useMemo } from 'react' -import { NotionPageSelector } from '@/app/components/base/notion-page-selector' import type { NotionPage } from '@/models/common' -import NotionConnector from '@/app/components/base/notion-connector' -import { useModalContextSelector } from '@/context/modal-context' +import NotionPageSelector from './notion-page-selector' type NotionProps = { + nodeId: string notionPages: NotionPage[] updateNotionPages: (value: NotionPage[]) => void } const Notion = ({ + nodeId, notionPages, updateNotionPages, }: NotionProps) => { - const { data: dataSources } = useDataSources() - const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal) - - const hasConnection = useMemo(() => { - const notionDataSources = dataSources?.data.filter(item => item.provider === 'notion') || [] - return notionDataSources.length > 0 - }, [dataSources]) - - const handleConnect = useCallback(() => { - setShowAccountSettingModal({ payload: 'data-source' }) - }, [setShowAccountSettingModal]) - return ( - <> - {!hasConnection && } - {hasConnection && ( - page.page_id)} - onSelect={updateNotionPages} - canPreview={false} - isInPipeline - /> - )} - + page.page_id)} + onSelect={updateNotionPages} + canPreview={false} + isInPipeline + /> ) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx new file mode 100644 index 0000000000..8d759def6b --- /dev/null +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx @@ -0,0 +1,146 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector' +import SearchInput from '@/app/education-apply/search-input' +import PageSelector from '@/app/components/base/notion-page-selector/page-selector' +import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' +import Header from '@/app/components/datasets/create/website/base/header' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { useDatasourceNodeRun } from '@/service/use-pipeline' +import { useTranslation } from 'react-i18next' + +type NotionPageSelectorProps = { + value?: string[] + onSelect: (selectedPages: NotionPage[]) => void + canPreview?: boolean + previewPageId?: string + onPreview?: (selectedPage: NotionPage) => void + isInPipeline?: boolean + nodeId: string +} + +const NotionPageSelector = ({ + value, + onSelect, + canPreview, + previewPageId, + onPreview, + isInPipeline = false, + nodeId, +}: NotionPageSelectorProps) => { + const { t } = useTranslation() + const pipeline_id = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const { mutateAsync: getNotionPages } = useDatasourceNodeRun() + const [notionData, setNotionData] = useState() + const [searchValue, setSearchValue] = useState('') + const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') + + const getNotionData = useCallback(async () => { + if (pipeline_id) { + const notionData = await getNotionPages({ + pipeline_id, + node_id: nodeId, + inputs: {}, + }) as DataSourceNotionWorkspace[] + setNotionData(notionData) + } + }, [getNotionPages, nodeId, pipeline_id]) + + useEffect(() => { + getNotionData() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + const notionWorkspaces = useMemo(() => { + return notionData || [] + }, [notionData]) + + const firstWorkspaceId = notionWorkspaces[0]?.workspace_id + const currentWorkspace = notionWorkspaces.find(workspace => workspace.workspace_id === currentWorkspaceId) + + const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { + const selectedPagesId = new Set() + const boundPagesId = new Set() + const pagesMap = notionWorkspaces.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { + next.pages.forEach((page) => { + if (page.is_bound) { + selectedPagesId.add(page.page_id) + boundPagesId.add(page.page_id) + } + prev[page.page_id] = { + ...page, + workspace_id: next.workspace_id, + } + }) + + return prev + }, {}) + return [pagesMap, selectedPagesId, boundPagesId] + }, [notionWorkspaces]) + const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])] + const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) + + const handleSearchValueChange = useCallback((value: string) => { + setSearchValue(value) + }, []) + const handleSelectWorkspace = useCallback((workspaceId: string) => { + setCurrentWorkspaceId(workspaceId) + }, []) + const handleSelectPages = (newSelectedPagesId: Set) => { + const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId]) + + setSelectedPagesId(new Set(Array.from(newSelectedPagesId))) + onSelect(selectedPages) + } + const handlePreviewPage = (previewPageId: string) => { + if (onPreview) + onPreview(PagesMapAndSelectedPagesId[0][previewPageId]) + } + + useEffect(() => { + setCurrentWorkspaceId(firstWorkspaceId) + }, [firstWorkspaceId]) + + if (!notionData?.length) + return null + + return ( +
+
+
+
+
+ +
+ +
+
+ +
+
+
+ ) +} + +export default NotionPageSelector diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index fac5005b94..ac053853a4 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -49,7 +49,7 @@ const TestRunPanel = () => { }, [fileList, isShowVectorSpaceFull]) const nextBtnDisabled = useMemo(() => { - if (!datasource) return false + if (!datasource) return true if (datasource.type === DataSourceType.FILE) return nextDisabled if (datasource.type === DataSourceType.NOTION) @@ -98,9 +98,13 @@ const TestRunPanel = () => { if (!datasource) return const datasourceInfo: Record = {} - if (datasource.type === DataSourceType.FILE) + let datasource_type = '' + if (datasource.type === DataSourceType.FILE) { + datasource_type = 'local_file' datasourceInfo.fileId = fileList.map(file => file.fileID) + } if (datasource.type === DataSourceType.NOTION) { + datasource_type = 'online_document' datasourceInfo.workspaceId = notionPages[0].workspace_id datasourceInfo.page = notionPages.map((page) => { const { workspace_id, ...rest } = page @@ -110,13 +114,13 @@ const TestRunPanel = () => { if (datasource.type === DataSourceProvider.fireCrawl || datasource.type === DataSourceProvider.jinaReader || datasource.type === DataSourceProvider.waterCrawl) { + datasource_type = 'website_crawl' datasourceInfo.jobId = websiteCrawlJobId datasourceInfo.result = websitePages } - // todo: TBD handleRun({ inputs: data, - datasource_type: datasource, + datasource_type, datasource_info: datasourceInfo, }) }, [datasource, fileList, handleRun, notionPages, websiteCrawlJobId, websitePages]) diff --git a/web/app/components/rag-pipeline/hooks/use-pipeline-run.ts b/web/app/components/rag-pipeline/hooks/use-pipeline-run.ts index 7a70d28e97..056dcbe909 100644 --- a/web/app/components/rag-pipeline/hooks/use-pipeline-run.ts +++ b/web/app/components/rag-pipeline/hooks/use-pipeline-run.ts @@ -128,7 +128,7 @@ export const usePipelineRun = () => { clientHeight, } = workflowContainer! - const url = `/rag/pipeline/${pipelineId}/workflows/draft/run` + const url = `/rag/pipelines/${pipelineId}/workflows/draft/run` const { setWorkflowRunningData, @@ -253,25 +253,25 @@ export const usePipelineRun = () => { }, ) }, [ - store, - workflowStore, - doSyncWorkflowDraft, - handleWorkflowStarted, - handleWorkflowFinished, - handleWorkflowFailed, - handleWorkflowNodeStarted, - handleWorkflowNodeFinished, - handleWorkflowNodeIterationStarted, - handleWorkflowNodeIterationNext, - handleWorkflowNodeIterationFinished, - handleWorkflowNodeLoopStarted, - handleWorkflowNodeLoopNext, - handleWorkflowNodeLoopFinished, - handleWorkflowNodeRetry, - handleWorkflowTextChunk, - handleWorkflowTextReplace, - handleWorkflowAgentLog, - ], + store, + workflowStore, + doSyncWorkflowDraft, + handleWorkflowStarted, + handleWorkflowFinished, + handleWorkflowFailed, + handleWorkflowNodeStarted, + handleWorkflowNodeFinished, + handleWorkflowNodeIterationStarted, + handleWorkflowNodeIterationNext, + handleWorkflowNodeIterationFinished, + handleWorkflowNodeLoopStarted, + handleWorkflowNodeLoopNext, + handleWorkflowNodeLoopFinished, + handleWorkflowNodeRetry, + handleWorkflowTextChunk, + handleWorkflowTextReplace, + handleWorkflowAgentLog, + ], ) const handleStopRun = useCallback((taskId: string) => { diff --git a/web/i18n/en-US/dataset-pipeline.ts b/web/i18n/en-US/dataset-pipeline.ts index 7a7dd8fb25..5665d42d0b 100644 --- a/web/i18n/en-US/dataset-pipeline.ts +++ b/web/i18n/en-US/dataset-pipeline.ts @@ -54,6 +54,10 @@ const translation = { dataSource: { localFiles: 'Local Files', }, + notion: { + title: 'Choose Notion Pages', + docTitle: 'Notion docs', + }, }, inputField: 'Input Field', inputFieldPanel: { diff --git a/web/i18n/zh-Hans/dataset-pipeline.ts b/web/i18n/zh-Hans/dataset-pipeline.ts index cedd736124..aae121bcaf 100644 --- a/web/i18n/zh-Hans/dataset-pipeline.ts +++ b/web/i18n/zh-Hans/dataset-pipeline.ts @@ -54,18 +54,22 @@ const translation = { dataSource: { localFiles: '本地文件', }, - inputField: '输入字段', - inputFieldPanel: { - title: '用户输入字段', - description: '用户输入字段用于定义和收集流水线执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。', - sharedInputs: { - title: '共享输入', - tooltip: '共享输入可被数据源中的所有下游节点使用。例如，在处理来自多个来源的文档时，delimiter（分隔符）和 maximum chunk length（最大分块长度）等变量可以统一应用。', - }, - addInputField: '添加输入字段', - editInputField: '编辑输入字段', + notion: { + title: '选择 Notion 页面', + docTitle: 'Notion 文档', }, }, + inputField: '输入字段', + inputFieldPanel: { + title: '用户输入字段', + description: '用户输入字段用于定义和收集流水线执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。', + sharedInputs: { + title: '共享输入', + tooltip: '共享输入可被数据源中的所有下游节点使用。例如，在处理来自多个来源的文档时，delimiter（分隔符）和 maximum chunk length（最大分块长度）等变量可以统一应用。', + }, + addInputField: '添加输入字段', + editInputField: '编辑输入字段', + }, } export default translation