refactor: Notion component and add NotionPageSelector for improved page selection

web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx

@@ -1,43 +1,25 @@
-import { useDataSources } from '@/service/use-common'
-import { useCallback, useMemo } from 'react'
-import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
 import type { NotionPage } from '@/models/common'
-import NotionConnector from '@/app/components/base/notion-connector'
-import { useModalContextSelector } from '@/context/modal-context'
+import NotionPageSelector from './notion-page-selector'
 
 type NotionProps = {
+  nodeId: string
   notionPages: NotionPage[]
   updateNotionPages: (value: NotionPage[]) => void
 }
 
 const Notion = ({
+  nodeId,
   notionPages,
   updateNotionPages,
 }: NotionProps) => {
-  const { data: dataSources } = useDataSources()
-  const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal)
-
-  const hasConnection = useMemo(() => {
-    const notionDataSources = dataSources?.data.filter(item => item.provider === 'notion') || []
-    return notionDataSources.length > 0
-  }, [dataSources])
-
-  const handleConnect = useCallback(() => {
-    setShowAccountSettingModal({ payload: 'data-source' })
-  }, [setShowAccountSettingModal])
-
   return (
-    <>
-      {!hasConnection && <NotionConnector onSetting={handleConnect} />}
-      {hasConnection && (
-        <NotionPageSelector
-          value={notionPages.map(page => page.page_id)}
-          onSelect={updateNotionPages}
-          canPreview={false}
-          isInPipeline
-        />
-      )}
-    </>
+    <NotionPageSelector
+      nodeId={nodeId}
+      value={notionPages.map(page => page.page_id)}
+      onSelect={updateNotionPages}
+      canPreview={false}
+      isInPipeline
+    />
   )
 }
 

web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx

@@ -0,0 +1,146 @@
+import { useCallback, useEffect, useMemo, useState } from 'react'
+import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
+import SearchInput from '@/app/education-apply/search-input'
+import PageSelector from '@/app/components/base/notion-page-selector/page-selector'
+import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
+import Header from '@/app/components/datasets/create/website/base/header'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { useDatasourceNodeRun } from '@/service/use-pipeline'
+import { useTranslation } from 'react-i18next'
+
+type NotionPageSelectorProps = {
+  value?: string[]
+  onSelect: (selectedPages: NotionPage[]) => void
+  canPreview?: boolean
+  previewPageId?: string
+  onPreview?: (selectedPage: NotionPage) => void
+  isInPipeline?: boolean
+  nodeId: string
+}
+
+const NotionPageSelector = ({
+  value,
+  onSelect,
+  canPreview,
+  previewPageId,
+  onPreview,
+  isInPipeline = false,
+  nodeId,
+}: NotionPageSelectorProps) => {
+  const { t } = useTranslation()
+  const pipeline_id = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
+  const { mutateAsync: getNotionPages } = useDatasourceNodeRun()
+  const [notionData, setNotionData] = useState<DataSourceNotionWorkspace[]>()
+  const [searchValue, setSearchValue] = useState('')
+  const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
+
+  const getNotionData = useCallback(async () => {
+    if (pipeline_id) {
+      const notionData = await getNotionPages({
+        pipeline_id,
+        node_id: nodeId,
+        inputs: {},
+      }) as DataSourceNotionWorkspace[]
+      setNotionData(notionData)
+    }
+  }, [getNotionPages, nodeId, pipeline_id])
+
+  useEffect(() => {
+    getNotionData()
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
+  const notionWorkspaces = useMemo(() => {
+    return notionData || []
+  }, [notionData])
+
+  const firstWorkspaceId = notionWorkspaces[0]?.workspace_id
+  const currentWorkspace = notionWorkspaces.find(workspace => workspace.workspace_id === currentWorkspaceId)
+
+  const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
+    const selectedPagesId = new Set<string>()
+    const boundPagesId = new Set<string>()
+    const pagesMap = notionWorkspaces.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
+      next.pages.forEach((page) => {
+        if (page.is_bound) {
+          selectedPagesId.add(page.page_id)
+          boundPagesId.add(page.page_id)
+        }
+        prev[page.page_id] = {
+          ...page,
+          workspace_id: next.workspace_id,
+        }
+      })
+
+      return prev
+    }, {})
+    return [pagesMap, selectedPagesId, boundPagesId]
+  }, [notionWorkspaces])
+  const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])]
+  const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
+
+  const handleSearchValueChange = useCallback((value: string) => {
+    setSearchValue(value)
+  }, [])
+  const handleSelectWorkspace = useCallback((workspaceId: string) => {
+    setCurrentWorkspaceId(workspaceId)
+  }, [])
+  const handleSelectPages = (newSelectedPagesId: Set<string>) => {
+    const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId])
+
+    setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
+    onSelect(selectedPages)
+  }
+  const handlePreviewPage = (previewPageId: string) => {
+    if (onPreview)
+      onPreview(PagesMapAndSelectedPagesId[0][previewPageId])
+  }
+
+  useEffect(() => {
+    setCurrentWorkspaceId(firstWorkspaceId)
+  }, [firstWorkspaceId])
+
+  if (!notionData?.length)
+    return null
+
+  return (
+    <div className='flex flex-col gap-y-2'>
+      <Header
+        isInPipeline={isInPipeline}
+        title={t('datasetPipeline.testRun.notion.title')}
+        docTitle={t('datasetPipeline.testRun.notion.docTitle')}
+        docLink={'https://www.notion.so/docs'}
+      />
+      <div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
+        <div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
+          <div className='flex grow items-center gap-x-1'>
+            <WorkspaceSelector
+              value={currentWorkspaceId || firstWorkspaceId}
+              items={notionWorkspaces}
+              onSelect={handleSelectWorkspace}
+            />
+          </div>
+          <SearchInput
+            value={searchValue}
+            onChange={handleSearchValueChange}
+          />
+        </div>
+        <div className='overflow-hidden rounded-b-xl'>
+          <PageSelector
+            value={selectedPagesId}
+            disabledValue={PagesMapAndSelectedPagesId[2]}
+            searchValue={searchValue}
+            list={currentWorkspace?.pages || []}
+            pagesMap={PagesMapAndSelectedPagesId[0]}
+            onSelect={handleSelectPages}
+            canPreview={canPreview}
+            previewPageId={previewPageId}
+            onPreview={handlePreviewPage}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default NotionPageSelector

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -49,7 +49,7 @@ const TestRunPanel = () => {
   }, [fileList, isShowVectorSpaceFull])
 
   const nextBtnDisabled = useMemo(() => {
-    if (!datasource) return false
+    if (!datasource) return true
     if (datasource.type === DataSourceType.FILE)
       return nextDisabled
     if (datasource.type === DataSourceType.NOTION)
@@ -98,9 +98,13 @@ const TestRunPanel = () => {
     if (!datasource)
       return
     const datasourceInfo: Record<string, any> = {}
-    if (datasource.type === DataSourceType.FILE)
+    let datasource_type = ''
+    if (datasource.type === DataSourceType.FILE) {
+      datasource_type = 'local_file'
       datasourceInfo.fileId = fileList.map(file => file.fileID)
+    }
     if (datasource.type === DataSourceType.NOTION) {
+      datasource_type = 'online_document'
       datasourceInfo.workspaceId = notionPages[0].workspace_id
       datasourceInfo.page = notionPages.map((page) => {
         const { workspace_id, ...rest } = page
@@ -110,13 +114,13 @@ const TestRunPanel = () => {
     if (datasource.type === DataSourceProvider.fireCrawl
       || datasource.type === DataSourceProvider.jinaReader
       || datasource.type === DataSourceProvider.waterCrawl) {
+      datasource_type = 'website_crawl'
       datasourceInfo.jobId = websiteCrawlJobId
       datasourceInfo.result = websitePages
     }
-    // todo: TBD
     handleRun({
       inputs: data,
-      datasource_type: datasource,
+      datasource_type,
       datasource_info: datasourceInfo,
     })
   }, [datasource, fileList, handleRun, notionPages, websiteCrawlJobId, websitePages])

web/app/components/rag-pipeline/hooks/use-pipeline-run.ts

@@ -128,7 +128,7 @@ export const usePipelineRun = () => {
       clientHeight,
     } = workflowContainer!
 
-    const url = `/rag/pipeline/${pipelineId}/workflows/draft/run`
+    const url = `/rag/pipelines/${pipelineId}/workflows/draft/run`
 
     const {
       setWorkflowRunningData,
@@ -253,25 +253,25 @@ export const usePipelineRun = () => {
       },
     )
   }, [
-      store,
-      workflowStore,
-      doSyncWorkflowDraft,
-      handleWorkflowStarted,
-      handleWorkflowFinished,
-      handleWorkflowFailed,
-      handleWorkflowNodeStarted,
-      handleWorkflowNodeFinished,
-      handleWorkflowNodeIterationStarted,
-      handleWorkflowNodeIterationNext,
-      handleWorkflowNodeIterationFinished,
-      handleWorkflowNodeLoopStarted,
-      handleWorkflowNodeLoopNext,
-      handleWorkflowNodeLoopFinished,
-      handleWorkflowNodeRetry,
-      handleWorkflowTextChunk,
-      handleWorkflowTextReplace,
-      handleWorkflowAgentLog,
-    ],
+    store,
+    workflowStore,
+    doSyncWorkflowDraft,
+    handleWorkflowStarted,
+    handleWorkflowFinished,
+    handleWorkflowFailed,
+    handleWorkflowNodeStarted,
+    handleWorkflowNodeFinished,
+    handleWorkflowNodeIterationStarted,
+    handleWorkflowNodeIterationNext,
+    handleWorkflowNodeIterationFinished,
+    handleWorkflowNodeLoopStarted,
+    handleWorkflowNodeLoopNext,
+    handleWorkflowNodeLoopFinished,
+    handleWorkflowNodeRetry,
+    handleWorkflowTextChunk,
+    handleWorkflowTextReplace,
+    handleWorkflowAgentLog,
+  ],
   )
 
   const handleStopRun = useCallback((taskId: string) => {

web/i18n/en-US/dataset-pipeline.ts

@@ -54,6 +54,10 @@ const translation = {
     dataSource: {
       localFiles: 'Local Files',
     },
+    notion: {
+      title: 'Choose Notion Pages',
+      docTitle: 'Notion docs',
+    },
   },
   inputField: 'Input Field',
   inputFieldPanel: {

web/i18n/zh-Hans/dataset-pipeline.ts

@@ -54,18 +54,22 @@ const translation = {
     dataSource: {
       localFiles: '本地文件',
     },
-    inputField: '输入字段',
-    inputFieldPanel: {
-      title: '用户输入字段',
-      description: '用户输入字段用于定义和收集流水线执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。',
-      sharedInputs: {
-        title: '共享输入',
-        tooltip: '共享输入可被数据源中的所有下游节点使用。例如，在处理来自多个来源的文档时，delimiter（分隔符）和 maximum chunk length（最大分块长度）等变量可以统一应用。',
-      },
-      addInputField: '添加输入字段',
-      editInputField: '编辑输入字段',
+    notion: {
+      title: '选择 Notion 页面',
+      docTitle: 'Notion 文档',
     },
   },
+  inputField: '输入字段',
+  inputFieldPanel: {
+    title: '用户输入字段',
+    description: '用户输入字段用于定义和收集流水线执行过程中所需的变量，用户可以自定义字段类型，并灵活配置输入，以满足不同数据源或文档处理的需求。',
+    sharedInputs: {
+      title: '共享输入',
+      tooltip: '共享输入可被数据源中的所有下游节点使用。例如，在处理来自多个来源的文档时，delimiter（分隔符）和 maximum chunk length（最大分块长度）等变量可以统一应用。',
+    },
+    addInputField: '添加输入字段',
+    editInputField: '编辑输入字段',
+  },
 }
 
 export default translation