refactor: Refactor useOnlineDocument hook

web/app/components/datasets/documents/create-from-pipeline/data-source/online-documents/index.tsx

@@ -24,7 +24,7 @@ type OnlineDocumentsProps = {
   setSearchValue: (value: string) => void
   currentWorkspaceId: string
   setCurrentWorkspaceId: (workspaceId: string) => void
-  PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>]
+  PagesMapAndSelectedPagesId: DataSourceNotionPageMap
   selectedPagesId: Set<string>
   setSelectedPagesId: (selectedPagesId: Set<string>) => void
 }
@@ -93,7 +93,7 @@ const OnlineDocuments = ({
   }, [setCurrentWorkspaceId])
 
   const handleSelectPages = useCallback((newSelectedPagesId: Set<string>) => {
-    const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId])
+    const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[pageId])
 
     setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
     onSelect(selectedPages)
@@ -101,7 +101,7 @@ const OnlineDocuments = ({
 
   const handlePreviewPage = useCallback((previewPageId: string) => {
     if (onPreview)
-      onPreview(PagesMapAndSelectedPagesId[0][previewPageId])
+      onPreview(PagesMapAndSelectedPagesId[previewPageId])
   }, [PagesMapAndSelectedPagesId, onPreview])
 
   const headerInfo = useMemo(() => {
@@ -138,10 +138,10 @@ const OnlineDocuments = ({
         <div className='overflow-hidden rounded-b-xl'>
           <PageSelector
             checkedIds={selectedPagesId}
-            disabledValue={PagesMapAndSelectedPagesId[2]}
+            disabledValue={new Set()}
             searchValue={searchValue}
             list={currentWorkspace?.pages || []}
-            pagesMap={PagesMapAndSelectedPagesId[0]}
+            pagesMap={PagesMapAndSelectedPagesId}
             onSelect={handleSelectPages}
             canPreview={!isInPipeline}
             previewPageId={previewPageId}

web/app/components/datasets/documents/create-from-pipeline/hooks.ts

@@ -135,15 +135,9 @@ export const useOnlineDocuments = () => {
   const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
   const [currentDocument, setCurrentDocument] = useState<NotionPage | undefined>()
 
-  const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
+  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
-    const selectedPagesId = new Set<string>()
-    const boundPagesId = new Set<string>()
     const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
       next.pages.forEach((page) => {
-        if (page.is_bound) {
-          selectedPagesId.add(page.page_id)
-          boundPagesId.add(page.page_id)
-        }
         prev[page.page_id] = {
           ...page,
           workspace_id: next.workspace_id,
@@ -152,9 +146,9 @@ export const useOnlineDocuments = () => {
 
       return prev
     }, {})
-    return [pagesMap, selectedPagesId, boundPagesId]
+    return pagesMap
   }, [documentsData])
-  const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(onlineDocuments.map(doc => doc.page_id) || [])]
+  const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])]
   const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
 
   const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -2,12 +2,13 @@
 import { useCallback, useMemo, useRef, useState } from 'react'
 import DataSourceOptions from './data-source-options'
 import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEstimateResponse } from '@/models/datasets'
-import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
+import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
 import { useProviderContextSelector } from '@/context/provider-context'
 import type { NotionPage } from '@/models/common'
-import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents'
+import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
-import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl'
+import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl'
+import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive'
 import Actions from './actions'
 import { useTranslation } from 'react-i18next'
 import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
@@ -27,7 +28,6 @@ import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, Publis
 import { DatasourceType } from '@/models/pipeline'
 import { TransferMethod } from '@/types/app'
 import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
-import OnlineDrive from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive'
 
 const CreateFormPipeline = () => {
   const { t } = useTranslation()

web/app/components/rag-pipeline/components/panel/test-run/hooks.ts

@@ -114,15 +114,9 @@ export const useOnlineDocuments = () => {
   const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
   const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
 
-  const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
+  const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
-    const selectedPagesId = new Set<string>()
-    const boundPagesId = new Set<string>()
     const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
       next.pages.forEach((page) => {
-        if (page.is_bound) {
-          selectedPagesId.add(page.page_id)
-          boundPagesId.add(page.page_id)
-        }
         prev[page.page_id] = {
           ...page,
           workspace_id: next.workspace_id,
@@ -131,9 +125,9 @@ export const useOnlineDocuments = () => {
 
       return prev
     }, {})
-    return [pagesMap, selectedPagesId, boundPagesId]
+    return pagesMap
   }, [documentsData])
-  const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(onlineDocuments.map(doc => doc.page_id) || [])]
+  const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])]
   const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
 
   const updateOnlineDocuments = (value: NotionPage[]) => {

web/app/components/rag-pipeline/components/panel/test-run/index.tsx

@@ -2,10 +2,11 @@ import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workf
 import { useCallback, useMemo, useState } from 'react'
 import { useLocalFile, useOnlineDocuments, useOnlineDrive, useTestRunSteps, useWebsiteCrawl } from './hooks'
 import DataSourceOptions from './data-source-options'
-import LocalFile from './data-source/local-file'
+import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
-import OnlineDocuments from './data-source/online-documents'
+import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
-import WebsiteCrawl from './data-source/website-crawl'
+import WebsiteCrawl from '@/app/components/datasets/documents/create-from-pipeline/data-source/website-crawl'
-import Actions from './data-source/actions'
+import OnlineDrive from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-drive'
+import Actions from './actions'
 import DocumentProcessing from './document-processing'
 import { useWorkflowRun } from '@/app/components/workflow/hooks'
 import type { Datasource } from './types'
@@ -14,7 +15,6 @@ import { TransferMethod } from '@/types/app'
 import CloseButton from './close-button'
 import Header from './header'
 import FooterTips from './footer-tips'
-import OnlineDrive from './data-source/online-drive'
 
 const TestRunPanel = () => {
   const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)