feat: enhance OnlineDocumentPreview with datasourceNodeId and implement preview functionality

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -324,7 +324,13 @@ const CreateFormPipeline = () => {
           <div className='h-full min-w-0 flex-1'>
             <div className='flex h-full flex-col pl-2 pt-2'>
               {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
-              {currentDocument && <OnlineDocumentPreview currentPage={currentDocument} hidePreview={hideOnlineDocumentPreview} />}
+              {currentDocument && (
+                <OnlineDocumentPreview
+                  datasourceNodeId={datasource!.nodeId}
+                  currentPage={currentDocument}
+                  hidePreview={hideOnlineDocumentPreview}
+                />
+              )}
               {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
             </div>
           </div>

web/app/components/datasets/documents/create-from-pipeline/preview/online-document-preview.tsx

@@ -1,48 +1,64 @@
 'use client'
-import React from 'react'
+import React, { useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import type { NotionPage } from '@/models/common'
-import { usePreviewNotionPage } from '@/service/knowledge/use-dataset'
 import { RiCloseLine } from '@remixicon/react'
 import { formatNumberAbbreviated } from '@/utils/format'
 import Loading from './loading'
 import { Notion } from '@/app/components/base/icons/src/public/common'
+import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
+import { usePreviewOnlineDocument } from '@/service/use-pipeline'
+import Toast from '@/app/components/base/toast'
+import { Markdown } from '@/app/components/base/markdown'
 
 type OnlineDocumentPreviewProps = {
   currentPage: NotionPage
+  datasourceNodeId: string
   hidePreview: () => void
 }
 
 const OnlineDocumentPreview = ({
   currentPage,
+  datasourceNodeId,
   hidePreview,
 }: OnlineDocumentPreviewProps) => {
   const { t } = useTranslation()
+  const [content, setContent] = useState('')
+  const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
+  const { mutateAsync: getOnlineDocumentContent, isPending } = usePreviewOnlineDocument()
 
-  // todo: replace with a generic hook for previewing online documents
-  const { data: notionPageData, isFetching } = usePreviewNotionPage({
-    workspaceID: currentPage.workspace_id,
-    pageID: currentPage.page_id,
-    pageType: currentPage.type,
-  })
+  useEffect(() => {
+    getOnlineDocumentContent({
+      workspaceID: currentPage.workspace_id,
+      pageID: currentPage.page_id,
+      pageType: currentPage.type,
+      pipelineId: pipelineId || '',
+      datasourceNodeId,
+    }, {
+      onSuccess(data) {
+        setContent(data.content)
+      },
+      onError(error) {
+        Toast.notify({
+          type: 'error',
+          message: error.message,
+        })
+      },
+    })
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
 
   return (
     <div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
       <div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
         <div className='flex grow flex-col gap-y-1'>
-          <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
+          <div className='system-2xs-semibold-uppercase text-text-accent'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
           <div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div>
-          <div className='system-xs-medium flex gap-x-1  text-text-tertiary'>
+          <div className='system-xs-medium flex items-center gap-x-1 text-text-tertiary'>
             <Notion className='size-3.5' />
+            <span>{currentPage.type}</span>
             <span>·</span>
-            <span>Notion Page</span>
-            <span>·</span>
-            {notionPageData && (
-              <>
-                <span>·</span>
-                <span>{`${formatNumberAbbreviated(notionPageData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
-              </>
-            )}
+            <span>{`${formatNumberAbbreviated(content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
           </div>
         </div>
         <button
@@ -53,14 +69,14 @@ const OnlineDocumentPreview = ({
           <RiCloseLine className='size-[18px]' />
         </button>
       </div>
-      {isFetching && (
+      {isPending && (
         <div className='grow'>
           <Loading />
         </div>
       )}
-      {!isFetching && notionPageData && (
+      {!isPending && content && (
         <div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
-          {notionPageData.content}
+          <Markdown content={content} />
         </div>
       )}
     </div>

web/models/datasets.ts

@@ -777,16 +777,6 @@ export type CreateDatasetResponse = {
   dataset_id: string
 }
 
-export type NotionPagePreviewRequest = {
-  workspaceID: string
-  pageID: string
-  pageType: string
-}
-
-export type NotionPagePreviewResponse = {
-  content: string
-}
-
 export type IndexingStatusBatchRequest = {
   datasetId: string
   batchId: string

web/models/pipeline.ts

@@ -254,3 +254,15 @@ export type PipelineExecutionLogResponse = {
   input_data: Record<string, any>
   datasource_node_id: string
 }
+
+export type OnlineDocumentPreviewRequest = {
+  workspaceID: string
+  pageID: string
+  pageType: string
+  pipelineId: string
+  datasourceNodeId: string
+}
+
+export type OnlineDocumentPreviewResponse = {
+  content: string
+}

web/service/knowledge/use-dataset.ts

@@ -6,8 +6,6 @@ import type {
   DatasetListRequest,
   IndexingStatusBatchRequest,
   IndexingStatusBatchResponse,
-  NotionPagePreviewRequest,
-  NotionPagePreviewResponse,
   ProcessRuleResponse,
   RelatedAppResponse,
 } from '@/models/datasets'
@@ -57,16 +55,6 @@ export const useDatasetRelatedApps = (datasetId: string) => {
   })
 }
 
-export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => {
-  const { workspaceID, pageID, pageType } = params
-  return useQuery({
-    queryKey: [NAME_SPACE, 'preview-notion-page'],
-    queryFn: () => get<NotionPagePreviewResponse>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`),
-    enabled: !!workspaceID && !!pageID && !!pageType,
-    staleTime: 0,
-  })
-}
-
 export const useIndexingStatusBatch = (
   params: IndexingStatusBatchRequest,
   mutationOptions: MutationOptions<IndexingStatusBatchResponse, Error> = {},

web/service/use-pipeline.ts

@@ -1,12 +1,15 @@
 import type { MutationOptions } from '@tanstack/react-query'
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
 import { del, get, patch, post } from './base'
+import { DatasourceType } from '@/models/pipeline'
 import type {
   DeleteTemplateResponse,
   ExportTemplateDSLResponse,
   ImportPipelineDSLConfirmResponse,
   ImportPipelineDSLRequest,
   ImportPipelineDSLResponse,
+  OnlineDocumentPreviewRequest,
+  OnlineDocumentPreviewResponse,
   PipelineCheckDependenciesResponse,
   PipelineExecutionLogRequest,
   PipelineExecutionLogResponse,
@@ -324,3 +327,25 @@ export const usePipelineExecutionLog = (params: PipelineExecutionLogRequest) =>
     staleTime: 0,
   })
 }
+
+export const usePreviewOnlineDocument = () => {
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'preview-online-document'],
+    mutationFn: (params: OnlineDocumentPreviewRequest) => {
+      const { pipelineId, datasourceNodeId, workspaceID, pageID, pageType } = params
+      return post<OnlineDocumentPreviewResponse>(
+        `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${datasourceNodeId}/preview`,
+        {
+          body: {
+            datasource_type: DatasourceType.onlineDocument,
+            inputs: {
+              workspace_id: workspaceID,
+              page_id: pageID,
+              type: pageType,
+            },
+          },
+        },
+      )
+    },
+  })
+}