fix: Enhance data source handling by adding error response type and updating local file and online document slices

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/local-file.ts

@@ -1,11 +1,12 @@
 import type { StateCreator } from 'zustand'
-import type { FileItem } from '@/models/datasets'
+import type { DocumentItem, FileItem } from '@/models/datasets'
 
 export type LocalFileSliceShape = {
   localFileList: FileItem[]
   setLocalFileList: (fileList: FileItem[]) => void
   currentLocalFile: File | undefined
   setCurrentLocalFile: (file: File | undefined) => void
+  previewLocalFileRef: React.MutableRefObject<DocumentItem | undefined>
 }
 
 export const createLocalFileSlice: StateCreator<LocalFileSliceShape> = (set) => {
@@ -18,5 +19,6 @@ export const createLocalFileSlice: StateCreator<LocalFileSliceShape> = (set) =>
     setCurrentLocalFile: (file: File | undefined) => set(() => ({
       currentLocalFile: file,
     })),
+    previewLocalFileRef: { current: undefined },
   })
 }

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-document.ts

@@ -16,7 +16,7 @@ export type OnlineDocumentSliceShape = {
   setSelectedPagesId: (selectedPagesId: Set<string>) => void
 }
 
-export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set, get) => {
+export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set) => {
   return ({
     documentData: [],
     setDocumentData: (documentData: DataSourceNotionWorkspace[]) => set(() => ({
@@ -38,7 +38,7 @@ export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> =
     setCurrentDocument: (document: NotionPage | undefined) => set(() => ({
       currentDocument: document,
     })),
-    selectedPagesId: new Set([...get().onlineDocuments.map(doc => doc.page_id)]),
+    selectedPagesId: new Set(),
     setSelectedPagesId: (selectedPagesId: Set<string>) => set(() => ({
       selectedPagesId,
     })),

web/app/components/datasets/documents/create-from-pipeline/data-source/store/slices/online-drive.ts

@@ -1,5 +1,5 @@
 import type { StateCreator } from 'zustand'
-import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline'
+import type { OnlineDriveFile } from '@/models/pipeline'
 
 export type OnlineDriveSliceShape = {
   prefix: string[]
@@ -32,11 +32,7 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
     setSelectedFileList: (selectedFileList: string[]) => set(() => ({
       selectedFileList,
     })),
-    fileList: [{
-      key: 'Bucket_1',
-      size: 1024, // unit bytes
-      type: OnlineDriveFileType.bucket,
-    }],
+    fileList: [],
     setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
       fileList,
     })),

web/service/base.ts

@@ -27,6 +27,7 @@ import { ContentType, base, baseOptions, getAccessToken } from './fetch'
 import { asyncRunSafe } from '@/utils'
 import type {
   DataSourceNodeCompletedResponse,
+  DataSourceNodeErrorResponse,
   DataSourceNodeProcessingResponse,
 } from '@/types/pipeline'
 const TIME_OUT = 100000
@@ -69,6 +70,7 @@ export type IOnAgentLog = (agentLog: AgentLogResponse) => void
 
 export type IOnDataSourceNodeProcessing = (dataSourceNodeProcessing: DataSourceNodeProcessingResponse) => void
 export type IOnDataSourceNodeCompleted = (dataSourceNodeCompleted: DataSourceNodeCompletedResponse) => void
+export type IOnDataSourceNodeError = (dataSourceNodeError: DataSourceNodeErrorResponse) => void
 
 export type IOtherOptions = {
   isPublicAPI?: boolean
@@ -108,6 +110,7 @@ export type IOtherOptions = {
   // Pipeline data source node run
   onDataSourceNodeProcessing?: IOnDataSourceNodeProcessing
   onDataSourceNodeCompleted?: IOnDataSourceNodeCompleted
+  onDataSourceNodeError?: IOnDataSourceNodeError
 }
 
 function unicodeToChar(text: string) {
@@ -165,6 +168,7 @@ const handleStream = (
   onAgentLog?: IOnAgentLog,
   onDataSourceNodeProcessing?: IOnDataSourceNodeProcessing,
   onDataSourceNodeCompleted?: IOnDataSourceNodeCompleted,
+  onDataSourceNodeError?: IOnDataSourceNodeError,
 ) => {
   if (!response.ok)
     throw new Error('Network response was not ok')
@@ -289,6 +293,9 @@ const handleStream = (
             else if (bufferObj.event === 'datasource_completed') {
               onDataSourceNodeCompleted?.(bufferObj as DataSourceNodeCompletedResponse)
             }
+            else if (bufferObj.event === 'datasource_error') {
+              onDataSourceNodeError?.(bufferObj as DataSourceNodeErrorResponse)
+            }
             else {
               console.warn(`Unknown event: ${bufferObj.event}`, bufferObj)
             }
@@ -387,6 +394,7 @@ export const ssePost = async (
     onLoopFinish,
     onDataSourceNodeProcessing,
     onDataSourceNodeCompleted,
+    onDataSourceNodeError,
   } = otherOptions
   const abortController = new AbortController()
 
@@ -486,6 +494,7 @@ export const ssePost = async (
         onAgentLog,
         onDataSourceNodeProcessing,
         onDataSourceNodeCompleted,
+        onDataSourceNodeError,
       )
     }).catch((e) => {
       if (e.toString() !== 'AbortError: The user aborted a request.' && !e.toString().errorMessage.includes('TypeError: Cannot assign to read only property'))

web/types/pipeline.tsx

@@ -15,3 +15,8 @@ export type DataSourceNodeCompletedResponse = {
   data: any
   time_consuming?: number
 }
+
+export type DataSourceNodeErrorResponse = {
+  event: 'datasource_error'
+  error: string
+}