feat: datasource output schema

web/app/components/workflow/block-selector/data-sources.tsx

@@ -44,6 +44,7 @@ const DataSources = ({
       datasource_name: toolDefaultValue?.tool_name,
       datasource_label: toolDefaultValue?.tool_label,
       title: toolDefaultValue?.title,
+      output_schema: toolDefaultValue?.output_schema,
     })
   }, [onSelect])
   const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures)

web/app/components/workflow/block-selector/types.ts

@@ -46,6 +46,7 @@ export type DataSourceDefaultValue = {
   provider_name: string
   datasource_name: string
   datasource_label: string
+  output_schema?: Record<string, any>
 }
 
 export type ToolValue = {
@@ -86,6 +87,10 @@ export type DataSourceItem = {
         provider: string
       }
       parameters: any[]
+      output_schema?: {
+        type: string
+        properties: Record<string, any>
+      }
     }[]
   }
   is_authorized: boolean

web/app/components/workflow/block-selector/utils.ts

@@ -25,7 +25,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
         description: datasource.description,
         parameters: datasource.parameters,
         labels: [],
-        output_schema: {},
+        output_schema: datasource.output_schema || {},
       } as Tool
     }),
     credentialsSchema: dataSourceItem.declaration.credentials_schema || [],

web/app/components/workflow/nodes/_base/components/variable/utils.ts

@@ -517,7 +517,40 @@ const formatItem = (
     }
 
     case BlockEnum.DataSource: {
-      res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType, ragVars) || []
+      const payload = data as DataSourceNodeType
+      const baseVars = DataSourceNodeDefault.getOutputVars?.(payload, ragVars) || []
+      if (payload.output_schema?.properties) {
+        const dynamicOutputSchema: any[] = []
+        Object.keys(payload.output_schema.properties).forEach((outputKey) => {
+          const output = payload.output_schema!.properties[outputKey]
+          const dataType = output.type
+          dynamicOutputSchema.push({
+            variable: outputKey,
+            type: dataType === 'array'
+              ? `array[${output.items?.type.slice(0, 1).toLocaleLowerCase()}${output.items?.type.slice(1)}]`
+              : `${output.type.slice(0, 1).toLocaleLowerCase()}${output.type.slice(1)}`,
+            description: output.description,
+            children: output.type === 'object' ? {
+              schema: {
+                type: 'object',
+                properties: output.properties,
+              },
+            } : undefined,
+          })
+        })
+        res.vars = [
+          ...baseVars,
+          ...dynamicOutputSchema,
+          {
+            variable: 'output',
+            type: VarType.object,
+            children: dynamicOutputSchema,
+          },
+        ]
+      }
+      else {
+        res.vars = baseVars
+      }
       break
     }
 

web/app/components/workflow/nodes/data-source/hooks/use-config.ts

@@ -1,6 +1,7 @@
 import {
   useCallback,
   useEffect,
+  useMemo,
 } from 'react'
 import { useStoreApi } from 'reactflow'
 import { useNodeDataUpdate } from '@/app/components/workflow/hooks'
@@ -10,7 +11,7 @@ import type {
 } from '../types'
 import { DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE } from '../constants'
 
-export const useConfig = (id: string) => {
+export const useConfig = (id: string, dataSourceList?: any[]) => {
   const store = useStoreApi()
   const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
 
@@ -60,8 +61,59 @@ export const useConfig = (id: string) => {
     })
   }, [handleNodeDataUpdate, getNodeData])
 
+  const outputSchema = useMemo(() => {
+    const nodeData = getNodeData()
+    if (!nodeData?.data || !dataSourceList) return []
+
+    const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id)
+    const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name)
+    const output_schema = currentDataSourceItem?.output_schema
+
+    const res: any[] = []
+    if (!output_schema || !output_schema.properties)
+      return []
+
+    Object.keys(output_schema.properties).forEach((outputKey) => {
+      const output = output_schema.properties[outputKey]
+      const type = output.type
+      if (type === 'object') {
+        res.push({
+          name: outputKey,
+          value: output,
+        })
+      }
+      else {
+        res.push({
+          name: outputKey,
+          type: output.type === 'array'
+            ? `Array[${output.items?.type.slice(0, 1).toLocaleUpperCase()}${output.items?.type.slice(1)}]`
+            : `${output.type.slice(0, 1).toLocaleUpperCase()}${output.type.slice(1)}`,
+          description: output.description,
+        })
+      }
+    })
+    return res
+  }, [getNodeData, dataSourceList])
+
+  const hasObjectOutput = useMemo(() => {
+    const nodeData = getNodeData()
+    if (!nodeData?.data || !dataSourceList) return false
+
+    const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id)
+    const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name)
+    const output_schema = currentDataSourceItem?.output_schema
+
+    if (!output_schema || !output_schema.properties)
+      return false
+
+    const properties = output_schema.properties
+    return Object.keys(properties).some(key => properties[key].type === 'object')
+  }, [getNodeData, dataSourceList])
+
   return {
     handleFileExtensionsChange,
     handleParametersChange,
+    outputSchema,
+    hasObjectOutput,
   }
 }

web/app/components/workflow/nodes/data-source/panel.tsx

@@ -11,15 +11,13 @@ import {
   BoxGroupField,
 } from '@/app/components/workflow/nodes/_base/components/layout'
 import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
+import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
 import TagInput from '@/app/components/base/tag-input'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
 import { useConfig } from './hooks/use-config'
+import { Type } from '@/app/components/workflow/nodes/llm/types'
 import {
   COMMON_OUTPUT,
-  LOCAL_FILE_OUTPUT,
-  ONLINE_DOCUMENT_OUTPUT,
-  ONLINE_DRIVE_OUTPUT,
-  WEBSITE_CRAWL_OUTPUT,
 } from './constants'
 import { useStore } from '@/app/components/workflow/store'
 import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
@@ -38,13 +36,12 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const {
     handleFileExtensionsChange,
     handleParametersChange,
-  } = useConfig(id)
+    outputSchema,
+    hasObjectOutput,
+  } = useConfig(id, dataSourceList)
   const isLocalFile = provider_type === DataSourceClassification.localFile
-  const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl
-  const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument
-  const isOnlineDrive = provider_type === DataSourceClassification.onlineDrive
   const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id)
-  const currentDataSourceItem: any = currentDataSource?.tools.find(tool => tool.name === data.datasource_name)
+  const currentDataSourceItem: any = currentDataSource?.tools?.find((tool: any) => tool.name === data.datasource_name)
   const formSchemas = useMemo(() => {
     return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : []
   }, [currentDataSourceItem])
@@ -116,57 +113,34 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
               name={item.name}
               type={item.type}
               description={item.description}
+              isIndent={hasObjectOutput}
             />
           ))
         }
         {
-          isLocalFile && LOCAL_FILE_OUTPUT.map((item, index) => (
+          outputSchema.map(outputItem => (
-            <VarItem
+            <div key={outputItem.name}>
-              key={index}
+              {outputItem.value?.type === 'object' ? (
-              name={item.name}
+                <StructureOutputItem
-              type={item.type}
+                  rootClassName='code-sm-semibold text-text-secondary'
-              description={item.description}
+                  payload={{
-              subItems={item.subItems.map(item => ({
+                    schema: {
-                name: item.name,
+                      type: Type.object,
-                type: item.type,
+                      properties: {
-                description: item.description,
+                        [outputItem.name]: outputItem.value,
-              }))}
+                      },
-            />
+                      additionalProperties: false,
-          ))
+                    },
-        }
+                  }} />
-        {
+              ) : (
-          isWebsiteCrawl && WEBSITE_CRAWL_OUTPUT.map((item, index) => (
+                <VarItem
-            <VarItem
+                  name={outputItem.name}
-              key={index}
+                  type={outputItem.type.toLocaleLowerCase()}
-              name={item.name}
+                  description={outputItem.description}
-              type={item.type}
+                  isIndent={hasObjectOutput}
-              description={item.description}
+                />
-            />
+              )}
-          ))
+            </div>
-        }
-        {
-          isOnlineDocument && ONLINE_DOCUMENT_OUTPUT.map((item, index) => (
-            <VarItem
-              key={index}
-              name={item.name}
-              type={item.type}
-              description={item.description}
-            />
-          ))
-        }
-        {
-          isOnlineDrive && ONLINE_DRIVE_OUTPUT.map((item, index) => (
-            <VarItem
-              key={index}
-              name={item.name}
-              type={item.type}
-              description={item.description}
-              subItems={item.subItems.map(item => ({
-                name: item.name,
-                type: item.type,
-                description: item.description,
-              }))}
-            />
           ))
         }
       </OutputVars>

web/app/components/workflow/nodes/data-source/types.ts

@@ -27,6 +27,10 @@ export type DataSourceNodeType = CommonNodeType & {
   datasource_label: string
   datasource_parameters: ToolVarInputs
   datasource_configurations: Record<string, any>
+  output_schema?: {
+    type: string
+    properties: Record<string, any>
+  }
 }
 
 export type CustomRunFormProps = {