From 2a1e1a804292bd7d3bed558e30153d5e6598d964 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 31 Jul 2025 16:19:10 +0800 Subject: [PATCH 1/3] feat: datasource output schema --- api/core/datasource/entities/datasource_entities.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/core/datasource/entities/datasource_entities.py b/api/core/datasource/entities/datasource_entities.py index f1179e4514..9ce580912f 100644 --- a/api/core/datasource/entities/datasource_entities.py +++ b/api/core/datasource/entities/datasource_entities.py @@ -125,6 +125,8 @@ class DatasourceEntity(BaseModel): identity: DatasourceIdentity parameters: list[DatasourceParameter] = Field(default_factory=list) description: I18nObject = Field(..., description="The label of the datasource") + output_schema: Optional[dict] = None + @field_validator("parameters", mode="before") @classmethod From facbe02cf70744458c55b3acd53c73f0d67bbcf7 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 31 Jul 2025 16:22:07 +0800 Subject: [PATCH 2/3] feat: datasource output schema --- .../workflow/block-selector/data-sources.tsx | 1 + .../workflow/block-selector/types.ts | 5 ++ .../workflow/block-selector/utils.ts | 2 +- .../nodes/_base/components/variable/utils.ts | 35 +++++++- .../nodes/data-source/hooks/use-config.ts | 54 +++++++++++- .../workflow/nodes/data-source/panel.tsx | 86 +++++++------------ .../workflow/nodes/data-source/types.ts | 4 + 7 files changed, 128 insertions(+), 59 deletions(-) diff --git a/web/app/components/workflow/block-selector/data-sources.tsx b/web/app/components/workflow/block-selector/data-sources.tsx index 6b2293afa5..0dffe9df99 100644 --- a/web/app/components/workflow/block-selector/data-sources.tsx +++ b/web/app/components/workflow/block-selector/data-sources.tsx @@ -44,6 +44,7 @@ const DataSources = ({ datasource_name: toolDefaultValue?.tool_name, datasource_label: toolDefaultValue?.tool_label, title: toolDefaultValue?.title, + output_schema: toolDefaultValue?.output_schema, }) }, [onSelect]) const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures) diff --git a/web/app/components/workflow/block-selector/types.ts b/web/app/components/workflow/block-selector/types.ts index c0770f8109..e152a54b7f 100644 --- a/web/app/components/workflow/block-selector/types.ts +++ b/web/app/components/workflow/block-selector/types.ts @@ -46,6 +46,7 @@ export type DataSourceDefaultValue = { provider_name: string datasource_name: string datasource_label: string + output_schema?: Record } export type ToolValue = { @@ -86,6 +87,10 @@ export type DataSourceItem = { provider: string } parameters: any[] + output_schema?: { + type: string + properties: Record + } }[] } is_authorized: boolean diff --git a/web/app/components/workflow/block-selector/utils.ts b/web/app/components/workflow/block-selector/utils.ts index 0a4bed25ff..c1eee2202c 100644 --- a/web/app/components/workflow/block-selector/utils.ts +++ b/web/app/components/workflow/block-selector/utils.ts @@ -25,7 +25,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => { description: datasource.description, parameters: datasource.parameters, labels: [], - output_schema: {}, + output_schema: datasource.output_schema || {}, } as Tool }), credentialsSchema: dataSourceItem.declaration.credentials_schema || [], diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index 43164118c4..4872d6714c 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -517,7 +517,40 @@ const formatItem = ( } case BlockEnum.DataSource: { - res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType, ragVars) || [] + const payload = data as DataSourceNodeType + const baseVars = DataSourceNodeDefault.getOutputVars?.(payload, ragVars) || [] + if (payload.output_schema?.properties) { + const dynamicOutputSchema: any[] = [] + Object.keys(payload.output_schema.properties).forEach((outputKey) => { + const output = payload.output_schema!.properties[outputKey] + const dataType = output.type + dynamicOutputSchema.push({ + variable: outputKey, + type: dataType === 'array' + ? `array[${output.items?.type.slice(0, 1).toLocaleLowerCase()}${output.items?.type.slice(1)}]` + : `${output.type.slice(0, 1).toLocaleLowerCase()}${output.type.slice(1)}`, + description: output.description, + children: output.type === 'object' ? { + schema: { + type: 'object', + properties: output.properties, + }, + } : undefined, + }) + }) + res.vars = [ + ...baseVars, + ...dynamicOutputSchema, + { + variable: 'output', + type: VarType.object, + children: dynamicOutputSchema, + }, + ] + } + else { + res.vars = baseVars + } break } diff --git a/web/app/components/workflow/nodes/data-source/hooks/use-config.ts b/web/app/components/workflow/nodes/data-source/hooks/use-config.ts index 9473ebe717..10153f622d 100644 --- a/web/app/components/workflow/nodes/data-source/hooks/use-config.ts +++ b/web/app/components/workflow/nodes/data-source/hooks/use-config.ts @@ -1,6 +1,7 @@ import { useCallback, useEffect, + useMemo, } from 'react' import { useStoreApi } from 'reactflow' import { useNodeDataUpdate } from '@/app/components/workflow/hooks' @@ -10,7 +11,7 @@ import type { } from '../types' import { DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE } from '../constants' -export const useConfig = (id: string) => { +export const useConfig = (id: string, dataSourceList?: any[]) => { const store = useStoreApi() const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate() @@ -60,8 +61,59 @@ export const useConfig = (id: string) => { }) }, [handleNodeDataUpdate, getNodeData]) + const outputSchema = useMemo(() => { + const nodeData = getNodeData() + if (!nodeData?.data || !dataSourceList) return [] + + const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id) + const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name) + const output_schema = currentDataSourceItem?.output_schema + + const res: any[] = [] + if (!output_schema || !output_schema.properties) + return [] + + Object.keys(output_schema.properties).forEach((outputKey) => { + const output = output_schema.properties[outputKey] + const type = output.type + if (type === 'object') { + res.push({ + name: outputKey, + value: output, + }) + } + else { + res.push({ + name: outputKey, + type: output.type === 'array' + ? `Array[${output.items?.type.slice(0, 1).toLocaleUpperCase()}${output.items?.type.slice(1)}]` + : `${output.type.slice(0, 1).toLocaleUpperCase()}${output.type.slice(1)}`, + description: output.description, + }) + } + }) + return res + }, [getNodeData, dataSourceList]) + + const hasObjectOutput = useMemo(() => { + const nodeData = getNodeData() + if (!nodeData?.data || !dataSourceList) return false + + const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id) + const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name) + const output_schema = currentDataSourceItem?.output_schema + + if (!output_schema || !output_schema.properties) + return false + + const properties = output_schema.properties + return Object.keys(properties).some(key => properties[key].type === 'object') + }, [getNodeData, dataSourceList]) + return { handleFileExtensionsChange, handleParametersChange, + outputSchema, + hasObjectOutput, } } diff --git a/web/app/components/workflow/nodes/data-source/panel.tsx b/web/app/components/workflow/nodes/data-source/panel.tsx index 29390b10a9..776ec7b6da 100644 --- a/web/app/components/workflow/nodes/data-source/panel.tsx +++ b/web/app/components/workflow/nodes/data-source/panel.tsx @@ -11,15 +11,13 @@ import { BoxGroupField, } from '@/app/components/workflow/nodes/_base/components/layout' import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars' +import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show' import TagInput from '@/app/components/base/tag-input' import { useNodesReadOnly } from '@/app/components/workflow/hooks' import { useConfig } from './hooks/use-config' +import { Type } from '@/app/components/workflow/nodes/llm/types' import { COMMON_OUTPUT, - LOCAL_FILE_OUTPUT, - ONLINE_DOCUMENT_OUTPUT, - ONLINE_DRIVE_OUTPUT, - WEBSITE_CRAWL_OUTPUT, } from './constants' import { useStore } from '@/app/components/workflow/store' import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema' @@ -38,13 +36,12 @@ const Panel: FC> = ({ id, data }) => { const { handleFileExtensionsChange, handleParametersChange, - } = useConfig(id) + outputSchema, + hasObjectOutput, + } = useConfig(id, dataSourceList) const isLocalFile = provider_type === DataSourceClassification.localFile - const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl - const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument - const isOnlineDrive = provider_type === DataSourceClassification.onlineDrive const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id) - const currentDataSourceItem: any = currentDataSource?.tools.find(tool => tool.name === data.datasource_name) + const currentDataSourceItem: any = currentDataSource?.tools?.find((tool: any) => tool.name === data.datasource_name) const formSchemas = useMemo(() => { return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : [] }, [currentDataSourceItem]) @@ -116,57 +113,34 @@ const Panel: FC> = ({ id, data }) => { name={item.name} type={item.type} description={item.description} + isIndent={hasObjectOutput} /> )) } { - isLocalFile && LOCAL_FILE_OUTPUT.map((item, index) => ( - ({ - name: item.name, - type: item.type, - description: item.description, - }))} - /> - )) - } - { - isWebsiteCrawl && WEBSITE_CRAWL_OUTPUT.map((item, index) => ( - - )) - } - { - isOnlineDocument && ONLINE_DOCUMENT_OUTPUT.map((item, index) => ( - - )) - } - { - isOnlineDrive && ONLINE_DRIVE_OUTPUT.map((item, index) => ( - ({ - name: item.name, - type: item.type, - description: item.description, - }))} - /> + outputSchema.map(outputItem => ( +
+ {outputItem.value?.type === 'object' ? ( + + ) : ( + + )} +
)) } diff --git a/web/app/components/workflow/nodes/data-source/types.ts b/web/app/components/workflow/nodes/data-source/types.ts index 5d95c0f3c6..6d586734bd 100644 --- a/web/app/components/workflow/nodes/data-source/types.ts +++ b/web/app/components/workflow/nodes/data-source/types.ts @@ -27,6 +27,10 @@ export type DataSourceNodeType = CommonNodeType & { datasource_label: string datasource_parameters: ToolVarInputs datasource_configurations: Record + output_schema?: { + type: string + properties: Record + } } export type CustomRunFormProps = { From 29da2e5c19f9f3f77f2562f83dff46188299a8e4 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 1 Aug 2025 15:09:29 +0800 Subject: [PATCH 3/3] feat: enhance output schema descriptions and remove unused constants --- .../nodes/_base/components/variable/utils.ts | 8 ++++--- .../workflow/nodes/data-source/constants.ts | 16 +++++++------- .../workflow/nodes/data-source/default.ts | 21 ------------------- .../components/workflow/nodes/llm/types.ts | 1 + 4 files changed, 14 insertions(+), 32 deletions(-) diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index 4872d6714c..d09ffd8cf5 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -523,17 +523,19 @@ const formatItem = ( const dynamicOutputSchema: any[] = [] Object.keys(payload.output_schema.properties).forEach((outputKey) => { const output = payload.output_schema!.properties[outputKey] - const dataType = output.type + const dataType = output?.properties?.dify_builtin_type ? output.properties.dify_builtin_type.enum[0] : output.type dynamicOutputSchema.push({ variable: outputKey, type: dataType === 'array' ? `array[${output.items?.type.slice(0, 1).toLocaleLowerCase()}${output.items?.type.slice(1)}]` - : `${output.type.slice(0, 1).toLocaleLowerCase()}${output.type.slice(1)}`, + : `${dataType.slice(0, 1).toLocaleLowerCase()}${dataType.slice(1)}`, description: output.description, children: output.type === 'object' ? { schema: { type: 'object', - properties: output.properties, + properties: Object.fromEntries( + Object.entries(output.properties).filter(([key]) => key !== 'dify_builtin_type'), + ), }, } : undefined, }) diff --git a/web/app/components/workflow/nodes/data-source/constants.ts b/web/app/components/workflow/nodes/data-source/constants.ts index 70bc7b0fb5..f0f40e1af0 100644 --- a/web/app/components/workflow/nodes/data-source/constants.ts +++ b/web/app/components/workflow/nodes/data-source/constants.ts @@ -39,42 +39,42 @@ export const LOCAL_FILE_OUTPUT = [ { name: 'name', type: VarType.string, - description: '', + description: 'file name', }, { name: 'size', type: VarType.number, - description: '', + description: 'file size', }, { name: 'type', type: VarType.string, - description: '', + description: 'file type', }, { name: 'extension', type: VarType.string, - description: '', + description: 'file extension', }, { name: 'mime_type', type: VarType.string, - description: '', + description: 'file mime type', }, { name: 'transfer_method', type: VarType.string, - description: '', + description: 'file transfer method', }, { name: 'url', type: VarType.string, - description: '', + description: 'file url', }, { name: 'related_id', type: VarType.string, - description: '', + description: 'file related id', }, ], }, diff --git a/web/app/components/workflow/nodes/data-source/default.ts b/web/app/components/workflow/nodes/data-source/default.ts index 835e4a1dfa..f3a9e3e4cc 100644 --- a/web/app/components/workflow/nodes/data-source/default.ts +++ b/web/app/components/workflow/nodes/data-source/default.ts @@ -6,9 +6,6 @@ import { BlockEnum } from '@/app/components/workflow/types' import { COMMON_OUTPUT, LOCAL_FILE_OUTPUT, - ONLINE_DOCUMENT_OUTPUT, - ONLINE_DRIVE_OUTPUT, - WEBSITE_CRAWL_OUTPUT, } from './constants' import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' @@ -61,9 +58,6 @@ const nodeDefault: NodeDefault = { provider_type, } = payload const isLocalFile = provider_type === DataSourceClassification.localFile - const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl - const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument - const isOnlineDrive = provider_type === DataSourceClassification.onlineDrive return [ ...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })), ...( @@ -71,21 +65,6 @@ const nodeDefault: NodeDefault = { ? LOCAL_FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) : [] ), - ...( - isWebsiteCrawl - ? WEBSITE_CRAWL_OUTPUT.map(item => ({ variable: item.name, type: item.type })) - : [] - ), - ...( - isOnlineDocument - ? ONLINE_DOCUMENT_OUTPUT.map(item => ({ variable: item.name, type: item.type })) - : [] - ), - ...( - isOnlineDrive - ? ONLINE_DRIVE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) - : [] - ), ...ragVars, ] }, diff --git a/web/app/components/workflow/nodes/llm/types.ts b/web/app/components/workflow/nodes/llm/types.ts index c1d33a2e92..8314592e6a 100644 --- a/web/app/components/workflow/nodes/llm/types.ts +++ b/web/app/components/workflow/nodes/llm/types.ts @@ -28,6 +28,7 @@ export enum Type { arrayString = 'array[string]', arrayNumber = 'array[number]', arrayObject = 'array[object]', + file = 'file', } export enum ArrayType {