opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

This commit is contained in:
jyong 2025-08-06 15:39:27 +08:00
parent 1567b615dc 29da2e5c19
commit 218e778099
11 changed files with 141 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/core/datasource/entities/datasource_entities.py
View File

@ -125,6 +125,8 @@ class DatasourceEntity(BaseModel):
identity: DatasourceIdentity
parameters: list[DatasourceParameter] = Field(default_factory=list)
description: I18nObject = Field(..., description="The label of the datasource")
output_schema: Optional[dict] = None
@field_validator("parameters", mode="before")
@classmethod

1
web/app/components/workflow/block-selector/data-sources.tsx
View File

@ -44,6 +44,7 @@ const DataSources = ({
datasource_name: toolDefaultValue?.tool_name,
datasource_label: toolDefaultValue?.tool_label,
title: toolDefaultValue?.title,
output_schema: toolDefaultValue?.output_schema,
})
}, [onSelect])
const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures)

5
web/app/components/workflow/block-selector/types.ts
View File

@ -46,6 +46,7 @@ export type DataSourceDefaultValue = {
provider_name: string
datasource_name: string
datasource_label: string
output_schema?: Record<string, any>
}
export type ToolValue = {
@ -86,6 +87,10 @@ export type DataSourceItem = {
provider: string
}
parameters: any[]
output_schema?: {
type: string
properties: Record<string, any>
}
}[]
}
is_authorized: boolean

2
web/app/components/workflow/block-selector/utils.ts
View File

@ -25,7 +25,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
description: datasource.description,
parameters: datasource.parameters,
labels: [],
output_schema: {},
output_schema: datasource.output_schema || {},
} as Tool
}),
credentialsSchema: dataSourceItem.declaration.credentials_schema || [],

37
web/app/components/workflow/nodes/_base/components/variable/utils.ts
View File

@ -517,7 +517,42 @@ const formatItem = (
}
case BlockEnum.DataSource: {
res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType, ragVars) || []
const payload = data as DataSourceNodeType
const baseVars = DataSourceNodeDefault.getOutputVars?.(payload, ragVars) || []
if (payload.output_schema?.properties) {
const dynamicOutputSchema: any[] = []
Object.keys(payload.output_schema.properties).forEach((outputKey) => {
const output = payload.output_schema!.properties[outputKey]
const dataType = output?.properties?.dify_builtin_type ? output.properties.dify_builtin_type.enum[0] : output.type
dynamicOutputSchema.push({
variable: outputKey,
type: dataType === 'array'
? `array[${output.items?.type.slice(0, 1).toLocaleLowerCase()}${output.items?.type.slice(1)}]`
: `${dataType.slice(0, 1).toLocaleLowerCase()}${dataType.slice(1)}`,
description: output.description,
children: output.type === 'object' ? {
schema: {
type: 'object',
properties: Object.fromEntries(
Object.entries(output.properties).filter(([key]) => key !== 'dify_builtin_type'),
),
},
} : undefined,
})
})
res.vars = [
...baseVars,
...dynamicOutputSchema,
{
variable: 'output',
type: VarType.object,
children: dynamicOutputSchema,
},
]
}
else {
res.vars = baseVars
}
break
}

16
web/app/components/workflow/nodes/data-source/constants.ts
View File

@ -39,42 +39,42 @@ export const LOCAL_FILE_OUTPUT = [
{
name: 'name',
type: VarType.string,
description: '',
description: 'file name',
},
{
name: 'size',
type: VarType.number,
description: '',
description: 'file size',
},
{
name: 'type',
type: VarType.string,
description: '',
description: 'file type',
},
{
name: 'extension',
type: VarType.string,
description: '',
description: 'file extension',
},
{
name: 'mime_type',
type: VarType.string,
description: '',
description: 'file mime type',
},
{
name: 'transfer_method',
type: VarType.string,
description: '',
description: 'file transfer method',
},
{
name: 'url',
type: VarType.string,
description: '',
description: 'file url',
},
{
name: 'related_id',
type: VarType.string,
description: '',
description: 'file related id',
},
],
},

21
web/app/components/workflow/nodes/data-source/default.ts
View File

@ -6,9 +6,6 @@ import { BlockEnum } from '@/app/components/workflow/types'
import {
COMMON_OUTPUT,
LOCAL_FILE_OUTPUT,
ONLINE_DOCUMENT_OUTPUT,
ONLINE_DRIVE_OUTPUT,
WEBSITE_CRAWL_OUTPUT,
} from './constants'
import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types'
@ -61,9 +58,6 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
provider_type,
} = payload
const isLocalFile = provider_type === DataSourceClassification.localFile
const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl
const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument
const isOnlineDrive = provider_type === DataSourceClassification.onlineDrive
return [
...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })),
...(
@ -71,21 +65,6 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
? LOCAL_FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: []
),
...(
isWebsiteCrawl
? WEBSITE_CRAWL_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: []
),
...(
isOnlineDocument
? ONLINE_DOCUMENT_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: []
),
...(
isOnlineDrive
? ONLINE_DRIVE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: []
),
...ragVars,
]
},

54
web/app/components/workflow/nodes/data-source/hooks/use-config.ts
View File

@ -1,6 +1,7 @@
import {
useCallback,
useEffect,
useMemo,
} from 'react'
import { useStoreApi } from 'reactflow'
import { useNodeDataUpdate } from '@/app/components/workflow/hooks'
@ -10,7 +11,7 @@ import type {
} from '../types'
import { DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE } from '../constants'
export const useConfig = (id: string) => {
export const useConfig = (id: string, dataSourceList?: any[]) => {
const store = useStoreApi()
const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
@ -60,8 +61,59 @@ export const useConfig = (id: string) => {
})
}, [handleNodeDataUpdate, getNodeData])
const outputSchema = useMemo(() => {
const nodeData = getNodeData()
if (!nodeData?.data || !dataSourceList) return []
const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id)
const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name)
const output_schema = currentDataSourceItem?.output_schema
const res: any[] = []
if (!output_schema || !output_schema.properties)
return []
Object.keys(output_schema.properties).forEach((outputKey) => {
const output = output_schema.properties[outputKey]
const type = output.type
if (type === 'object') {
res.push({
name: outputKey,
value: output,
})
}
else {
res.push({
name: outputKey,
type: output.type === 'array'
? `Array[${output.items?.type.slice(0, 1).toLocaleUpperCase()}${output.items?.type.slice(1)}]`
: `${output.type.slice(0, 1).toLocaleUpperCase()}${output.type.slice(1)}`,
description: output.description,
})
}
})
return res
}, [getNodeData, dataSourceList])
const hasObjectOutput = useMemo(() => {
const nodeData = getNodeData()
if (!nodeData?.data || !dataSourceList) return false
const currentDataSource = dataSourceList.find((ds: any) => ds.plugin_id === nodeData.data.plugin_id)
const currentDataSourceItem = currentDataSource?.tools?.find((tool: any) => tool.name === nodeData.data.datasource_name)
const output_schema = currentDataSourceItem?.output_schema
if (!output_schema || !output_schema.properties)
return false
const properties = output_schema.properties
return Object.keys(properties).some(key => properties[key].type === 'object')
}, [getNodeData, dataSourceList])
return {
handleFileExtensionsChange,
handleParametersChange,
outputSchema,
hasObjectOutput,
}
}

86
web/app/components/workflow/nodes/data-source/panel.tsx
View File

@ -11,15 +11,13 @@ import {
BoxGroupField,
} from '@/app/components/workflow/nodes/_base/components/layout'
import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
import StructureOutputItem from '@/app/components/workflow/nodes/_base/components/variable/object-child-tree-panel/show'
import TagInput from '@/app/components/base/tag-input'
import { useNodesReadOnly } from '@/app/components/workflow/hooks'
import { useConfig } from './hooks/use-config'
import { Type } from '@/app/components/workflow/nodes/llm/types'
import {
COMMON_OUTPUT,
LOCAL_FILE_OUTPUT,
ONLINE_DOCUMENT_OUTPUT,
ONLINE_DRIVE_OUTPUT,
WEBSITE_CRAWL_OUTPUT,
} from './constants'
import { useStore } from '@/app/components/workflow/store'
import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
@ -38,13 +36,12 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
const {
handleFileExtensionsChange,
handleParametersChange,
} = useConfig(id)
outputSchema,
hasObjectOutput,
} = useConfig(id, dataSourceList)
const isLocalFile = provider_type === DataSourceClassification.localFile
const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl
const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument
const isOnlineDrive = provider_type === DataSourceClassification.onlineDrive
const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id)
const currentDataSourceItem: any = currentDataSource?.tools.find(tool => tool.name === data.datasource_name)
const currentDataSourceItem: any = currentDataSource?.tools?.find((tool: any) => tool.name === data.datasource_name)
const formSchemas = useMemo(() => {
return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : []
}, [currentDataSourceItem])
@ -116,57 +113,34 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
name={item.name}
type={item.type}
description={item.description}
isIndent={hasObjectOutput}
/>
))
}
{
isLocalFile && LOCAL_FILE_OUTPUT.map((item, index) => (
<VarItem
key={index}
name={item.name}
type={item.type}
description={item.description}
subItems={item.subItems.map(item => ({
name: item.name,
type: item.type,
description: item.description,
}))}
/>
))
}
{
isWebsiteCrawl && WEBSITE_CRAWL_OUTPUT.map((item, index) => (
<VarItem
key={index}
name={item.name}
type={item.type}
description={item.description}
/>
))
}
{
isOnlineDocument && ONLINE_DOCUMENT_OUTPUT.map((item, index) => (
<VarItem
key={index}
name={item.name}
type={item.type}
description={item.description}
/>
))
}
{
isOnlineDrive && ONLINE_DRIVE_OUTPUT.map((item, index) => (
<VarItem
key={index}
name={item.name}
type={item.type}
description={item.description}
subItems={item.subItems.map(item => ({
name: item.name,
type: item.type,
description: item.description,
}))}
/>
outputSchema.map(outputItem => (
<div key={outputItem.name}>
{outputItem.value?.type === 'object' ? (
<StructureOutputItem
rootClassName='code-sm-semibold text-text-secondary'
payload={{
schema: {
type: Type.object,
properties: {
[outputItem.name]: outputItem.value,
},
additionalProperties: false,
},
}} />
) : (
<VarItem
name={outputItem.name}
type={outputItem.type.toLocaleLowerCase()}
description={outputItem.description}
isIndent={hasObjectOutput}
/>
)}
</div>
))
}
</OutputVars>

4
web/app/components/workflow/nodes/data-source/types.ts
View File

@ -27,6 +27,10 @@ export type DataSourceNodeType = CommonNodeType & {
datasource_label: string
datasource_parameters: ToolVarInputs
datasource_configurations: Record<string, any>
output_schema?: {
type: string
properties: Record<string, any>
}
}
export type CustomRunFormProps = {

1
web/app/components/workflow/nodes/llm/types.ts
View File

@ -28,6 +28,7 @@ export enum Type {
arrayString = 'array[string]',
arrayNumber = 'array[number]',
arrayObject = 'array[object]',
file = 'file',
}
export enum ArrayType {