opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/rag-pipeline' into deploy/rag-dev

This commit is contained in:
zxhlyh 2025-06-04 18:10:03 +08:00
parent eb84134706 a7fa5044e3
commit f7f7952951
23 changed files with 163 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
web/app/components/base/form/components/field/input-type-select/hooks.tsx
View File

@ -12,7 +12,7 @@ import {
} from '@remixicon/react'
const i18nFileTypeMap: Record<string, string> = {
'number-input': 'number',
'number': 'number',
'file': 'single-file',
'file-list': 'multi-files',
}

2
web/app/components/base/form/components/field/input-type-select/types.tsx
View File

@ -4,7 +4,7 @@ import { z } from 'zod'
export const InputTypeEnum = z.enum([
'text-input',
'paragraph',
'number-input',
'number',
'select',
'checkbox',
'file',

1
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -51,7 +51,6 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
return {
nodeId: node.id,
type: node.data.provider_type as DatasourceType,
variables: node.data.variables || [],
description: node.data.desc || '',
docTitle: '', // todo: Add docTitle and docLink if needed, or remove these properties if not used
docLink: '',

2
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -274,7 +274,7 @@ const CreateFormPipeline = () => {
{datasource?.type === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
variables={[]} // todo: replace with actual variables if needed
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',

20
web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx
View File

@ -1,7 +1,18 @@
import React from 'react'
import { useTranslation } from 'react-i18next'
import DataSourceOptions from '../../panel/test-run/data-source-options'
import Form from './form'
import type { Datasource } from '../../panel/test-run/types'
const DataSource = () => {
type DatasourceProps = {
onSelect: (dataSource: Datasource) => void
datasourceNodeId: string
}
const DataSource = ({
onSelect: setDatasource,
datasourceNodeId,
}: DatasourceProps) => {
const { t } = useTranslation()
return (
@ -9,6 +20,13 @@ const DataSource = () => {
<div className='system-sm-semibold-uppercase px-4 pt-2 text-text-secondary'>
{t('datasetPipeline.inputFieldPanel.preview.stepOneTitle')}
</div>
<div className='px-4 py-2'>
<DataSourceOptions
onSelect={setDatasource}
datasourceNodeId={datasourceNodeId}
/>
</div>
<Form variables={[]} />
</div>
)
}

41
web/app/components/rag-pipeline/components/input-field/preview/form.tsx Normal file
View File

@ -0,0 +1,41 @@
import { useAppForm } from '@/app/components/base/form'
import BaseField from '@/app/components/base/form/form-scenarios/base/field'
import type { RAGPipelineVariables } from '@/models/pipeline'
import { useConfigurations, useInitialData } from '../../panel/test-run/data-source/website-crawl/base/options/hooks'
type FormProps = {
variables: RAGPipelineVariables
}
const Form = ({
variables,
}: FormProps) => {
const initialData = useInitialData(variables)
const configurations = useConfigurations(variables)
const form = useAppForm({
defaultValues: initialData,
})
return (
<form
className='w-full'
onSubmit={(e) => {
e.preventDefault()
e.stopPropagation()
}}
>
<div className='flex flex-col gap-y-3 px-4 py-3'>
{configurations.map((config, index) => {
const FieldComponent = BaseField({
initialData,
config,
})
return <FieldComponent key={index} form={form} />
})}
</div>
</form>
)
}
export default Form

14
web/app/components/rag-pipeline/components/input-field/preview/index.tsx
View File

@ -1,7 +1,12 @@
import { useState } from 'react'
import { RiCloseLine } from '@remixicon/react'
import DialogWrapper from './dialog-wrapper'
import { useTranslation } from 'react-i18next'
import Badge from '@/app/components/base/badge'
import DataSource from './data-source'
import Divider from '@/app/components/base/divider'
import ProcessDocuments from './process-documents'
import type { Datasource } from '../../panel/test-run/types'
type PreviewPanelProps = {
show: boolean
@ -13,6 +18,7 @@ const PreviewPanel = ({
onClose,
}: PreviewPanelProps) => {
const { t } = useTranslation()
const [datasource, setDatasource] = useState<Datasource>()
return (
<DialogWrapper
@ -34,6 +40,14 @@ const PreviewPanel = ({
<RiCloseLine className='size-4 text-text-tertiary' />
</button>
</div>
<DataSource
onSelect={setDatasource}
datasourceNodeId={datasource?.nodeId || ''}
/>
<div className='px-4 py-2'>
<Divider type='horizontal' className='bg-divider-subtle' />
</div>
<ProcessDocuments dataSourceNodeId={datasource?.nodeId || ''} />
</DialogWrapper>
)
}

31
web/app/components/rag-pipeline/components/input-field/preview/process-documents.tsx Normal file
View File

@ -0,0 +1,31 @@
import React from 'react'
import { useTranslation } from 'react-i18next'
import { useStore } from '@/app/components/workflow/store'
import { useDraftPipelineProcessingParams } from '@/service/use-pipeline'
import Form from './form'
type ProcessDocumentsProps = {
dataSourceNodeId: string
}
const ProcessDocuments = ({
dataSourceNodeId,
}: ProcessDocumentsProps) => {
const { t } = useTranslation()
const pipelineId = useStore(state => state.pipelineId)
const { data: paramsConfig } = useDraftPipelineProcessingParams({
pipeline_id: pipelineId!,
node_id: dataSourceNodeId,
})
return (
<div className='flex flex-col'>
<div className='system-sm-semibold-uppercase px-4 pt-2 text-text-secondary'>
{t('datasetPipeline.inputFieldPanel.preview.stepTwoTitle')}
</div>
<Form variables={paramsConfig?.variables || []} />
</div>
)
}
export default React.memo(ProcessDocuments)

1
web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
View File

@ -49,7 +49,6 @@ export const useDatasourceOptions = () => {
return {
nodeId: node.id,
type: node.data.provider_type as DatasourceType,
variables: node.data.variables || [],
description: '', // todo: Add description
docTitle: '', // todo: Add docTitle and docLink
docLink: '',

2
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@ -139,7 +139,7 @@ const TestRunPanel = () => {
{datasource?.type === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
variables={[]} // todo: replace with actual variables if needed
checkedCrawlResult={websitePages}
headerInfo={{
title: datasource.description,

3
web/app/components/rag-pipeline/components/panel/test-run/types.ts
View File

@ -1,5 +1,5 @@
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import type { DatasourceType, RAGPipelineVariables } from '@/models/pipeline'
import type { DatasourceType } from '@/models/pipeline'
export enum TestRunStep {
dataSource = 'dataSource',
@ -15,7 +15,6 @@ export type DataSourceOption = {
export type Datasource = {
nodeId: string
type: DatasourceType
variables: RAGPipelineVariables
description: string
docTitle?: string
docLink?: string

1
web/app/components/tools/types.ts
View File

@ -52,6 +52,7 @@ export type Collection = {
plugin_id?: string
letter?: string
is_authorized?: boolean
provider?: string
}
export type ToolParameter = {

8
web/app/components/workflow/nodes/_base/components/variable/utils.ts
View File

@ -496,11 +496,11 @@ const formatItem = (
}
case 'rag': {
res.vars = data.ragVariables.map((ragVarialbe: RAGPipelineVariable) => {
res.vars = data.ragVariables.map((ragVar: RAGPipelineVariable) => {
return {
variable: `rag.${ragVarialbe.variable}`,
type: inputVarTypeToVarType(ragVarialbe.type as any),
des: ragVarialbe.label,
variable: `rag.${ragVar.variable}`,
type: inputVarTypeToVarType(ragVar.type as any),
des: ragVar.label,
isRagVariable: true,
}
}) as Var[]

10
web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx
View File

@ -13,7 +13,7 @@ import { useReactFlow, useStoreApi } from 'reactflow'
import RemoveButton from '../remove-button'
import useAvailableVarList from '../../hooks/use-available-var-list'
import VarReferencePopup from './var-reference-popup'
import { getNodeInfoById, isConversationVar, isENV, isSystemVar, varTypeToStructType } from './utils'
import { getNodeInfoById, isConversationVar, isENV, isRagVariableVar, isSystemVar, varTypeToStructType } from './utils'
import ConstantField from './constant-field'
import cn from '@/utils/classnames'
import type { Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types'
@ -40,6 +40,7 @@ import Tooltip from '@/app/components/base/tooltip'
import { isExceptionVariable } from '@/app/components/workflow/utils'
import VarFullPathPanel from './var-full-path-panel'
import { noop } from 'lodash-es'
import { InputField } from '@/app/components/base/icons/src/vender/pipeline'
const TRIGGER_DEFAULT_WIDTH = 227
@ -274,14 +275,16 @@ const VarReferencePicker: FC<Props> = ({
isConstant: !!isConstant,
})
const { isEnv, isChatVar, isValidVar, isException } = useMemo(() => {
const { isEnv, isChatVar, isRagVar, isValidVar, isException } = useMemo(() => {
const isEnv = isENV(value as ValueSelector)
const isChatVar = isConversationVar(value as ValueSelector)
const isRagVar = isRagVariableVar(value as ValueSelector)
const isValidVar = Boolean(outputVarNode) || isEnv || isChatVar
const isException = isExceptionVariable(varName, outputVarNode?.type)
return {
isEnv,
isChatVar,
isRagVar,
isValidVar,
isException,
}
@ -385,7 +388,7 @@ const VarReferencePicker: FC<Props> = ({
{hasValue
? (
<>
{isShowNodeName && !isEnv && !isChatVar && (
{isShowNodeName && !isEnv && !isChatVar && !isRagVar && (
<div className='flex items-center' onClick={(e) => {
if (e.metaKey || e.ctrlKey) {
e.stopPropagation()
@ -414,6 +417,7 @@ const VarReferencePicker: FC<Props> = ({
{!hasValue && <Variable02 className='h-3.5 w-3.5' />}
{isEnv && <Env className='h-3.5 w-3.5 text-util-colors-violet-violet-600' />}
{isChatVar && <BubbleX className='h-3.5 w-3.5 text-util-colors-teal-teal-700' />}
{isRagVar && <InputField className='h-3.5 w-3.5 text-text-accent' />}
<div className={cn('ml-0.5 truncate text-xs font-medium', isEnv && '!text-text-secondary', isChatVar && 'text-util-colors-teal-teal-700', isException && 'text-text-warning')} title={varName} style={{
maxWidth: maxVarNameWidth,
}}>{varName}</div>

7
web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx
View File

@ -125,7 +125,7 @@ const Item: FC<ItemProps> = ({
if (!isSupportFileVar && isFile)
return
if (isSys || isEnv || isChatVar) { // system variable | environment variable | conversation variable
if (isSys || isEnv || isChatVar || isRagVariable) { // system variable | environment variable | conversation variable
onChange([...objPath, ...itemData.variable.split('.')], itemData)
}
else {
@ -155,7 +155,7 @@ const Item: FC<ItemProps> = ({
{isChatVar && <BubbleX className='h-3.5 w-3.5 shrink-0 text-util-colors-teal-teal-700' />}
{isLoopVar && <Loop className='h-3.5 w-3.5 shrink-0 text-util-colors-cyan-cyan-500' />}
{isRagVariable && <InputField className='h-3.5 w-3.5 shrink-0 text-text-accent' />}
{!isEnv && !isChatVar && (
{!isEnv && !isChatVar && !isRagVariable && (
<div title={itemData.variable} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable}</div>
)}
{isEnv && (
@ -164,6 +164,9 @@ const Item: FC<ItemProps> = ({
{isChatVar && (
<div title={itemData.des} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable.replace('conversation.', '')}</div>
)}
{isRagVariable && (
<div title={itemData.des} className='system-sm-medium ml-1 w-0 grow truncate text-text-secondary'>{itemData.variable.replace('rag.', '')}</div>
)}
</div>
<div className='ml-1 shrink-0 text-xs font-normal capitalize text-text-tertiary'>{itemData.type}</div>
{

17
web/app/components/workflow/nodes/assigner/node.tsx
View File

@ -4,7 +4,7 @@ import { useNodes } from 'reactflow'
import { useTranslation } from 'react-i18next'
import NodeVariableItem from '../variable-assigner/components/node-variable-item'
import type { AssignerNodeType } from './types'
import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { BlockEnum, type Node, type NodeProps } from '@/app/components/workflow/types'
const i18nPrefix = 'workflow.nodes.assigner'
@ -38,18 +38,13 @@ const NodeComponent: FC<NodeProps<AssignerNodeType>> = ({
if (!variable || variable.length === 0)
return null
const isSystem = isSystemVar(variable)
const isEnv = isENV(variable)
const isChatVar = isConversationVar(variable)
const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0])
const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
return (
<NodeVariableItem
key={index}
node={node as Node}
isEnv={isEnv}
isChatVar={isChatVar}
variable={variable}
writeMode={value.operation}
varName={varName}
className='bg-workflow-block-parma-bg'
/>
)
@ -63,19 +58,13 @@ const NodeComponent: FC<NodeProps<AssignerNodeType>> = ({
if (!variable || variable.length === 0)
return null
const isSystem = isSystemVar(variable)
const isEnv = isENV(variable)
const isChatVar = isConversationVar(variable)
const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0])
const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
return (
<div className='relative flex flex-col items-start gap-0.5 self-stretch px-3 py-1'>
<NodeVariableItem
node={node as Node}
isEnv={isEnv}
isChatVar={isChatVar}
varName={varName}
variable={variable}
writeMode={writeMode}
className='bg-workflow-block-parma-bg'
/>

6
web/app/components/workflow/nodes/data-source/panel.tsx
View File

@ -71,8 +71,8 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
const { mutateAsync } = useUpdateDataSourceCredentials()
const handleAuth = useCallback(async (value: any) => {
await mutateAsync({
provider: currentDataSourceItem?.provider,
pluginId: currentDataSourceItem?.plugin_id,
provider: currentDataSource?.provider || '',
pluginId: currentDataSource?.plugin_id || '',
credentials: value,
})
@ -81,7 +81,7 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
message: t('common.api.actionSuccess'),
})
hideAuthModal()
}, [currentDataSourceItem, mutateAsync, notify, t, hideAuthModal])
}, [currentDataSource, mutateAsync, notify, t, hideAuthModal])
return (
<div >

9
web/app/components/workflow/nodes/document-extractor/node.tsx
View File

@ -4,7 +4,7 @@ import { useNodes } from 'reactflow'
import { useTranslation } from 'react-i18next'
import NodeVariableItem from '../variable-assigner/components/node-variable-item'
import type { DocExtractorNodeType } from './types'
import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { BlockEnum, type Node, type NodeProps } from '@/app/components/workflow/types'
const i18nPrefix = 'workflow.nodes.docExtractor'
@ -21,18 +21,13 @@ const NodeComponent: FC<NodeProps<DocExtractorNodeType>> = ({
return null
const isSystem = isSystemVar(variable)
const isEnv = isENV(variable)
const isChatVar = isConversationVar(variable)
const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0])
const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
return (
<div className='relative px-3'>
<div className='system-2xs-medium-uppercase mb-1 text-text-tertiary'>{t(`${i18nPrefix}.inputVar`)}</div>
<NodeVariableItem
node={node as Node}
isEnv={isEnv}
isChatVar={isChatVar}
varName={varName}
variable={variable}
className='bg-workflow-block-parma-bg'
/>
</div>

9
web/app/components/workflow/nodes/list-operator/node.tsx
View File

@ -4,7 +4,7 @@ import { useNodes } from 'reactflow'
import { useTranslation } from 'react-i18next'
import NodeVariableItem from '../variable-assigner/components/node-variable-item'
import type { ListFilterNodeType } from './types'
import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { BlockEnum, type Node, type NodeProps } from '@/app/components/workflow/types'
const i18nPrefix = 'workflow.nodes.listFilter'
@ -21,18 +21,13 @@ const NodeComponent: FC<NodeProps<ListFilterNodeType>> = ({
return null
const isSystem = isSystemVar(variable)
const isEnv = isENV(variable)
const isChatVar = isConversationVar(variable)
const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0])
const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
return (
<div className='relative px-3'>
<div className='system-2xs-medium-uppercase mb-1 text-text-tertiary'>{t(`${i18nPrefix}.inputVar`)}</div>
<NodeVariableItem
node={node as Node}
isEnv={isEnv}
isChatVar={isChatVar}
varName={varName}
variable={variable}
className='bg-workflow-block-parma-bg'
/>
</div>

9
web/app/components/workflow/nodes/variable-assigner/components/node-group-item.tsx
View File

@ -19,7 +19,7 @@ import {
import { filterVar } from '../utils'
import AddVariable from './add-variable'
import NodeVariableItem from './node-variable-item'
import { isConversationVar, isENV, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import cn from '@/utils/classnames'
import { isExceptionVariable } from '@/app/components/workflow/utils'
@ -124,9 +124,6 @@ const NodeGroupItem = ({
{
!!item.variables.length && item.variables.map((variable = [], index) => {
const isSystem = isSystemVar(variable)
const isEnv = isENV(variable)
const isChatVar = isConversationVar(variable)
const node = isSystem ? nodes.find(node => node.data.type === BlockEnum.Start) : nodes.find(node => node.id === variable[0])
const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
const isException = isExceptionVariable(varName, node?.data.type)
@ -134,11 +131,9 @@ const NodeGroupItem = ({
return (
<NodeVariableItem
key={index}
isEnv={isEnv}
isChatVar={isChatVar}
variable={variable}
isException={isException}
node={node as Node}
varName={varName}
showBorder={showSelectedBorder || showSelectionBorder}
/>
)

26
web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx
View File

@ -9,13 +9,13 @@ import { Line3 } from '@/app/components/base/icons/src/public/common'
import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development'
import { BubbleX, Env } from '@/app/components/base/icons/src/vender/line/others'
import Badge from '@/app/components/base/badge'
import type { Node } from '@/app/components/workflow/types'
import type { Node, ValueSelector } from '@/app/components/workflow/types'
import { isConversationVar, isENV, isRagVariableVar, isSystemVar } from '@/app/components/workflow/nodes/_base/components/variable/utils'
import { InputField } from '@/app/components/base/icons/src/vender/pipeline'
type NodeVariableItemProps = {
isEnv: boolean
isChatVar: boolean
node: Node
varName: string
variable: ValueSelector
writeMode?: string
showBorder?: boolean
className?: string
@ -25,10 +25,8 @@ type NodeVariableItemProps = {
const i18nPrefix = 'workflow.nodes.assigner'
const NodeVariableItem = ({
isEnv,
isChatVar,
node,
varName,
variable,
writeMode,
showBorder,
className,
@ -36,6 +34,12 @@ const NodeVariableItem = ({
}: NodeVariableItemProps) => {
const { t } = useTranslation()
const isSystem = isSystemVar(variable)
const isEnv = isENV(variable)
const isChatVar = isConversationVar(variable)
const isRagVar = isRagVariableVar(variable)
const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.')
const VariableIcon = useMemo(() => {
if (isEnv) {
return (
@ -49,6 +53,12 @@ const NodeVariableItem = ({
)
}
if(isRagVar) {
return (
<InputField className='h-3.5 w-3.5 shrink-0 text-text-accent' />
)
}
return (
<Variable02
className={cn(
@ -57,7 +67,7 @@ const NodeVariableItem = ({
)}
/>
)
}, [isEnv, isChatVar, isException])
}, [isEnv, isChatVar, isRagVar, isException])
const VariableName = useMemo(() => {
return (

2
web/models/pipeline.ts
View File

@ -107,7 +107,7 @@ export enum PipelineInputVarType {
textInput = 'text-input',
paragraph = 'paragraph',
select = 'select',
number = 'number-input',
number = 'number',
singleFile = 'file',
multiFiles = 'file-list',
checkbox = 'checkbox',

9
web/service/use-pipeline.ts
View File

@ -148,6 +148,7 @@ export const useDraftPipelineProcessingParams = (params: PipelineProcessingParam
})
},
staleTime: 0,
enabled: !!pipeline_id && !!node_id,
})
}
@ -211,12 +212,12 @@ export const useRunPublishedPipeline = (
}
export const useDataSourceCredentials = (provider: string, pluginId: string, onSuccess: (value: ToolCredential[]) => void) => {
return useQuery<ToolCredential[]>({
return useQuery({
queryKey: [NAME_SPACE, 'datasource-credentials', provider, pluginId],
queryFn: async () => {
const result = await get<ToolCredential[]>(`/auth/plugin/datasource?provider=${provider}&plugin_id=${pluginId}`)
onSuccess(result)
return result
const result = await get<{ result: ToolCredential[] }>(`/auth/plugin/datasource?provider=${provider}&plugin_id=${pluginId}`)
onSuccess(result.result)
return result.result
},
enabled: !!provider && !!pluginId,
retry: 2,