diff --git a/web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/index.tsx b/web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/index.tsx index e7914b32d7..06f0610dfd 100644 --- a/web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/index.tsx +++ b/web/app/components/rag-pipeline/components/rag-pipeline-header/publisher/index.tsx @@ -21,7 +21,7 @@ const Publisher = () => { const handleOpenChange = useCallback((newOpen: boolean) => { if (newOpen) - handleSyncWorkflowDraft() + handleSyncWorkflowDraft(true) setOpen(newOpen) }, [handleSyncWorkflowDraft]) diff --git a/web/app/components/workflow/hooks/use-workflow-variables.ts b/web/app/components/workflow/hooks/use-workflow-variables.ts index 35637bc775..d6bda297fd 100644 --- a/web/app/components/workflow/hooks/use-workflow-variables.ts +++ b/web/app/components/workflow/hooks/use-workflow-variables.ts @@ -1,6 +1,6 @@ import { useCallback } from 'react' import { useTranslation } from 'react-i18next' -import { useStore } from '../store' +import { useWorkflowStore } from '../store' import { getVarType, toNodeAvailableVars } from '@/app/components/workflow/nodes/_base/components/variable/utils' import type { Node, @@ -13,8 +13,7 @@ import { useStoreApi } from 'reactflow' export const useWorkflowVariables = () => { const { t } = useTranslation() - const environmentVariables = useStore(s => s.environmentVariables) - const conversationVariables = useStore(s => s.conversationVariables) + const workflowStore = useWorkflowStore() const getNodeAvailableVars = useCallback(({ parentNode, @@ -31,6 +30,11 @@ export const useWorkflowVariables = () => { hideEnv?: boolean hideChatVar?: boolean }): NodeOutPutVar[] => { + const { + conversationVariables, + environmentVariables, + ragPipelineVariables, + } = workflowStore.getState() return toNodeAvailableVars({ parentNode, t, @@ -38,9 +42,10 @@ export const useWorkflowVariables = () => { isChatMode, environmentVariables: hideEnv ? [] : environmentVariables, conversationVariables: (isChatMode && !hideChatVar) ? conversationVariables : [], + ragVariables: ragPipelineVariables, filterVar, }) - }, [conversationVariables, environmentVariables, t]) + }, [t, workflowStore]) const getCurrentVariableType = useCallback(({ parentNode, @@ -59,6 +64,11 @@ export const useWorkflowVariables = () => { isChatMode: boolean isConstant?: boolean }) => { + const { + conversationVariables, + environmentVariables, + ragPipelineVariables, + } = workflowStore.getState() return getVarType({ parentNode, valueSelector, @@ -69,8 +79,9 @@ export const useWorkflowVariables = () => { isConstant, environmentVariables, conversationVariables, + ragVariables: ragPipelineVariables, }) - }, [conversationVariables, environmentVariables]) + }, [workflowStore]) return { getNodeAvailableVars, diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index 65f32ba230..6e6c85dc89 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -22,6 +22,7 @@ import type { StartNodeType } from '@/app/components/workflow/nodes/start/types' import type { ConversationVariable, EnvironmentVariable, Node, NodeOutPutVar, ValueSelector, Var } from '@/app/components/workflow/types' import type { VariableAssignerNodeType } from '@/app/components/workflow/nodes/variable-assigner/types' import type { Field as StructField } from '@/app/components/workflow/nodes/llm/types' +import type { RAGPipelineVariable } from '@/models/pipeline' import { HTTP_REQUEST_OUTPUT_STRUCT, @@ -51,6 +52,10 @@ export const isConversationVar = (valueSelector: ValueSelector) => { return valueSelector[0] === 'conversation' } +export const isRagVariableVar = (valueSelector: ValueSelector) => { + return valueSelector[0] === 'rag' +} + const inputVarTypeToVarType = (type: InputVarType): VarType => { return ({ [InputVarType.number]: VarType.number, @@ -170,6 +175,7 @@ const formatItem = ( item: any, isChatMode: boolean, filterVar: (payload: Var, selector: ValueSelector) => boolean, + ragVars?: Var[], ): NodeOutPutVar => { const { id, data } = item @@ -460,7 +466,7 @@ const formatItem = ( } case BlockEnum.DataSource: { - res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType) || [] + res.vars = DataSourceNodeDefault.getOutputVars?.(data as DataSourceNodeType, ragVars) || [] break } @@ -484,6 +490,18 @@ const formatItem = ( }) as Var[] break } + + case 'rag': { + res.vars = data.ragVariables.map((ragVarialbe: RAGPipelineVariable) => { + return { + variable: `rag.${ragVarialbe.variable}`, + type: inputVarTypeToVarType(ragVarialbe.type as any), + des: ragVarialbe.label, + isRagVariable: true, + } + }) as Var[] + break + } } const { error_strategy } = data @@ -564,6 +582,7 @@ export const toNodeOutputVars = ( filterVar = (_payload: Var, _selector: ValueSelector) => true, environmentVariables: EnvironmentVariable[] = [], conversationVariables: ConversationVariable[] = [], + ragVariables: RAGPipelineVariable[] = [], ): NodeOutPutVar[] => { // ENV_NODE data format const ENV_NODE = { @@ -583,6 +602,15 @@ export const toNodeOutputVars = ( chatVarList: conversationVariables, }, } + // RAG_PIPELINE_NODE data format + const RAG_PIPELINE_NODE = { + id: 'rag', + data: { + title: 'SHARED INPUTS', + type: 'rag', + ragVariables: ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === 'shared'), + }, + } // Sort nodes in reverse chronological order (most recent first) const sortedNodes = [...nodes].sort((a, b) => { if (a.data.type === BlockEnum.Start) return 1 @@ -599,9 +627,20 @@ export const toNodeOutputVars = ( ...sortedNodes.filter(node => SUPPORT_OUTPUT_VARS_NODE.includes(node?.data?.type)), ...(environmentVariables.length > 0 ? [ENV_NODE] : []), ...((isChatMode && conversationVariables.length > 0) ? [CHAT_VAR_NODE] : []), + ...(RAG_PIPELINE_NODE.data.ragVariables.length > 0 ? [RAG_PIPELINE_NODE] : []), ].map((node) => { + let ragVariablesInDataSource: RAGPipelineVariable[] = [] + if (node.data.type === BlockEnum.DataSource) + ragVariablesInDataSource = ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === node.id) return { - ...formatItem(node, isChatMode, filterVar), + ...formatItem(node, isChatMode, filterVar, ragVariablesInDataSource.map( + (ragVariable: RAGPipelineVariable) => ({ + variable: ragVariable.variable, + type: inputVarTypeToVarType(ragVariable.type as any), + description: ragVariable.label, + isRagVariable: true, + } as Var), + )), isStartNode: node.data.type === BlockEnum.Start, } }).filter(item => item.vars.length > 0) @@ -719,6 +758,7 @@ export const getVarType = ({ isConstant, environmentVariables = [], conversationVariables = [], + ragVariables = [], }: { valueSelector: ValueSelector parentNode?: Node | null @@ -729,6 +769,7 @@ export const getVarType = ({ isConstant?: boolean environmentVariables?: EnvironmentVariable[] conversationVariables?: ConversationVariable[] + ragVariables?: RAGPipelineVariable[] }): VarType => { if (isConstant) return VarType.string @@ -739,6 +780,7 @@ export const getVarType = ({ undefined, environmentVariables, conversationVariables, + ragVariables, ) const isIterationInnerVar = parentNode?.data.type === BlockEnum.Iteration @@ -782,6 +824,7 @@ export const getVarType = ({ const isSystem = isSystemVar(valueSelector) const isEnv = isENV(valueSelector) const isChatVar = isConversationVar(valueSelector) + const isRagVariable = isRagVariableVar(valueSelector) const startNode = availableNodes.find((node: any) => { return node?.data.type === BlockEnum.Start }) @@ -795,7 +838,7 @@ export const getVarType = ({ let type: VarType = VarType.string let curr: any = targetVar.vars - if (isSystem || isEnv || isChatVar) { + if (isSystem || isEnv || isChatVar || isRagVariable) { return curr.find((v: any) => v.variable === (valueSelector as ValueSelector).join('.'))?.type } else { @@ -846,6 +889,7 @@ export const toNodeAvailableVars = ({ isChatMode, environmentVariables, conversationVariables, + ragVariables, filterVar, }: { parentNode?: Node | null @@ -857,6 +901,8 @@ export const toNodeAvailableVars = ({ environmentVariables?: EnvironmentVariable[] // chat var conversationVariables?: ConversationVariable[] + // rag variables + ragVariables?: RAGPipelineVariable[] filterVar: (payload: Var, selector: ValueSelector) => boolean }): NodeOutPutVar[] => { const beforeNodesOutputVars = toNodeOutputVars( @@ -865,6 +911,7 @@ export const toNodeAvailableVars = ({ filterVar, environmentVariables, conversationVariables, + ragVariables, ) const isInIteration = parentNode?.data.type === BlockEnum.Iteration if (isInIteration) { diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx index 023916ec5b..731a4bff1f 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx @@ -23,6 +23,7 @@ import type { Field } from '@/app/components/workflow/nodes/llm/types' import { FILE_STRUCT } from '@/app/components/workflow/constants' import { Loop } from '@/app/components/base/icons/src/vender/workflow' import { noop } from 'lodash-es' +import { InputField } from '@/app/components/base/icons/src/vender/pipeline' type ObjectChildrenProps = { nodeId: string @@ -67,6 +68,7 @@ const Item: FC = ({ const isSys = itemData.variable.startsWith('sys.') const isEnv = itemData.variable.startsWith('env.') const isChatVar = itemData.variable.startsWith('conversation.') + const isRagVariable = itemData.isRagVariable const objStructuredOutput: StructuredOutput | null = useMemo(() => { if (!isObj) return null @@ -148,10 +150,11 @@ const Item: FC = ({ onMouseDown={e => e.preventDefault()} >
- {!isEnv && !isChatVar && !isLoopVar && } + {!isEnv && !isChatVar && !isLoopVar && !isRagVariable && } {isEnv && } {isChatVar && } {isLoopVar && } + {isRagVariable && } {!isEnv && !isChatVar && (
{itemData.variable}
)} diff --git a/web/app/components/workflow/nodes/data-source/default.ts b/web/app/components/workflow/nodes/data-source/default.ts index e703bd2265..9859be30cb 100644 --- a/web/app/components/workflow/nodes/data-source/default.ts +++ b/web/app/components/workflow/nodes/data-source/default.ts @@ -11,6 +11,28 @@ const metaData = genNodeMetaData({ const nodeDefault: NodeDefault = { metaData, defaultValue: { + fileExtensions: [ + 'txt', + 'markdown', + 'mdx', + 'pdf', + 'html', + 'xlsx', + 'xls', + 'vtt', + 'properties', + 'doc', + 'docx', + 'csv', + 'eml', + 'msg', + 'pptx', + 'xml', + 'epub', + 'ppt', + 'md', + 'html', + ], datasource_parameters: {}, datasource_configurations: {}, }, @@ -20,7 +42,7 @@ const nodeDefault: NodeDefault = { errorMessage: '', } }, - getOutputVars(payload) { + getOutputVars(payload, ragVars = []) { const { provider_type, } = payload @@ -40,6 +62,7 @@ const nodeDefault: NodeDefault = { ] : [] ), + ...ragVars, ] }, } diff --git a/web/app/components/workflow/types.ts b/web/app/components/workflow/types.ts index 235f60a9a8..b291e4224e 100644 --- a/web/app/components/workflow/types.ts +++ b/web/app/components/workflow/types.ts @@ -288,6 +288,7 @@ export type Var = { isException?: boolean isLoopVariable?: boolean nodeId?: string + isRagVariable?: boolean } export type NodeOutPutVar = { @@ -313,7 +314,7 @@ export type NodeDefault = { } defaultValue: Partial checkValid: (payload: T, t: any, moreDataForCheckValid?: any) => { isValid: boolean; errorMessage?: string } - getOutputVars?: (payload: T) => Var[] + getOutputVars?: (payload: T, ragVariables?: Var[]) => Var[] } export type OnSelectBlock = (type: BlockEnum, toolDefaultValue?: ToolDefaultValue | DataSourceDefaultValue) => void