From 91fefa0e3756db356b4adb198e90914d66c834bf Mon Sep 17 00:00:00 2001 From: twwu Date: Thu, 5 Jun 2025 15:44:18 +0800 Subject: [PATCH 1/4] refactor: improve layout and event handling in Header and FieldItem components --- .../datasets/create/website/base/header.tsx | 30 ++++++++++--------- .../input-field/field-list/field-item.tsx | 6 ++-- .../notion/notion-page-selector.tsx | 11 ++++--- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/web/app/components/datasets/create/website/base/header.tsx b/web/app/components/datasets/create/website/base/header.tsx index 7710d8ada9..963efd1761 100644 --- a/web/app/components/datasets/create/website/base/header.tsx +++ b/web/app/components/datasets/create/website/base/header.tsx @@ -30,21 +30,23 @@ const Header = ({ )}> {title} - {!isInPipeline && ( - + <> + + + )} { + const handleOnClickEdit = useCallback((e: React.MouseEvent) => { + e.stopPropagation() if (readonly) return onClickEdit(payload.variable) }, [onClickEdit, payload.variable, readonly]) - const handleRemove = useCallback(() => { + const handleRemove = useCallback((e: React.MouseEvent) => { + e.stopPropagation() if (readonly) return onRemove(index) }, [index, onRemove, readonly]) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx index ca79c1a9ed..988b00a337 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx @@ -1,6 +1,6 @@ import { useCallback, useEffect, useMemo, useState } from 'react' import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector' -import SearchInput from '@/app/education-apply/search-input' +import SearchInput from '@/app/components/base/notion-page-selector/search-input' import PageSelector from '@/app/components/base/notion-page-selector/page-selector' import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' import Header from '@/app/components/datasets/create/website/base/header' @@ -36,12 +36,15 @@ const NotionPageSelector = ({ const getNotionData = useCallback(async () => { if (pipeline_id) { - const notionData = await getNotionPages({ + await getNotionPages({ pipeline_id, node_id: nodeId, inputs: {}, - }) as DataSourceNotionWorkspace[] - setNotionData(notionData) + }, { + onSuccess(notionData) { + setNotionData(notionData as DataSourceNotionWorkspace[]) + }, + }) } }, [getNotionPages, nodeId, pipeline_id]) From d4a1d045f807165f18e4348b9c2b9aa93445b4cc Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 5 Jun 2025 16:36:41 +0800 Subject: [PATCH 2/4] fix: to new var format --- .../workflow/nodes/_base/components/variable/utils.ts | 7 +++++-- .../_base/components/variable/var-reference-picker.tsx | 2 +- .../nodes/_base/components/variable/var-reference-vars.tsx | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts index d29459a161..1f6a9ea008 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts +++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts @@ -53,6 +53,8 @@ export const isConversationVar = (valueSelector: ValueSelector) => { } export const isRagVariableVar = (valueSelector: ValueSelector) => { + if(!valueSelector) + return false return valueSelector[0] === 'rag' } @@ -498,7 +500,7 @@ const formatItem = ( case 'rag': { res.vars = data.ragVariables.map((ragVar: RAGPipelineVariable) => { return { - variable: `rag.${ragVar.variable}`, + variable: `rag.shared.${ragVar.variable}`, type: inputVarTypeToVarType(ragVar.type as any), des: ragVar.label, isRagVariable: true, @@ -636,10 +638,11 @@ export const toNodeOutputVars = ( let ragVariablesInDataSource: RAGPipelineVariable[] = [] if (node.data.type === BlockEnum.DataSource) ragVariablesInDataSource = ragVariables.filter(ragVariable => ragVariable.belong_to_node_id === node.id) + console.log(ragVariables, ragVariablesInDataSource, node.id) return { ...formatItem(node, isChatMode, filterVar, ragVariablesInDataSource.map( (ragVariable: RAGPipelineVariable) => ({ - variable: ragVariable.variable, + variable: `rag.${node.id}.${ragVariable.variable}`, type: inputVarTypeToVarType(ragVariable.type as any), description: ragVariable.label, isRagVariable: true, diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx index 24de113e0d..f0b2b6c572 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-picker.tsx @@ -183,7 +183,7 @@ const VarReferencePicker: FC = ({ } }, [value, hasValue, isConstant, isIterationVar, iterationNode, availableNodes, outputVarNodeId, startNode, isLoopVar, loopNode]) - const isShowAPart = (value as ValueSelector).length > 2 + const isShowAPart = (value as ValueSelector).length > 2 && !isRagVariableVar((value as ValueSelector)) const varName = useMemo(() => { if (!hasValue) diff --git a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx index 1f72d57da7..ef202336ee 100644 --- a/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx +++ b/web/app/components/workflow/nodes/_base/components/variable/var-reference-vars.tsx @@ -165,7 +165,7 @@ const Item: FC = ({
{itemData.variable.replace('conversation.', '')}
)} {isRagVariable && ( -
{itemData.variable.replace('rag.', '')}
+
{itemData.variable.split('.').slice(-1)[0]}
)}
{itemData.type}
From 90ca98ff3aa8a4ef981c523d3c1bdbe7132d1f2f Mon Sep 17 00:00:00 2001 From: Joel Date: Thu, 5 Jun 2025 16:41:04 +0800 Subject: [PATCH 3/4] fix: in node show rag var --- .../variable-assigner/components/node-variable-item.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx b/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx index 81286e21f8..b4fc76d548 100644 --- a/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx +++ b/web/app/components/workflow/nodes/variable-assigner/components/node-variable-item.tsx @@ -38,7 +38,13 @@ const NodeVariableItem = ({ const isEnv = isENV(variable) const isChatVar = isConversationVar(variable) const isRagVar = isRagVariableVar(variable) - const varName = isSystem ? `sys.${variable[variable.length - 1]}` : variable.slice(1).join('.') + const varName = useMemo(() => { + if(isSystem) + return `sys.${variable[variable.length - 1]}` + if(isRagVar) + return variable[variable.length - 1] + return variable.slice(1).join('.') + }, [isRagVar, isSystem, variable]) const VariableIcon = useMemo(() => { if (isEnv) { From f0413f359ac7e17681b78f7c9538f063ef8bf100 Mon Sep 17 00:00:00 2001 From: zxhlyh Date: Thu, 5 Jun 2025 16:51:01 +0800 Subject: [PATCH 4/4] datasource --- .../hooks/use-nodes-sync-draft.ts | 3 +- .../workflow/block-selector/data-sources.tsx | 2 +- .../workflow/block-selector/types.ts | 2 +- .../workflow/block-selector/utils.ts | 2 +- web/app/components/workflow/index.tsx | 2 +- .../workflow/nodes/data-source/constants.ts | 34 +++++++++++-- .../workflow/nodes/data-source/default.ts | 27 +++++----- .../workflow/nodes/data-source/panel.tsx | 49 +++++++++++++------ .../workflow/nodes/data-source/types.ts | 7 ++- web/app/components/workflow/types.ts | 2 +- 10 files changed, 91 insertions(+), 39 deletions(-) diff --git a/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts b/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts index eb2a862be6..0a12cf6557 100644 --- a/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts +++ b/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts @@ -23,6 +23,7 @@ export const useNodesSyncDraft = () => { edges, transform, } = store.getState() + const nodes = getNodes() const [x, y, zoom] = transform const { pipelineId, @@ -32,8 +33,6 @@ export const useNodesSyncDraft = () => { } = workflowStore.getState() if (pipelineId) { - const nodes = getNodes() - const producedNodes = produce(nodes, (draft) => { draft.forEach((node) => { Object.keys(node.data).forEach((key) => { diff --git a/web/app/components/workflow/block-selector/data-sources.tsx b/web/app/components/workflow/block-selector/data-sources.tsx index 795adf5cc0..947c97a010 100644 --- a/web/app/components/workflow/block-selector/data-sources.tsx +++ b/web/app/components/workflow/block-selector/data-sources.tsx @@ -32,7 +32,7 @@ const DataSources = ({ const wrapElemRef = useRef(null) const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => { onSelect(BlockEnum.DataSource, toolDefaultValue && { - provider_id: toolDefaultValue?.provider_id, + plugin_id: toolDefaultValue?.provider_id, provider_type: toolDefaultValue?.provider_type, provider_name: toolDefaultValue?.provider_name, datasource_name: toolDefaultValue?.tool_name, diff --git a/web/app/components/workflow/block-selector/types.ts b/web/app/components/workflow/block-selector/types.ts index 22b9b7497e..75cfdf4818 100644 --- a/web/app/components/workflow/block-selector/types.ts +++ b/web/app/components/workflow/block-selector/types.ts @@ -36,7 +36,7 @@ export type ToolDefaultValue = { } export type DataSourceDefaultValue = { - provider_id: string + plugin_id: string provider_type: string provider_name: string datasource_name: string diff --git a/web/app/components/workflow/block-selector/utils.ts b/web/app/components/workflow/block-selector/utils.ts index 3e73d7a808..b1ad7b1256 100644 --- a/web/app/components/workflow/block-selector/utils.ts +++ b/web/app/components/workflow/block-selector/utils.ts @@ -5,7 +5,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => { return { id: dataSourceItem.plugin_id, provider: dataSourceItem.provider, - name: dataSourceItem.declaration.identity.name, + name: dataSourceItem.provider, author: dataSourceItem.declaration.identity.author, description: dataSourceItem.declaration.identity.description, icon: dataSourceItem.declaration.identity.icon, diff --git a/web/app/components/workflow/index.tsx b/web/app/components/workflow/index.tsx index beea4116cf..ad74e88df5 100644 --- a/web/app/components/workflow/index.tsx +++ b/web/app/components/workflow/index.tsx @@ -375,4 +375,4 @@ const WorkflowWithDefaultContext = ({ ) } -export default memo(WorkflowWithDefaultContext) +export default WorkflowWithDefaultContext diff --git a/web/app/components/workflow/nodes/data-source/constants.ts b/web/app/components/workflow/nodes/data-source/constants.ts index 10aa058a71..5b6de604f1 100644 --- a/web/app/components/workflow/nodes/data-source/constants.ts +++ b/web/app/components/workflow/nodes/data-source/constants.ts @@ -23,13 +23,16 @@ export const DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE = [ 'html', ] -export const OUTPUT_VARIABLES_MAP = { - datasource_type: { +export const COMMON_OUTPUT = [ + { name: 'datasource_type', type: VarType.string, description: 'local_file, online_document, website_crawl', }, - file: { +] + +export const FILE_OUTPUT = [ + { name: 'file', type: VarType.file, description: 'file', @@ -76,4 +79,27 @@ export const OUTPUT_VARIABLES_MAP = { }, ], }, -} +] + +export const WEBSITE_OUTPUT = [ + { + name: 'source_url', + type: VarType.string, + description: 'The URL of the crawled website', + }, + { + name: 'content', + type: VarType.string, + description: 'The content of the crawled website', + }, + { + name: 'title', + type: VarType.string, + description: 'The title of the crawled website', + }, + { + name: 'description', + type: VarType.string, + description: 'The description of the crawled website', + }, +] diff --git a/web/app/components/workflow/nodes/data-source/default.ts b/web/app/components/workflow/nodes/data-source/default.ts index c6e73c5c34..5f46bed514 100644 --- a/web/app/components/workflow/nodes/data-source/default.ts +++ b/web/app/components/workflow/nodes/data-source/default.ts @@ -1,8 +1,13 @@ import type { NodeDefault } from '../../types' import type { DataSourceNodeType } from './types' +import { DataSourceClassification } from './types' import { genNodeMetaData } from '@/app/components/workflow/utils' import { BlockEnum } from '@/app/components/workflow/types' -import { OUTPUT_VARIABLES_MAP } from './constants' +import { + COMMON_OUTPUT, + FILE_OUTPUT, + WEBSITE_OUTPUT, +} from './constants' const metaData = genNodeMetaData({ sort: -1, @@ -24,20 +29,18 @@ const nodeDefault: NodeDefault = { const { provider_type, } = payload - const isLocalFile = provider_type === 'local_file' + const isLocalFile = provider_type === DataSourceClassification.file + const isWebsiteCrawl = provider_type === DataSourceClassification.website return [ - { - variable: OUTPUT_VARIABLES_MAP.datasource_type.name, - type: OUTPUT_VARIABLES_MAP.datasource_type.type, - }, + ...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })), ...( isLocalFile - ? [ - { - variable: OUTPUT_VARIABLES_MAP.file.name, - type: OUTPUT_VARIABLES_MAP.file.type, - }, - ] + ? FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) + : [] + ), + ...( + isWebsiteCrawl + ? WEBSITE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) : [] ), ...ragVars, diff --git a/web/app/components/workflow/nodes/data-source/panel.tsx b/web/app/components/workflow/nodes/data-source/panel.tsx index fb21678e53..c043de9df1 100644 --- a/web/app/components/workflow/nodes/data-source/panel.tsx +++ b/web/app/components/workflow/nodes/data-source/panel.tsx @@ -8,6 +8,7 @@ import { useTranslation } from 'react-i18next' import { memo } from 'react' import { useBoolean } from 'ahooks' import type { DataSourceNodeType } from './types' +import { DataSourceClassification } from './types' import type { NodePanelProps } from '@/app/components/workflow/types' import { BoxGroupField, @@ -17,7 +18,11 @@ import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/compo import TagInput from '@/app/components/base/tag-input' import { useNodesReadOnly } from '@/app/components/workflow/hooks' import { useConfig } from './hooks/use-config' -import { OUTPUT_VARIABLES_MAP } from './constants' +import { + COMMON_OUTPUT, + FILE_OUTPUT, + WEBSITE_OUTPUT, +} from './constants' import { useStore } from '@/app/components/workflow/store' import Button from '@/app/components/base/button' import ConfigCredential from './components/config-credential' @@ -35,7 +40,7 @@ const Panel: FC> = ({ id, data }) => { const dataSourceList = useStore(s => s.dataSourceList) const { provider_type, - provider_id, + plugin_id, fileExtensions = [], datasource_parameters, } = data @@ -43,8 +48,9 @@ const Panel: FC> = ({ id, data }) => { handleFileExtensionsChange, handleParametersChange, } = useConfig(id) - const isLocalFile = provider_type === 'local_file' - const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === provider_id) + const isLocalFile = provider_type === DataSourceClassification.file + const isWebsiteCrawl = provider_type === DataSourceClassification.website + const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id) const isAuthorized = !!currentDataSource?.is_authorized const [showAuthModal, { setTrue: openAuthModal, @@ -150,24 +156,37 @@ const Panel: FC> = ({ id, data }) => { ) } - { - isLocalFile && ( + COMMON_OUTPUT.map(item => ( ({ + name={item.name} + type={item.type} + description={item.description} + /> + )) + } + { + isLocalFile && FILE_OUTPUT.map(item => ( + ({ name: item.name, type: item.type, description: item.description, }))} /> - ) + )) + } + { + isWebsiteCrawl && WEBSITE_OUTPUT.map(item => ( + + )) } { diff --git a/web/app/components/workflow/nodes/data-source/types.ts b/web/app/components/workflow/nodes/data-source/types.ts index c4b5559dfb..0c49a2019e 100644 --- a/web/app/components/workflow/nodes/data-source/types.ts +++ b/web/app/components/workflow/nodes/data-source/types.ts @@ -6,6 +6,11 @@ export enum VarType { mixed = 'mixed', } +export enum DataSourceClassification { + file = 'local_file', + website = 'website_crawl', +} + export type ToolVarInputs = Record = { retry_config?: WorkflowRetryConfig default_value?: DefaultValueForm[] } & T & Partial> - & Partial> + & Partial> export type CommonEdgeType = { _hovering?: boolean