diff --git a/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts b/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts index eb2a862be6..0a12cf6557 100644 --- a/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts +++ b/web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts @@ -23,6 +23,7 @@ export const useNodesSyncDraft = () => { edges, transform, } = store.getState() + const nodes = getNodes() const [x, y, zoom] = transform const { pipelineId, @@ -32,8 +33,6 @@ export const useNodesSyncDraft = () => { } = workflowStore.getState() if (pipelineId) { - const nodes = getNodes() - const producedNodes = produce(nodes, (draft) => { draft.forEach((node) => { Object.keys(node.data).forEach((key) => { diff --git a/web/app/components/workflow/block-selector/data-sources.tsx b/web/app/components/workflow/block-selector/data-sources.tsx index 795adf5cc0..947c97a010 100644 --- a/web/app/components/workflow/block-selector/data-sources.tsx +++ b/web/app/components/workflow/block-selector/data-sources.tsx @@ -32,7 +32,7 @@ const DataSources = ({ const wrapElemRef = useRef(null) const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => { onSelect(BlockEnum.DataSource, toolDefaultValue && { - provider_id: toolDefaultValue?.provider_id, + plugin_id: toolDefaultValue?.provider_id, provider_type: toolDefaultValue?.provider_type, provider_name: toolDefaultValue?.provider_name, datasource_name: toolDefaultValue?.tool_name, diff --git a/web/app/components/workflow/block-selector/types.ts b/web/app/components/workflow/block-selector/types.ts index 22b9b7497e..75cfdf4818 100644 --- a/web/app/components/workflow/block-selector/types.ts +++ b/web/app/components/workflow/block-selector/types.ts @@ -36,7 +36,7 @@ export type ToolDefaultValue = { } export type DataSourceDefaultValue = { - provider_id: string + plugin_id: string provider_type: string provider_name: string datasource_name: string diff --git a/web/app/components/workflow/block-selector/utils.ts b/web/app/components/workflow/block-selector/utils.ts index 3e73d7a808..b1ad7b1256 100644 --- a/web/app/components/workflow/block-selector/utils.ts +++ b/web/app/components/workflow/block-selector/utils.ts @@ -5,7 +5,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => { return { id: dataSourceItem.plugin_id, provider: dataSourceItem.provider, - name: dataSourceItem.declaration.identity.name, + name: dataSourceItem.provider, author: dataSourceItem.declaration.identity.author, description: dataSourceItem.declaration.identity.description, icon: dataSourceItem.declaration.identity.icon, diff --git a/web/app/components/workflow/index.tsx b/web/app/components/workflow/index.tsx index beea4116cf..ad74e88df5 100644 --- a/web/app/components/workflow/index.tsx +++ b/web/app/components/workflow/index.tsx @@ -375,4 +375,4 @@ const WorkflowWithDefaultContext = ({ ) } -export default memo(WorkflowWithDefaultContext) +export default WorkflowWithDefaultContext diff --git a/web/app/components/workflow/nodes/data-source/constants.ts b/web/app/components/workflow/nodes/data-source/constants.ts index 10aa058a71..5b6de604f1 100644 --- a/web/app/components/workflow/nodes/data-source/constants.ts +++ b/web/app/components/workflow/nodes/data-source/constants.ts @@ -23,13 +23,16 @@ export const DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE = [ 'html', ] -export const OUTPUT_VARIABLES_MAP = { - datasource_type: { +export const COMMON_OUTPUT = [ + { name: 'datasource_type', type: VarType.string, description: 'local_file, online_document, website_crawl', }, - file: { +] + +export const FILE_OUTPUT = [ + { name: 'file', type: VarType.file, description: 'file', @@ -76,4 +79,27 @@ export const OUTPUT_VARIABLES_MAP = { }, ], }, -} +] + +export const WEBSITE_OUTPUT = [ + { + name: 'source_url', + type: VarType.string, + description: 'The URL of the crawled website', + }, + { + name: 'content', + type: VarType.string, + description: 'The content of the crawled website', + }, + { + name: 'title', + type: VarType.string, + description: 'The title of the crawled website', + }, + { + name: 'description', + type: VarType.string, + description: 'The description of the crawled website', + }, +] diff --git a/web/app/components/workflow/nodes/data-source/default.ts b/web/app/components/workflow/nodes/data-source/default.ts index c6e73c5c34..5f46bed514 100644 --- a/web/app/components/workflow/nodes/data-source/default.ts +++ b/web/app/components/workflow/nodes/data-source/default.ts @@ -1,8 +1,13 @@ import type { NodeDefault } from '../../types' import type { DataSourceNodeType } from './types' +import { DataSourceClassification } from './types' import { genNodeMetaData } from '@/app/components/workflow/utils' import { BlockEnum } from '@/app/components/workflow/types' -import { OUTPUT_VARIABLES_MAP } from './constants' +import { + COMMON_OUTPUT, + FILE_OUTPUT, + WEBSITE_OUTPUT, +} from './constants' const metaData = genNodeMetaData({ sort: -1, @@ -24,20 +29,18 @@ const nodeDefault: NodeDefault = { const { provider_type, } = payload - const isLocalFile = provider_type === 'local_file' + const isLocalFile = provider_type === DataSourceClassification.file + const isWebsiteCrawl = provider_type === DataSourceClassification.website return [ - { - variable: OUTPUT_VARIABLES_MAP.datasource_type.name, - type: OUTPUT_VARIABLES_MAP.datasource_type.type, - }, + ...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })), ...( isLocalFile - ? [ - { - variable: OUTPUT_VARIABLES_MAP.file.name, - type: OUTPUT_VARIABLES_MAP.file.type, - }, - ] + ? FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) + : [] + ), + ...( + isWebsiteCrawl + ? WEBSITE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) : [] ), ...ragVars, diff --git a/web/app/components/workflow/nodes/data-source/panel.tsx b/web/app/components/workflow/nodes/data-source/panel.tsx index fb21678e53..c043de9df1 100644 --- a/web/app/components/workflow/nodes/data-source/panel.tsx +++ b/web/app/components/workflow/nodes/data-source/panel.tsx @@ -8,6 +8,7 @@ import { useTranslation } from 'react-i18next' import { memo } from 'react' import { useBoolean } from 'ahooks' import type { DataSourceNodeType } from './types' +import { DataSourceClassification } from './types' import type { NodePanelProps } from '@/app/components/workflow/types' import { BoxGroupField, @@ -17,7 +18,11 @@ import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/compo import TagInput from '@/app/components/base/tag-input' import { useNodesReadOnly } from '@/app/components/workflow/hooks' import { useConfig } from './hooks/use-config' -import { OUTPUT_VARIABLES_MAP } from './constants' +import { + COMMON_OUTPUT, + FILE_OUTPUT, + WEBSITE_OUTPUT, +} from './constants' import { useStore } from '@/app/components/workflow/store' import Button from '@/app/components/base/button' import ConfigCredential from './components/config-credential' @@ -35,7 +40,7 @@ const Panel: FC> = ({ id, data }) => { const dataSourceList = useStore(s => s.dataSourceList) const { provider_type, - provider_id, + plugin_id, fileExtensions = [], datasource_parameters, } = data @@ -43,8 +48,9 @@ const Panel: FC> = ({ id, data }) => { handleFileExtensionsChange, handleParametersChange, } = useConfig(id) - const isLocalFile = provider_type === 'local_file' - const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === provider_id) + const isLocalFile = provider_type === DataSourceClassification.file + const isWebsiteCrawl = provider_type === DataSourceClassification.website + const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id) const isAuthorized = !!currentDataSource?.is_authorized const [showAuthModal, { setTrue: openAuthModal, @@ -150,24 +156,37 @@ const Panel: FC> = ({ id, data }) => { ) } - { - isLocalFile && ( + COMMON_OUTPUT.map(item => ( ({ + name={item.name} + type={item.type} + description={item.description} + /> + )) + } + { + isLocalFile && FILE_OUTPUT.map(item => ( + ({ name: item.name, type: item.type, description: item.description, }))} /> - ) + )) + } + { + isWebsiteCrawl && WEBSITE_OUTPUT.map(item => ( + + )) } { diff --git a/web/app/components/workflow/nodes/data-source/types.ts b/web/app/components/workflow/nodes/data-source/types.ts index c4b5559dfb..0c49a2019e 100644 --- a/web/app/components/workflow/nodes/data-source/types.ts +++ b/web/app/components/workflow/nodes/data-source/types.ts @@ -6,6 +6,11 @@ export enum VarType { mixed = 'mixed', } +export enum DataSourceClassification { + file = 'local_file', + website = 'website_crawl', +} + export type ToolVarInputs = Record = { retry_config?: WorkflowRetryConfig default_value?: DefaultValueForm[] } & T & Partial> - & Partial> + & Partial> export type CommonEdgeType = { _hovering?: boolean