diff --git a/web/app/components/evaluation/__tests__/store.spec.ts b/web/app/components/evaluation/__tests__/store.spec.ts index 18bd31a24c..88610c6ccb 100644 --- a/web/app/components/evaluation/__tests__/store.spec.ts +++ b/web/app/components/evaluation/__tests__/store.spec.ts @@ -30,9 +30,10 @@ describe('evaluation store', () => { workflowAppId: 'custom-workflow-app-id', workflowName: config.workflowOptions[0].label, }) - store.updateCustomMetricMapping(resourceType, resourceId, initialMetric!.id, initialMetric!.customConfig!.mappings[0].id, { - sourceFieldId: config.fieldOptions[0].id, - targetVariableId: config.workflowOptions[0].targetVariables[0].id, + store.syncCustomMetricMappings(resourceType, resourceId, initialMetric!.id, ['query']) + const syncedMetric = useEvaluationStore.getState().resources['apps:app-1'].metrics.find(metric => metric.id === initialMetric!.id) + store.updateCustomMetricMapping(resourceType, resourceId, initialMetric!.id, syncedMetric!.customConfig!.mappings[0].id, { + outputVariableId: 'answer', }) const configuredMetric = useEvaluationStore.getState().resources['apps:app-1'].metrics.find(metric => metric.id === initialMetric!.id) @@ -159,7 +160,7 @@ describe('evaluation store', () => { customized_metrics: { evaluation_workflow_id: 'workflow-precision-review', input_fields: { - 'app.input.query': 'query', + query: 'answer', }, }, judgement_conditions: [{ @@ -203,8 +204,8 @@ describe('evaluation store', () => { ]) expect(hydratedState.metrics[1].kind).toBe('custom-workflow') expect(hydratedState.metrics[1].customConfig?.workflowId).toBe('workflow-precision-review') - expect(hydratedState.metrics[1].customConfig?.mappings[0].sourceFieldId).toBe('app.input.query') - expect(hydratedState.metrics[1].customConfig?.mappings[0].targetVariableId).toBe('query') + expect(hydratedState.metrics[1].customConfig?.mappings[0].inputVariableId).toBe('query') + expect(hydratedState.metrics[1].customConfig?.mappings[0].outputVariableId).toBe('answer') expect(hydratedState.conditions[0].logicalOperator).toBe('or') expect(hydratedState.conditions[0].items[0]).toMatchObject({ fieldId: 'system.has_context', diff --git a/web/app/components/evaluation/components/custom-metric-editor/__tests__/index.spec.tsx b/web/app/components/evaluation/components/custom-metric-editor/__tests__/index.spec.tsx index 21063a0391..668447592c 100644 --- a/web/app/components/evaluation/components/custom-metric-editor/__tests__/index.spec.tsx +++ b/web/app/components/evaluation/components/custom-metric-editor/__tests__/index.spec.tsx @@ -1,21 +1,30 @@ import type { EvaluationMetric } from '../../../types' +import type { CodeNodeType } from '@/app/components/workflow/nodes/code/types' import type { EndNodeType } from '@/app/components/workflow/nodes/end/types' import type { StartNodeType } from '@/app/components/workflow/nodes/start/types' import type { Node } from '@/app/components/workflow/types' +import type { SnippetWorkflow } from '@/types/snippet' import type { FetchWorkflowDraftResponse } from '@/types/workflow' import { render, screen } from '@testing-library/react' -import { BlockEnum, VarType } from '@/app/components/workflow/types' +import { CodeLanguage } from '@/app/components/workflow/nodes/code/types' +import { BlockEnum, InputVarType, VarType } from '@/app/components/workflow/types' import CustomMetricEditorCard from '..' import { useEvaluationStore } from '../../../store' const mockUseAppWorkflow = vi.hoisted(() => vi.fn()) +const mockUseSnippetPublishedWorkflow = vi.hoisted(() => vi.fn()) const mockUseAvailableEvaluationWorkflows = vi.hoisted(() => vi.fn()) const mockUseInfiniteScroll = vi.hoisted(() => vi.fn()) +const mockPublishedGraphVariablePicker = vi.hoisted(() => vi.fn()) vi.mock('@/service/use-workflow', () => ({ useAppWorkflow: (...args: unknown[]) => mockUseAppWorkflow(...args), })) +vi.mock('@/service/use-snippet-workflows', () => ({ + useSnippetPublishedWorkflow: (...args: unknown[]) => mockUseSnippetPublishedWorkflow(...args), +})) + vi.mock('@/service/use-evaluation', () => ({ useAvailableEvaluationWorkflows: (...args: unknown[]) => mockUseAvailableEvaluationWorkflows(...args), })) @@ -24,6 +33,13 @@ vi.mock('ahooks', () => ({ useInfiniteScroll: (...args: unknown[]) => mockUseInfiniteScroll(...args), })) +vi.mock('../published-graph-variable-picker', () => ({ + default: (props: Record) => { + mockPublishedGraphVariablePicker(props) + return
+ }, +})) + const createStartNode = (): Node => ({ id: 'start-node', type: 'custom', @@ -32,7 +48,20 @@ const createStartNode = (): Node => ({ type: BlockEnum.Start, title: 'Start', desc: '', - variables: [], + variables: [ + { + variable: 'user_question', + label: 'User Question', + type: InputVarType.textInput, + required: true, + }, + { + variable: 'retrieved_context', + label: 'Retrieved Context', + type: InputVarType.textInput, + required: true, + }, + ], }, }) @@ -50,6 +79,33 @@ const createEndNode = ( }, }) +const createCodeNode = ( + id: string, + title: string, + outputs: Record, +): Node => ({ + id, + type: 'custom', + position: { x: 100, y: 0 }, + data: { + type: BlockEnum.Code, + title, + desc: '', + code: '', + code_language: CodeLanguage.python3, + outputs: Object.fromEntries( + Object.entries(outputs).map(([key, value]) => [ + key, + { + type: value.type, + children: null, + }, + ]), + ), + variables: [], + }, +}) + const createWorkflow = ( nodes: Node[], ): FetchWorkflowDraftResponse => ({ @@ -80,6 +136,20 @@ const createWorkflow = ( marked_comment: 'Published', }) +const createSnippetWorkflow = ( + nodes: Node[], +): SnippetWorkflow => ({ + id: 'snippet-workflow-1', + graph: { + nodes, + edges: [], + }, + features: {}, + hash: 'snippet-hash-1', + created_at: 1710000000, + updated_at: 1710000001, +}) + const createMetric = (): EvaluationMetric => ({ id: 'metric-1', optionId: 'custom-1', @@ -88,12 +158,16 @@ const createMetric = (): EvaluationMetric => ({ description: 'Map workflow variables to your evaluation inputs.', customConfig: { workflowId: 'workflow-1', - workflowAppId: 'app-1', + workflowAppId: 'workflow-app-1', workflowName: 'Evaluation Workflow', mappings: [{ id: 'mapping-1', - sourceFieldId: null, - targetVariableId: null, + inputVariableId: 'user_question', + outputVariableId: 'current-node.answer', + }, { + id: 'mapping-2', + inputVariableId: 'retrieved_context', + outputVariableId: 'current-node.score', }], }, }) @@ -102,6 +176,7 @@ describe('CustomMetricEditorCard', () => { beforeEach(() => { vi.clearAllMocks() useEvaluationStore.setState({ resources: {} }) + mockPublishedGraphVariablePicker.mockReset() mockUseInfiniteScroll.mockImplementation(() => undefined) mockUseAvailableEvaluationWorkflows.mockReturnValue({ @@ -119,48 +194,74 @@ describe('CustomMetricEditorCard', () => { isFetchingNextPage: false, isLoading: false, }) + mockUseSnippetPublishedWorkflow.mockReturnValue({ data: undefined }) }) - // Verify end-node outputs are shown after a workflow is selected. + // Verify the selected evaluation workflow still drives the output summary section. describe('Outputs', () => { it('should render the selected workflow outputs from the end node', () => { - mockUseAppWorkflow.mockReturnValue({ - data: createWorkflow([ - createStartNode(), - createEndNode([ - { variable: 'answer_score', value_selector: ['end', 'answer_score'], value_type: VarType.number }, - { variable: 'reason', value_selector: ['end', 'reason'], value_type: VarType.string }, - ]), + const selectedWorkflow = createWorkflow([ + createStartNode(), + createEndNode([ + { variable: 'answer_score', value_selector: ['end', 'answer_score'], value_type: VarType.number }, + { variable: 'reason', value_selector: ['end', 'reason'], value_type: VarType.string }, ]), + ]) + const currentAppWorkflow = createWorkflow([ + createCodeNode('current-node', 'Current Node', { + answer: { type: VarType.string }, + score: { type: VarType.number }, + }), + ]) + + mockUseAppWorkflow.mockImplementation((appId: string) => { + if (appId === 'workflow-app-1') + return { data: selectedWorkflow } + if (appId === 'app-under-test') + return { data: currentAppWorkflow } + + return { data: undefined } }) render( , ) expect(screen.getByText('evaluation.metrics.custom.outputTitle')).toBeInTheDocument() - expect(screen.getByText('answer_score')).toBeInTheDocument() - expect(screen.getByText('number')).toBeInTheDocument() - expect(screen.getByText('reason')).toBeInTheDocument() - expect(screen.getByText('string')).toBeInTheDocument() + expect(screen.getAllByText('answer_score').length).toBeGreaterThan(0) + expect(screen.getAllByText('number').length).toBeGreaterThan(0) + expect(screen.getAllByText('reason').length).toBeGreaterThan(0) + expect(screen.getAllByText('string').length).toBeGreaterThan(0) }) it('should hide the output section when the selected workflow has no end outputs', () => { - mockUseAppWorkflow.mockReturnValue({ - data: createWorkflow([ - createStartNode(), - createEndNode([]), - ]), + const selectedWorkflow = createWorkflow([ + createStartNode(), + createEndNode([]), + ]) + const currentAppWorkflow = createWorkflow([ + createCodeNode('current-node', 'Current Node', { + answer: { type: VarType.string }, + }), + ]) + + mockUseAppWorkflow.mockImplementation((appId: string) => { + if (appId === 'workflow-app-1') + return { data: selectedWorkflow } + if (appId === 'app-under-test') + return { data: currentAppWorkflow } + + return { data: undefined } }) render( , ) @@ -168,4 +269,94 @@ describe('CustomMetricEditorCard', () => { expect(screen.queryByText('evaluation.metrics.custom.outputTitle')).not.toBeInTheDocument() }) }) + + // Verify mapping rows use workflow start variables on the left and current published graph variables on the right. + describe('Variable Mapping', () => { + it('should pass the current app published graph and saved selector values to the picker', () => { + const selectedWorkflow = createWorkflow([ + createStartNode(), + createEndNode([ + { variable: 'answer_score', value_selector: ['end', 'answer_score'], value_type: VarType.number }, + { variable: 'reason', value_selector: ['end', 'reason'], value_type: VarType.string }, + ]), + ]) + const currentAppWorkflow = createWorkflow([ + createStartNode(), + createCodeNode('current-node', 'Current Node', { + answer: { type: VarType.string }, + score: { type: VarType.number }, + }), + ]) + + mockUseAppWorkflow.mockImplementation((appId: string) => { + if (appId === 'workflow-app-1') + return { data: selectedWorkflow } + if (appId === 'app-under-test') + return { data: currentAppWorkflow } + + return { data: undefined } + }) + + render( + , + ) + + expect(screen.getByText('user_question')).toBeInTheDocument() + expect(screen.getByText('retrieved_context')).toBeInTheDocument() + expect(screen.getAllByText('string')).toHaveLength(3) + expect(mockPublishedGraphVariablePicker).toHaveBeenCalledTimes(2) + expect(mockPublishedGraphVariablePicker.mock.calls[0][0]).toMatchObject({ + nodes: currentAppWorkflow.graph.nodes, + edges: currentAppWorkflow.graph.edges, + value: 'current-node.answer', + }) + expect(mockPublishedGraphVariablePicker.mock.calls[1][0]).toMatchObject({ + nodes: currentAppWorkflow.graph.nodes, + edges: currentAppWorkflow.graph.edges, + value: 'current-node.score', + }) + }) + + it('should use the current snippet published graph when editing a snippet evaluation', () => { + const selectedWorkflow = createWorkflow([ + createStartNode(), + createEndNode([ + { variable: 'reason', value_selector: ['end', 'reason'], value_type: VarType.string }, + ]), + ]) + const currentSnippetWorkflow = createSnippetWorkflow([ + createCodeNode('snippet-node', 'Snippet Node', { + result: { type: VarType.string }, + }), + ]) + + mockUseAppWorkflow.mockImplementation((appId: string) => { + if (appId === 'workflow-app-1') + return { data: selectedWorkflow } + + return { data: undefined } + }) + mockUseSnippetPublishedWorkflow.mockReturnValue({ + data: currentSnippetWorkflow, + }) + + render( + , + ) + + expect(mockPublishedGraphVariablePicker).toHaveBeenCalledTimes(2) + expect(mockPublishedGraphVariablePicker.mock.calls[0][0]).toMatchObject({ + nodes: currentSnippetWorkflow.graph.nodes, + edges: currentSnippetWorkflow.graph.edges, + }) + }) + }) }) diff --git a/web/app/components/evaluation/components/custom-metric-editor/index.tsx b/web/app/components/evaluation/components/custom-metric-editor/index.tsx index 7fa385a4ce..5235c06163 100644 --- a/web/app/components/evaluation/components/custom-metric-editor/index.tsx +++ b/web/app/components/evaluation/components/custom-metric-editor/index.tsx @@ -3,11 +3,12 @@ import type { EvaluationMetric, EvaluationResourceProps } from '../../types' import type { EndNodeType } from '@/app/components/workflow/nodes/end/types' import type { StartNodeType } from '@/app/components/workflow/nodes/start/types' -import type { Node } from '@/app/components/workflow/types' -import { useMemo } from 'react' +import type { Edge, InputVar, Node } from '@/app/components/workflow/types' +import { useEffect, useMemo } from 'react' import { useTranslation } from 'react-i18next' -import Button from '@/app/components/base/button' -import { BlockEnum } from '@/app/components/workflow/types' +import { inputVarTypeToVarType } from '@/app/components/workflow/nodes/_base/components/variable/utils' +import { BlockEnum, InputVarType } from '@/app/components/workflow/types' +import { useSnippetPublishedWorkflow } from '@/service/use-snippet-workflows' import { useAppWorkflow } from '@/service/use-workflow' import { isCustomMetricConfigured, useEvaluationStore } from '../../store' import MappingRow from './mapping-row' @@ -17,16 +18,16 @@ type CustomMetricEditorCardProps = EvaluationResourceProps & { metric: EvaluationMetric } -const getWorkflowTargetVariables = ( +const getWorkflowInputVariables = ( nodes?: Array, ) => { const startNode = nodes?.find(node => node.data.type === BlockEnum.Start) as Node | undefined if (!startNode || !Array.isArray(startNode.data.variables)) return [] - return startNode.data.variables.map(variable => ({ + return startNode.data.variables.map((variable: InputVar) => ({ id: variable.variable, - label: typeof variable.label === 'string' ? variable.label : variable.variable, + valueType: inputVarTypeToVarType(variable.type ?? InputVarType.textInput), })) } @@ -39,9 +40,11 @@ const getWorkflowOutputs = (nodes?: Array) => { return [] return endNode.data.outputs + .filter(output => typeof output.variable === 'string' && !!output.variable) .map(output => ({ - variable: output.variable, - valueType: output.value_type, + id: output.variable, + valueType: typeof output.value_type === 'string' ? output.value_type : null, + nodeTitle: typeof endNode.data.title === 'string' && endNode.data.title ? endNode.data.title : 'End', })) }) } @@ -54,6 +57,14 @@ const getWorkflowName = (workflow: { return workflow.marked_name || workflow.app_name || workflow.id } +const getGraphNodes = (graph?: Record) => { + return Array.isArray(graph?.nodes) ? graph.nodes as Node[] : [] +} + +const getGraphEdges = (graph?: Record) => { + return Array.isArray(graph?.edges) ? graph.edges as Edge[] : [] +} + const CustomMetricEditorCard = ({ resourceType, resourceId, @@ -61,18 +72,60 @@ const CustomMetricEditorCard = ({ }: CustomMetricEditorCardProps) => { const { t } = useTranslation('evaluation') const setCustomMetricWorkflow = useEvaluationStore(state => state.setCustomMetricWorkflow) - const addCustomMetricMapping = useEvaluationStore(state => state.addCustomMetricMapping) + const syncCustomMetricMappings = useEvaluationStore(state => state.syncCustomMetricMappings) const updateCustomMetricMapping = useEvaluationStore(state => state.updateCustomMetricMapping) - const removeCustomMetricMapping = useEvaluationStore(state => state.removeCustomMetricMapping) const { data: selectedWorkflow } = useAppWorkflow(metric.customConfig?.workflowAppId ?? '') - const targetOptions = useMemo(() => { - return getWorkflowTargetVariables(selectedWorkflow?.graph.nodes) + const { data: currentAppWorkflow } = useAppWorkflow(resourceType === 'apps' ? resourceId : '') + const { data: currentSnippetWorkflow } = useSnippetPublishedWorkflow(resourceType === 'snippets' ? resourceId : '') + const inputVariables = useMemo(() => { + return getWorkflowInputVariables(selectedWorkflow?.graph.nodes) }, [selectedWorkflow?.graph.nodes]) const workflowOutputs = useMemo(() => { return getWorkflowOutputs(selectedWorkflow?.graph.nodes) }, [selectedWorkflow?.graph.nodes]) + const publishedGraph = useMemo(() => { + if (resourceType === 'apps') { + return { + nodes: currentAppWorkflow?.graph.nodes ?? [], + edges: currentAppWorkflow?.graph.edges ?? [], + environmentVariables: currentAppWorkflow?.environment_variables ?? [], + conversationVariables: currentAppWorkflow?.conversation_variables ?? [], + } + } + + return { + nodes: getGraphNodes(currentSnippetWorkflow?.graph), + edges: getGraphEdges(currentSnippetWorkflow?.graph), + environmentVariables: [], + conversationVariables: [], + } + }, [ + currentAppWorkflow?.conversation_variables, + currentAppWorkflow?.environment_variables, + currentAppWorkflow?.graph.edges, + currentAppWorkflow?.graph.nodes, + currentSnippetWorkflow?.graph, + resourceType, + ]) + const inputVariableIds = useMemo(() => inputVariables.map(variable => variable.id), [inputVariables]) const isConfigured = isCustomMetricConfigured(metric) + useEffect(() => { + if (!metric.customConfig?.workflowId) + return + + const currentInputVariableIds = metric.customConfig.mappings + .map(mapping => mapping.inputVariableId) + .filter((value): value is string => !!value) + + if (currentInputVariableIds.length === inputVariableIds.length + && currentInputVariableIds.every((value, index) => value === inputVariableIds[index])) { + return + } + + syncCustomMetricMappings(resourceType, resourceId, metric.id, inputVariableIds) + }, [inputVariableIds, metric.customConfig?.mappings, metric.customConfig?.workflowId, metric.id, resourceId, resourceType, syncCustomMetricMappings]) + if (!metric.customConfig) return null @@ -88,6 +141,37 @@ const CustomMetricEditorCard = ({ })} /> +
+
+
{t('metrics.custom.mappingTitle')}
+
+
+ {inputVariables.map((inputVariable) => { + const mapping = metric.customConfig?.mappings.find(item => item.inputVariableId === inputVariable.id) + + return ( + { + if (!mapping) + return + + updateCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id, { outputVariableId }) + }} + /> + ) + })} +
+ {!isConfigured && ( +
+ {t('metrics.custom.mappingWarning')} +
+ )} +
+ {!!workflowOutputs.length && (
@@ -95,8 +179,8 @@ const CustomMetricEditorCard = ({
{workflowOutputs.map((output, index) => ( -
- {output.variable} +
+ {output.id} {output.valueType && ( {output.valueType} )} @@ -108,38 +192,6 @@ const CustomMetricEditorCard = ({
)} - -
-
-
{t('metrics.custom.mappingTitle')}
- -
-
- {metric.customConfig.mappings.map(mapping => ( - updateCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id, patch)} - onRemove={() => removeCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id)} - /> - ))} -
- {!isConfigured && ( -
- {t('metrics.custom.mappingWarning')} -
- )} -
) } diff --git a/web/app/components/evaluation/components/custom-metric-editor/mapping-row.tsx b/web/app/components/evaluation/components/custom-metric-editor/mapping-row.tsx index 782e67e321..1340682674 100644 --- a/web/app/components/evaluation/components/custom-metric-editor/mapping-row.tsx +++ b/web/app/components/evaluation/components/custom-metric-editor/mapping-row.tsx @@ -1,74 +1,62 @@ 'use client' -import type { CustomMetricMapping, EvaluationResourceType } from '../../types' +import type { + ConversationVariable, + Edge, + EnvironmentVariable, + Node, +} from '@/app/components/workflow/types' import { useTranslation } from 'react-i18next' -import Button from '@/app/components/base/button' -import { - Select, - SelectContent, - SelectGroup, - SelectGroupLabel, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/app/components/base/ui/select' -import { getEvaluationMockConfig } from '../../mock' -import { groupFieldOptions } from '../../utils' +import { Variable02 } from '@/app/components/base/icons/src/vender/solid/development' +import PublishedGraphVariablePicker from './published-graph-variable-picker' type MappingRowProps = { - resourceType: EvaluationResourceType - mapping: CustomMetricMapping - targetOptions: Array<{ id: string, label: string }> - onUpdate: (patch: { sourceFieldId?: string | null, targetVariableId?: string | null }) => void - onRemove: () => void + inputVariable: { + id: string + valueType: string + } + publishedGraph: { + nodes: Node[] + edges: Edge[] + environmentVariables: EnvironmentVariable[] + conversationVariables: ConversationVariable[] + } + value: string | null + onUpdate: (outputVariableId: string | null) => void } const MappingRow = ({ - resourceType, - mapping, - targetOptions, + inputVariable, + publishedGraph, + value, onUpdate, - onRemove, }: MappingRowProps) => { const { t } = useTranslation('evaluation') - const config = getEvaluationMockConfig(resourceType) return ( -
- - -
-