diff --git a/web/app/components/evaluation/__tests__/index.spec.tsx b/web/app/components/evaluation/__tests__/index.spec.tsx index 5146e49535..1346b363ce 100644 --- a/web/app/components/evaluation/__tests__/index.spec.tsx +++ b/web/app/components/evaluation/__tests__/index.spec.tsx @@ -1,5 +1,6 @@ import { act, fireEvent, render, screen } from '@testing-library/react' import Evaluation from '..' +import ConditionsSection from '../components/conditions-section' import { useEvaluationStore } from '../store' const mockUseAvailableEvaluationMetrics = vi.hoisted(() => vi.fn()) @@ -150,6 +151,50 @@ describe('Evaluation', () => { expect(screen.queryByPlaceholderText('evaluation.conditions.valuePlaceholder')).not.toBeInTheDocument() }) + it('should add a condition from grouped metric dropdown items', () => { + const resourceType = 'apps' + const resourceId = 'app-conditions-dropdown' + const store = useEvaluationStore.getState() + + act(() => { + store.ensureResource(resourceType, resourceId) + store.setJudgeModel(resourceType, resourceId, 'openai::gpt-4o-mini') + store.addBuiltinMetric(resourceType, resourceId, 'faithfulness', [ + { node_id: 'node-faithfulness', title: 'Retriever Node', type: 'retriever' }, + ]) + store.addCustomMetric(resourceType, resourceId) + + const customMetric = useEvaluationStore.getState().resources['apps:app-conditions-dropdown'].metrics.find(metric => metric.kind === 'custom-workflow')! + store.setCustomMetricWorkflow(resourceType, resourceId, customMetric.id, { + workflowId: 'workflow-1', + workflowAppId: 'workflow-app-1', + workflowName: 'Review Workflow', + }) + store.syncCustomMetricOutputs(resourceType, resourceId, customMetric.id, [{ + id: 'reason', + valueType: 'string', + }]) + }) + + render() + + fireEvent.click(screen.getByRole('combobox', { name: 'evaluation.conditions.addCondition' })) + + expect(screen.getByText('Faithfulness')).toBeInTheDocument() + expect(screen.getByText('Review Workflow')).toBeInTheDocument() + expect(screen.getByText('Retriever Node')).toBeInTheDocument() + expect(screen.getByText('reason')).toBeInTheDocument() + expect(screen.getByText('evaluation.conditions.valueTypes.number')).toBeInTheDocument() + expect(screen.getByText('evaluation.conditions.valueTypes.string')).toBeInTheDocument() + + fireEvent.click(screen.getByRole('option', { name: /reason/i })) + + const condition = useEvaluationStore.getState().resources['apps:app-conditions-dropdown'].judgmentConfig.conditions[0] + + expect(condition.variableSelector).toEqual(['workflow-1', 'reason']) + expect(screen.getAllByText('Review Workflow').length).toBeGreaterThan(0) + }) + it('should render the metric no-node empty state', () => { mockUseAvailableEvaluationMetrics.mockReturnValue({ data: { diff --git a/web/app/components/evaluation/__tests__/store.spec.ts b/web/app/components/evaluation/__tests__/store.spec.ts index 8f6e557d5a..eb7cd74b4d 100644 --- a/web/app/components/evaluation/__tests__/store.spec.ts +++ b/web/app/components/evaluation/__tests__/store.spec.ts @@ -132,6 +132,35 @@ describe('evaluation store', () => { expect(getAllowedOperators(state.metrics, condition.variableSelector)).toEqual(['=', '≠', '>', '<', '≥', '≤', 'is null', 'is not null']) }) + it('should add a condition from the selected custom metric output', () => { + const resourceType = 'apps' + const resourceId = 'app-condition-selector' + const store = useEvaluationStore.getState() + const config = getEvaluationMockConfig(resourceType) + + store.ensureResource(resourceType, resourceId) + store.addCustomMetric(resourceType, resourceId) + + const customMetric = useEvaluationStore.getState().resources['apps:app-condition-selector'].metrics.find(metric => metric.kind === 'custom-workflow')! + store.setCustomMetricWorkflow(resourceType, resourceId, customMetric.id, { + workflowId: config.workflowOptions[0].id, + workflowAppId: 'custom-workflow-app-id', + workflowName: config.workflowOptions[0].label, + }) + store.syncCustomMetricOutputs(resourceType, resourceId, customMetric.id, [{ + id: 'reason', + valueType: 'string', + }]) + + store.addCondition(resourceType, resourceId, [config.workflowOptions[0].id, 'reason']) + + const condition = useEvaluationStore.getState().resources['apps:app-condition-selector'].judgmentConfig.conditions[0] + + expect(condition.variableSelector).toEqual([config.workflowOptions[0].id, 'reason']) + expect(condition.comparisonOperator).toBe('contains') + expect(condition.value).toBeNull() + }) + it('should clear values for operators without values', () => { const resourceType = 'apps' const resourceId = 'app-3' diff --git a/web/app/components/evaluation/components/conditions-section/add-condition-select.tsx b/web/app/components/evaluation/components/conditions-section/add-condition-select.tsx new file mode 100644 index 0000000000..9e370e565a --- /dev/null +++ b/web/app/components/evaluation/components/conditions-section/add-condition-select.tsx @@ -0,0 +1,75 @@ +'use client' + +import type { ConditionMetricOptionGroup, EvaluationResourceProps } from '../../types' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { + Select, + SelectContent, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectTrigger, +} from '@/app/components/base/ui/select' +import { cn } from '@/utils/classnames' +import { useEvaluationStore } from '../../store' +import { getConditionMetricValueTypeTranslationKey } from '../../utils' + +type AddConditionSelectProps = EvaluationResourceProps & { + metricOptionGroups: ConditionMetricOptionGroup[] + disabled: boolean +} + +const AddConditionSelect = ({ + resourceType, + resourceId, + metricOptionGroups, + disabled, +}: AddConditionSelectProps) => { + const { t } = useTranslation('evaluation') + const addCondition = useEvaluationStore(state => state.addCondition) + const [selectKey, setSelectKey] = useState(0) + + return ( + + ) +} + +export default AddConditionSelect diff --git a/web/app/components/evaluation/components/conditions-section/condition-group.tsx b/web/app/components/evaluation/components/conditions-section/condition-group.tsx index 2825488893..7cb73f1ca7 100644 --- a/web/app/components/evaluation/components/conditions-section/condition-group.tsx +++ b/web/app/components/evaluation/components/conditions-section/condition-group.tsx @@ -24,6 +24,8 @@ import { getAllowedOperators, requiresConditionValue, useEvaluationResource, use import { buildConditionMetricOptions, getComparisonOperatorLabel, + getConditionMetricValueTypeTranslationKey, + groupConditionMetricOptions, isSelectorEqual, serializeVariableSelector, } from '../../utils' @@ -75,9 +77,9 @@ const ConditionMetricLabel = ({
- {metric.label} + {metric.itemLabel}
- {metric.group} + {metric.groupLabel}
) } @@ -88,11 +90,9 @@ const ConditionMetricSelect = ({ placeholder, onChange, }: ConditionMetricSelectProps) => { + const { t } = useTranslation('evaluation') const groupedMetricOptions = useMemo(() => { - return Object.entries(metricOptions.reduce>((acc, option) => { - acc[option.group] = [...(acc[option.group] ?? []), option] - return acc - }, {})) + return groupConditionMetricOptions(metricOptions) }, [metricOptions]) return ( @@ -108,15 +108,17 @@ const ConditionMetricSelect = ({ - {groupedMetricOptions.map(([groupName, options]) => ( - - {groupName} - {options.map(option => ( + {groupedMetricOptions.map(group => ( + + {group.label} + {group.options.map(option => ( -
+
- {option.label} - {option.description} + {option.itemLabel} + + {t(getConditionMetricValueTypeTranslationKey(option.valueType))} +
))} diff --git a/web/app/components/evaluation/components/conditions-section/index.tsx b/web/app/components/evaluation/components/conditions-section/index.tsx index d00830c1c8..fb28a56a38 100644 --- a/web/app/components/evaluation/components/conditions-section/index.tsx +++ b/web/app/components/evaluation/components/conditions-section/index.tsx @@ -3,10 +3,10 @@ import type { EvaluationResourceProps } from '../../types' import { useMemo } from 'react' import { useTranslation } from 'react-i18next' -import { cn } from '@/utils/classnames' -import { useEvaluationResource, useEvaluationStore } from '../../store' -import { buildConditionMetricOptions } from '../../utils' +import { useEvaluationResource } from '../../store' +import { buildConditionMetricOptions, groupConditionMetricOptions } from '../../utils' import { InlineSectionHeader } from '../section-header' +import AddConditionSelect from './add-condition-select' import ConditionGroup from './condition-group' const ConditionsSection = ({ @@ -15,8 +15,8 @@ const ConditionsSection = ({ }: EvaluationResourceProps) => { const { t } = useTranslation('evaluation') const resource = useEvaluationResource(resourceType, resourceId) - const addCondition = useEvaluationStore(state => state.addCondition) const conditionMetricOptions = useMemo(() => buildConditionMetricOptions(resource.metrics), [resource.metrics]) + const groupedConditionMetricOptions = useMemo(() => groupConditionMetricOptions(conditionMetricOptions), [conditionMetricOptions]) const canAddCondition = conditionMetricOptions.length > 0 return ( @@ -37,18 +37,12 @@ const ConditionsSection = ({ resourceId={resourceId} /> )} - + />
) diff --git a/web/app/components/evaluation/store-utils.ts b/web/app/components/evaluation/store-utils.ts index e29d2a6bdb..0f547e7488 100644 --- a/web/app/components/evaluation/store-utils.ts +++ b/web/app/components/evaluation/store-utils.ts @@ -330,8 +330,17 @@ export function createCustomMetric(): EvaluationMetric { } } -export const buildConditionItem = (metrics: EvaluationMetric[]): JudgmentConditionItem => { - const metricOption = buildConditionMetricOptions(metrics)[0] +export const buildConditionItem = ( + metrics: EvaluationMetric[], + variableSelector?: [string, string] | null, +): JudgmentConditionItem => { + const metricOptions = buildConditionMetricOptions(metrics) + const metricOption = variableSelector + ? metricOptions.find(option => + option.variableSelector[0] === variableSelector[0] + && option.variableSelector[1] === variableSelector[1], + ) ?? metricOptions[0] + : metricOptions[0] const comparisonOperator = metricOption ? getDefaultComparisonOperator(metricOption.valueType) : 'is' return { diff --git a/web/app/components/evaluation/store.ts b/web/app/components/evaluation/store.ts index 454c8ee569..cfc006b595 100644 --- a/web/app/components/evaluation/store.ts +++ b/web/app/components/evaluation/store.ts @@ -61,7 +61,11 @@ type EvaluationStore = { patch: { inputVariableId?: string | null, outputVariableId?: string | null }, ) => void setConditionLogicalOperator: (resourceType: EvaluationResourceType, resourceId: string, logicalOperator: 'and' | 'or') => void - addCondition: (resourceType: EvaluationResourceType, resourceId: string) => void + addCondition: ( + resourceType: EvaluationResourceType, + resourceId: string, + variableSelector?: [string, string] | null, + ) => void removeCondition: (resourceType: EvaluationResourceType, resourceId: string, conditionId: string) => void updateConditionMetric: (resourceType: EvaluationResourceType, resourceId: string, conditionId: string, variableSelector: [string, string]) => void updateConditionOperator: (resourceType: EvaluationResourceType, resourceId: string, conditionId: string, operator: ComparisonOperator) => void @@ -270,13 +274,13 @@ export const useEvaluationStore = create((set, get) => ({ })), })) }, - addCondition: (resourceType, resourceId) => { + addCondition: (resourceType, resourceId, variableSelector) => { set(state => ({ resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ ...resource, judgmentConfig: { ...resource.judgmentConfig, - conditions: [...resource.judgmentConfig.conditions, buildConditionItem(resource.metrics)], + conditions: [...resource.judgmentConfig.conditions, buildConditionItem(resource.metrics, variableSelector)], }, })), })) diff --git a/web/app/components/evaluation/types.ts b/web/app/components/evaluation/types.ts index acfcfdd11b..baa73f6c3f 100644 --- a/web/app/components/evaluation/types.ts +++ b/web/app/components/evaluation/types.ts @@ -112,13 +112,17 @@ export type JudgmentConfig = { export type ConditionMetricOption = { id: string - group: string - label: string - description: string + groupLabel: string + itemLabel: string valueType: ConditionMetricValueType variableSelector: [string, string] } +export type ConditionMetricOptionGroup = { + label: string + options: ConditionMetricOption[] +} + export type BatchTestRecord = { id: string fileName: string diff --git a/web/app/components/evaluation/utils.ts b/web/app/components/evaluation/utils.ts index 48b75d5c3e..3fba8158d6 100644 --- a/web/app/components/evaluation/utils.ts +++ b/web/app/components/evaluation/utils.ts @@ -2,6 +2,7 @@ import type { TFunction } from 'i18next' import type { ComparisonOperator, ConditionMetricOption, + ConditionMetricOptionGroup, ConditionMetricValueType, EvaluationMetric, } from './types' @@ -69,9 +70,8 @@ export const buildConditionMetricOptions = (metrics: EvaluationMetric[]): Condit return (metric.nodeInfoList ?? []).map((nodeInfo) => { return { id: `${nodeInfo.node_id}:${metric.optionId}`, - group: nodeInfo.title, - label: metric.label, - description: nodeInfo.type, + groupLabel: metric.label, + itemLabel: nodeInfo.title || nodeInfo.node_id, valueType: metric.valueType, variableSelector: [nodeInfo.node_id, metric.optionId] as [string, string], } @@ -86,9 +86,8 @@ export const buildConditionMetricOptions = (metrics: EvaluationMetric[]): Condit return customConfig.outputs.map((output) => { return { id: `${customConfig.workflowId}:${output.id}`, - group: customConfig.workflowName ?? metric.label, - label: output.id, - description: customConfig.workflowName ?? metric.label, + groupLabel: customConfig.workflowName ?? metric.label, + itemLabel: output.id, valueType: getMetricValueType(output.valueType), variableSelector: [customConfig.workflowId, output.id] as [string, string], } @@ -96,6 +95,30 @@ export const buildConditionMetricOptions = (metrics: EvaluationMetric[]): Condit }) } +export const groupConditionMetricOptions = (metricOptions: ConditionMetricOption[]): ConditionMetricOptionGroup[] => { + const groups = metricOptions.reduce>((acc, option) => { + acc.set(option.groupLabel, [...(acc.get(option.groupLabel) ?? []), option]) + return acc + }, new Map()) + + return Array.from(groups.entries()).map(([label, options]) => ({ + label, + options, + })) +} + +const conditionMetricValueTypeTranslationKeys = { + string: 'conditions.valueTypes.string', + number: 'conditions.valueTypes.number', + boolean: 'conditions.valueTypes.boolean', +} as const + +export const getConditionMetricValueTypeTranslationKey = ( + valueType: ConditionMetricValueType, +) => { + return conditionMetricValueTypeTranslationKeys[valueType] +} + export const serializeVariableSelector = (value: [string, string] | null | undefined) => { return value ? JSON.stringify(value) : '' } diff --git a/web/i18n/en-US/evaluation.json b/web/i18n/en-US/evaluation.json index 92f6b2a643..33191cd39a 100644 --- a/web/i18n/en-US/evaluation.json +++ b/web/i18n/en-US/evaluation.json @@ -43,6 +43,9 @@ "conditions.selectValue": "Choose a value", "conditions.title": "Judgment Conditions", "conditions.valuePlaceholder": "Enter a value", + "conditions.valueTypes.boolean": "Boolean", + "conditions.valueTypes.number": "Number", + "conditions.valueTypes.string": "String", "description": "Configure automated testing to grade your application's performance.", "history.columns.creator": "Creator", "history.columns.status": "Status", diff --git a/web/i18n/zh-Hans/evaluation.json b/web/i18n/zh-Hans/evaluation.json index 8a1ff78ebf..04883d95e1 100644 --- a/web/i18n/zh-Hans/evaluation.json +++ b/web/i18n/zh-Hans/evaluation.json @@ -43,6 +43,9 @@ "conditions.selectValue": "选择值", "conditions.title": "判定条件", "conditions.valuePlaceholder": "输入值", + "conditions.valueTypes.boolean": "布尔", + "conditions.valueTypes.number": "数值", + "conditions.valueTypes.string": "文本", "description": "配置自动化测试，对应用表现进行评分。", "judgeModel.description": "选择用于打分和判定评测结果的模型。", "judgeModel.title": "判定模型",