diff --git a/web/app/components/evaluation/components/batch-test-panel.tsx b/web/app/components/evaluation/components/batch-test-panel.tsx new file mode 100644 index 0000000000..4a03de1b38 --- /dev/null +++ b/web/app/components/evaluation/components/batch-test-panel.tsx @@ -0,0 +1,164 @@ +'use client' + +import type { EvaluationResourceProps } from '../types' +import { useRef } from 'react' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import Button from '@/app/components/base/button' +import { toast } from '@/app/components/base/ui/toast' +import { cn } from '@/utils/classnames' +import { getEvaluationMockConfig } from '../mock' +import { isEvaluationRunnable, useEvaluationResource, useEvaluationStore } from '../store' +import { TAB_CLASS_NAME } from '../utils' + +const BatchTestPanel = ({ + resourceType, + resourceId, +}: EvaluationResourceProps) => { + const { t } = useTranslation('evaluation') + const config = getEvaluationMockConfig(resourceType) + const tabLabels = { + 'input-fields': t('batch.tabs.input-fields'), + 'history': t('batch.tabs.history'), + } + const statusLabels = { + running: t('batch.status.running'), + success: t('batch.status.success'), + failed: t('batch.status.failed'), + } + const resource = useEvaluationResource(resourceType, resourceId) + const setBatchTab = useEvaluationStore(state => state.setBatchTab) + const setUploadedFileName = useEvaluationStore(state => state.setUploadedFileName) + const runBatchTest = useEvaluationStore(state => state.runBatchTest) + const fileInputRef = useRef(null) + const isRunnable = isEvaluationRunnable(resource) + + const handleDownloadTemplate = () => { + const content = ['case_id,input,expected', '1,Example input,Example output'].join('\n') + const link = document.createElement('a') + link.href = `data:text/csv;charset=utf-8,${encodeURIComponent(content)}` + link.download = config.templateFileName + link.click() + } + + const handleRun = () => { + if (!isRunnable) { + toast.warning(t('batch.validation')) + return + } + + runBatchTest(resourceType, resourceId) + } + + return ( +
+
+
+
+
+
{t('batch.noticeTitle')}
+
{t('batch.noticeDescription')}
+
+
+ {(['input-fields', 'history'] as const).map(tab => ( + + ))} +
+
+
+ {resource.activeBatchTab === 'input-fields' && ( +
+
+
{t('batch.requirementsTitle')}
+
+ {config.batchRequirements.map(requirement => ( +
+ + {requirement} +
+ ))} +
+
+
+ + { + const file = event.target.files?.[0] + setUploadedFileName(resourceType, resourceId, file?.name ?? null) + }} + /> + +
+ {!isRunnable && ( +
+ {t('batch.validation')} +
+ )} + +
+ )} + {resource.activeBatchTab === 'history' && ( +
+ {resource.batchRecords.length === 0 && ( +
+ {t('batch.emptyHistory')} +
+ )} + {resource.batchRecords.map(record => ( +
+
+
+
{record.summary}
+
{record.fileName}
+
+ + {record.status === 'running' + ? ( + + + ) + : statusLabels[record.status]} + +
+
{record.startedAt}
+
+ ))} +
+ )} +
+
+ ) +} + +export default BatchTestPanel diff --git a/web/app/components/evaluation/components/condition-group.tsx b/web/app/components/evaluation/components/condition-group.tsx new file mode 100644 index 0000000000..a678776054 --- /dev/null +++ b/web/app/components/evaluation/components/condition-group.tsx @@ -0,0 +1,344 @@ +'use client' + +import type { + ComparisonOperator, + EvaluationFieldOption, + EvaluationResourceProps, + JudgmentConditionGroup, +} from '../types' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import Button from '@/app/components/base/button' +import DatePicker from '@/app/components/base/date-and-time-picker/date-picker' +import dayjs from '@/app/components/base/date-and-time-picker/utils/dayjs' +import Input from '@/app/components/base/input' +import { + Select, + SelectContent, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/app/components/base/ui/select' +import { cn } from '@/utils/classnames' +import { getEvaluationMockConfig } from '../mock' +import { getAllowedOperators, requiresConditionValue, useEvaluationStore } from '../store' +import { getFieldTypeIconClassName, getOperatorLabel, groupFieldOptions } from '../utils' + +type ConditionFieldLabelProps = { + field?: EvaluationFieldOption + placeholder: string +} + +type ConditionFieldSelectProps = { + field?: EvaluationFieldOption + fieldOptions: EvaluationFieldOption[] + placeholder: string + onChange: (fieldId: string) => void +} + +type ConditionOperatorSelectProps = { + field?: EvaluationFieldOption + operator: ComparisonOperator + operators: ComparisonOperator[] + onChange: (operator: ComparisonOperator) => void +} + +type FieldValueInputProps = { + field?: EvaluationFieldOption + operator: ComparisonOperator + value: string | number | boolean | null + onChange: (value: string | number | boolean | null) => void +} + +type ConditionGroupProps = EvaluationResourceProps & { + group: JudgmentConditionGroup + index: number +} + +const ConditionFieldLabel = ({ + field, + placeholder, +}: ConditionFieldLabelProps) => { + if (!field) + return {placeholder} + + return ( +
+
+ + {field.label} +
+ {field.type} +
+ ) +} + +const ConditionFieldSelect = ({ + field, + fieldOptions, + placeholder, + onChange, +}: ConditionFieldSelectProps) => { + return ( + + ) +} + +const ConditionOperatorSelect = ({ + field, + operator, + operators, + onChange, +}: ConditionOperatorSelectProps) => { + const { t } = useTranslation('evaluation') + + return ( + + ) +} + +const FieldValueInput = ({ + field, + operator, + value, + onChange, +}: FieldValueInputProps) => { + const { t } = useTranslation('evaluation') + + if (!field || !requiresConditionValue(operator)) + return null + + if (field.type === 'time') { + const selectedTime = typeof value === 'string' && value ? dayjs(value) : undefined + + return ( +
+ onChange(date ? date.toISOString() : null)} + onClear={() => onChange(null)} + placeholder={t('conditions.selectTime')} + triggerWrapClassName="w-full" + popupZIndexClassname="z-[1002]" + renderTrigger={({ handleClickTrigger }) => ( + + )} + /> +
+ ) + } + + if (field.type === 'boolean') { + return ( +
+ +
+ ) + } + + if (field.type === 'enum') { + return ( +
+ +
+ ) + } + + return ( +
+ { + if (field.type === 'number') { + const nextValue = e.target.value + onChange(nextValue === '' ? null : Number(nextValue)) + return + } + + onChange(e.target.value) + }} + /> +
+ ) +} + +const ConditionGroup = ({ + resourceType, + resourceId, + group, + index, +}: ConditionGroupProps) => { + const { t } = useTranslation('evaluation') + const config = getEvaluationMockConfig(resourceType) + const logicalLabels = { + and: t('conditions.logical.and'), + or: t('conditions.logical.or'), + } + const removeConditionGroup = useEvaluationStore(state => state.removeConditionGroup) + const setConditionGroupOperator = useEvaluationStore(state => state.setConditionGroupOperator) + const addConditionItem = useEvaluationStore(state => state.addConditionItem) + const removeConditionItem = useEvaluationStore(state => state.removeConditionItem) + const updateConditionField = useEvaluationStore(state => state.updateConditionField) + const updateConditionOperator = useEvaluationStore(state => state.updateConditionOperator) + const updateConditionValue = useEvaluationStore(state => state.updateConditionValue) + + return ( +
+
+
+ {t('conditions.groupLabel', { index: index + 1 })} +
+ {(['and', 'or'] as const).map(operator => ( + + ))} +
+
+
+ + +
+
+
+ {group.items.map((item) => { + const field = config.fieldOptions.find(option => option.id === item.fieldId) + const allowedOperators = getAllowedOperators(resourceType, item.fieldId) + const showValue = !!field && requiresConditionValue(item.operator) + + return ( +
+
+
+
+ updateConditionField(resourceType, resourceId, group.id, item.id, value)} + /> +
+
+ updateConditionOperator(resourceType, resourceId, group.id, item.id, value)} + /> +
+ {showValue && ( +
+ updateConditionValue(resourceType, resourceId, group.id, item.id, value)} + /> +
+ )} +
+
+ +
+
+ ) + })} +
+
+ ) +} + +export default ConditionGroup diff --git a/web/app/components/evaluation/components/conditions-section.tsx b/web/app/components/evaluation/components/conditions-section.tsx new file mode 100644 index 0000000000..192ef17788 --- /dev/null +++ b/web/app/components/evaluation/components/conditions-section.tsx @@ -0,0 +1,51 @@ +'use client' + +import type { EvaluationResourceProps } from '../types' +import { useTranslation } from 'react-i18next' +import Button from '@/app/components/base/button' +import { useEvaluationResource, useEvaluationStore } from '../store' +import ConditionGroup from './condition-group' +import SectionHeader from './section-header' + +const ConditionsSection = ({ + resourceType, + resourceId, +}: EvaluationResourceProps) => { + const { t } = useTranslation('evaluation') + const resource = useEvaluationResource(resourceType, resourceId) + const addConditionGroup = useEvaluationStore(state => state.addConditionGroup) + + return ( +
+ addConditionGroup(resourceType, resourceId)}> +
+ ) +} + +export default ConditionsSection diff --git a/web/app/components/evaluation/components/custom-metric-editor.tsx b/web/app/components/evaluation/components/custom-metric-editor.tsx new file mode 100644 index 0000000000..2c35566bdd --- /dev/null +++ b/web/app/components/evaluation/components/custom-metric-editor.tsx @@ -0,0 +1,151 @@ +'use client' + +import type { CustomMetricMapping, EvaluationMetric, EvaluationResourceProps, EvaluationResourceType } from '../types' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import Button from '@/app/components/base/button' +import { + Select, + SelectContent, + SelectGroup, + SelectGroupLabel, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/app/components/base/ui/select' +import { getEvaluationMockConfig } from '../mock' +import { isCustomMetricConfigured, useEvaluationStore } from '../store' +import { groupFieldOptions } from '../utils' + +type CustomMetricEditorProps = EvaluationResourceProps & { + metric: EvaluationMetric +} + +type MappingRowProps = { + resourceType: EvaluationResourceType + mapping: CustomMetricMapping + targetOptions: Array<{ id: string, label: string }> + onUpdate: (patch: { sourceFieldId?: string | null, targetVariableId?: string | null }) => void + onRemove: () => void +} + +function MappingRow({ + resourceType, + mapping, + targetOptions, + onUpdate, + onRemove, +}: MappingRowProps) { + const { t } = useTranslation('evaluation') + const config = getEvaluationMockConfig(resourceType) + + return ( +
+ +
+
+ + +
+ ) +} + +const CustomMetricEditor = ({ + resourceType, + resourceId, + metric, +}: CustomMetricEditorProps) => { + const { t } = useTranslation('evaluation') + const config = getEvaluationMockConfig(resourceType) + const setCustomMetricWorkflow = useEvaluationStore(state => state.setCustomMetricWorkflow) + const addCustomMetricMapping = useEvaluationStore(state => state.addCustomMetricMapping) + const updateCustomMetricMapping = useEvaluationStore(state => state.updateCustomMetricMapping) + const removeCustomMetricMapping = useEvaluationStore(state => state.removeCustomMetricMapping) + const selectedWorkflow = config.workflowOptions.find(option => option.id === metric.customConfig?.workflowId) + const isConfigured = isCustomMetricConfigured(metric) + + if (!metric.customConfig) + return null + + return ( +
+
+
+
{t('metrics.custom.title')}
+
{t('metrics.custom.description')}
+
+ {!isConfigured && {t('metrics.custom.warningBadge')}} +
+
+
+
{t('metrics.custom.workflowLabel')}
+ + {selectedWorkflow &&
{selectedWorkflow.description}
} +
+
+
+
{t('metrics.custom.mappingTitle')}
+ +
+
+ {metric.customConfig.mappings.map(mapping => ( + updateCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id, patch)} + onRemove={() => removeCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id)} + /> + ))} +
+ {!isConfigured && ( +
+ {t('metrics.custom.mappingWarning')} +
+ )} +
+
+
+ ) +} + +export default CustomMetricEditor diff --git a/web/app/components/evaluation/components/judge-model-selector.tsx b/web/app/components/evaluation/components/judge-model-selector.tsx new file mode 100644 index 0000000000..27520e4aca --- /dev/null +++ b/web/app/components/evaluation/components/judge-model-selector.tsx @@ -0,0 +1,43 @@ +'use client' + +import type { EvaluationResourceProps } from '../types' +import { useEffect } from 'react' +import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' +import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks' +import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' +import { useEvaluationResource, useEvaluationStore } from '../store' +import { decodeModelSelection, encodeModelSelection } from '../utils' + +const JudgeModelSelector = ({ + resourceType, + resourceId, +}: EvaluationResourceProps) => { + const { data: modelList } = useModelList(ModelTypeEnum.textGeneration) + const resource = useEvaluationResource(resourceType, resourceId) + const setJudgeModel = useEvaluationStore(state => state.setJudgeModel) + const selectedModel = decodeModelSelection(resource.judgeModelId) + + useEffect(() => { + if (resource.judgeModelId || !modelList.length) + return + + const firstProvider = modelList[0] + const firstModel = firstProvider.models[0] + if (!firstProvider || !firstModel) + return + + setJudgeModel(resourceType, resourceId, encodeModelSelection(firstProvider.provider, firstModel.model)) + }, [modelList, resource.judgeModelId, resourceId, resourceType, setJudgeModel]) + + return ( + setJudgeModel(resourceType, resourceId, encodeModelSelection(model.provider, model.model))} + showDeprecatedWarnIcon + triggerClassName="h-11" + /> + ) +} + +export default JudgeModelSelector diff --git a/web/app/components/evaluation/components/metric-section.tsx b/web/app/components/evaluation/components/metric-section.tsx new file mode 100644 index 0000000000..0cb587f355 --- /dev/null +++ b/web/app/components/evaluation/components/metric-section.tsx @@ -0,0 +1,63 @@ +'use client' + +import type { EvaluationResourceProps } from '../types' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import Button from '@/app/components/base/button' +import { useEvaluationResource, useEvaluationStore } from '../store' +import CustomMetricEditor from './custom-metric-editor' +import MetricSelector from './metric-selector' +import SectionHeader from './section-header' + +const MetricSection = ({ + resourceType, + resourceId, +}: EvaluationResourceProps) => { + const { t } = useTranslation('evaluation') + const resource = useEvaluationResource(resourceType, resourceId) + const removeMetric = useEvaluationStore(state => state.removeMetric) + + return ( +
+ } + /> +
+ {resource.metrics.map(metric => ( +
+
+
+
{metric.label}
+
{metric.description}
+
+ {metric.badges.map(badge => ( + {badge} + ))} +
+
+ +
+ {metric.kind === 'custom-workflow' && ( + + )} +
+ ))} +
+
+ ) +} + +export default MetricSection diff --git a/web/app/components/evaluation/components/metric-selector.tsx b/web/app/components/evaluation/components/metric-selector.tsx new file mode 100644 index 0000000000..8c9e5e413f --- /dev/null +++ b/web/app/components/evaluation/components/metric-selector.tsx @@ -0,0 +1,181 @@ +'use client' + +import type { ChangeEvent } from 'react' +import type { EvaluationResourceProps } from '../types' +import { useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import Badge from '@/app/components/base/badge' +import Button from '@/app/components/base/button' +import Input from '@/app/components/base/input' +import { + Popover, + PopoverContent, + PopoverTrigger, +} from '@/app/components/base/ui/popover' +import { cn } from '@/utils/classnames' +import { getEvaluationMockConfig } from '../mock' +import { useEvaluationResource, useEvaluationStore } from '../store' + +const MetricSelector = ({ + resourceType, + resourceId, +}: EvaluationResourceProps) => { + const { t } = useTranslation('evaluation') + const config = getEvaluationMockConfig(resourceType) + const metricGroupLabels = { + quality: t('metrics.groups.quality'), + operations: t('metrics.groups.operations'), + } + const metrics = useEvaluationResource(resourceType, resourceId).metrics + const addBuiltinMetric = useEvaluationStore(state => state.addBuiltinMetric) + const addCustomMetric = useEvaluationStore(state => state.addCustomMetric) + const [open, setOpen] = useState(false) + const [query, setQuery] = useState('') + const [showAll, setShowAll] = useState(false) + const [isLoading, setIsLoading] = useState(false) + const loadingTimerRef = useRef(null) + + const triggerLoading = () => { + if (loadingTimerRef.current) + window.clearTimeout(loadingTimerRef.current) + + setIsLoading(true) + loadingTimerRef.current = window.setTimeout(() => { + setIsLoading(false) + }, 180) + } + + const handleOpenChange = (nextOpen: boolean) => { + setOpen(nextOpen) + + if (nextOpen) { + triggerLoading() + return + } + + if (loadingTimerRef.current) + window.clearTimeout(loadingTimerRef.current) + setIsLoading(false) + } + + const handleQueryChange = (event: ChangeEvent) => { + setQuery(event.target.value) + if (open) + triggerLoading() + } + + useEffect(() => { + return () => { + if (loadingTimerRef.current) + window.clearTimeout(loadingTimerRef.current) + } + }, []) + + const filteredGroups = useMemo(() => { + const filteredMetrics = config.builtinMetrics.filter((metric) => { + const keyword = query.trim().toLowerCase() + if (!keyword) + return true + + return metric.label.toLowerCase().includes(keyword) || metric.description.toLowerCase().includes(keyword) + }) + + const grouped = filteredMetrics.reduce>((acc, metric) => { + acc[metric.group] = [...(acc[metric.group] ?? []), metric] + return acc + }, {}) + + return Object.entries(grouped) + }, [config.builtinMetrics, query]) + + return ( + + + + +
+ +
+ {isLoading && ( +
+ {['metric-skeleton-1', 'metric-skeleton-2', 'metric-skeleton-3'].map(key => ( +
+ ))} +
+ )} + {!isLoading && filteredGroups.length === 0 && ( +
+ {t('metrics.noResults')} +
+ )} + {!isLoading && filteredGroups.map(([groupName, options]) => { + const shownOptions = showAll ? options : options.slice(0, 2) + return ( +
+
{metricGroupLabels[groupName as keyof typeof metricGroupLabels] ?? groupName}
+
+ {shownOptions.map(option => ( + + ))} +
+
+ ) + })} +
+ {filteredGroups.some(([, options]) => options.length > 2) && ( + + )} +
+ +
+
+ + + ) +} + +export default MetricSelector diff --git a/web/app/components/evaluation/components/section-header.tsx b/web/app/components/evaluation/components/section-header.tsx new file mode 100644 index 0000000000..e0913b76a4 --- /dev/null +++ b/web/app/components/evaluation/components/section-header.tsx @@ -0,0 +1,27 @@ +'use client' + +import type { ReactNode } from 'react' + +type SectionHeaderProps = { + title: string + description: string + action?: ReactNode +} + +const SectionHeader = ({ + title, + description, + action, +}: SectionHeaderProps) => { + return ( +
+
+
{title}
+
{description}
+
+ {action} +
+ ) +} + +export default SectionHeader diff --git a/web/app/components/evaluation/index.tsx b/web/app/components/evaluation/index.tsx index 798e092eed..0f734ea9cc 100644 --- a/web/app/components/evaluation/index.tsx +++ b/web/app/components/evaluation/index.tsx @@ -1,923 +1,21 @@ 'use client' -import type { TFunction } from 'i18next' -import type { ChangeEvent, ReactNode } from 'react' -import type { - ComparisonOperator, - CustomMetricMapping, - EvaluationFieldOption, - EvaluationMetric, - EvaluationResourceType, - JudgmentConditionGroup, -} from './types' -import { - RiAddLine, - RiArrowDownSLine, - RiCloseLine, - RiDeleteBinLine, - RiDownloadLine, - RiFileUploadLine, - RiFlaskLine, - RiLoader4Line, -} from '@remixicon/react' -import { useEffect, useMemo, useRef, useState } from 'react' +import type { EvaluationResourceProps } from './types' +import { useEffect } from 'react' import { useTranslation } from 'react-i18next' -import Badge from '@/app/components/base/badge' -import Button from '@/app/components/base/button' -import DatePicker from '@/app/components/base/date-and-time-picker/date-picker' -import dayjs from '@/app/components/base/date-and-time-picker/utils/dayjs' -import Input from '@/app/components/base/input' -import Toast from '@/app/components/base/toast' -import { - Popover, - PopoverContent, - PopoverTrigger, -} from '@/app/components/base/ui/popover' -import { - Select, - SelectContent, - SelectGroup, - SelectGroupLabel, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/app/components/base/ui/select' -import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' -import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks' -import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' -import { cn } from '@/utils/classnames' -import { getEvaluationMockConfig } from './mock' -import { - getAllowedOperators, - isCustomMetricConfigured, - isEvaluationRunnable, - requiresConditionValue, - useEvaluationResource, - useEvaluationStore, -} from './store' - -type EvaluationProps = { - resourceType: EvaluationResourceType - resourceId: string -} - -const TAB_CLASS_NAME = 'flex-1 rounded-lg px-3 py-2 text-left system-sm-medium' - -const encodeModelSelection = (provider: string, model: string) => `${provider}::${model}` - -const decodeModelSelection = (judgeModelId: string | null) => { - if (!judgeModelId) - return undefined - - const [provider, model] = judgeModelId.split('::') - if (!provider || !model) - return undefined - - return { provider, model } -} - -const compactOperatorLabels: Partial> = { - is: '=', - is_not: '!=', - greater_than: '>', - less_than: '<', - greater_or_equal: '>=', - less_or_equal: '<=', -} - -const groupFieldOptions = (fieldOptions: EvaluationFieldOption[]) => { - return Object.entries(fieldOptions.reduce>((acc, field) => { - acc[field.group] = [...(acc[field.group] ?? []), field] - return acc - }, {})) -} - -const getOperatorLabel = ( - operator: ComparisonOperator, - fieldType: EvaluationFieldOption['type'] | undefined, - t: TFunction<'evaluation'>, -) => { - if (fieldType === 'number' && compactOperatorLabels[operator]) - return compactOperatorLabels[operator] as string - - return t(`conditions.operators.${operator}` as const) -} - -const getFieldTypeIconClassName = (fieldType: EvaluationFieldOption['type']) => { - if (fieldType === 'number') - return 'i-ri-hashtag' - - if (fieldType === 'boolean') - return 'i-ri-checkbox-circle-line' - - if (fieldType === 'enum') - return 'i-ri-list-check-2' - - if (fieldType === 'time') - return 'i-ri-time-line' - - return 'i-ri-text' -} - -const ConditionFieldLabel = ({ - field, - placeholder, -}: { - field?: EvaluationFieldOption - placeholder: string -}) => { - if (!field) - return {placeholder} - - return ( -
-
- - {field.label} -
- {field.type} -
- ) -} - -const SectionHeader = ({ - title, - description, - action, -}: { - title: string - description: string - action?: ReactNode -}) => { - return ( -
-
-
{title}
-
{description}
-
- {action} -
- ) -} - -const FieldValueInput = ({ - field, - operator, - value, - onChange, -}: { - field?: EvaluationFieldOption - operator: ComparisonOperator - value: string | number | boolean | null - onChange: (value: string | number | boolean | null) => void -}) => { - const { t } = useTranslation('evaluation') - - if (!field || !requiresConditionValue(operator)) - return null - - if (field.type === 'time') { - const selectedTime = typeof value === 'string' && value ? dayjs(value) : undefined - - return ( -
- onChange(date ? date.toISOString() : null)} - onClear={() => onChange(null)} - placeholder={t('conditions.selectTime')} - triggerWrapClassName="w-full" - popupZIndexClassname="z-[1002]" - renderTrigger={({ handleClickTrigger }) => ( - - )} - /> -
- ) - } - - if (field.type === 'boolean') { - return ( -
- -
- ) - } - - if (field.type === 'enum') { - return ( -
- -
- ) - } - - return ( -
- { - if (field.type === 'number') { - const nextValue = e.target.value - onChange(nextValue === '' ? null : Number(nextValue)) - return - } - - onChange(e.target.value) - }} - /> -
- ) -} - -const ConditionFieldSelect = ({ - field, - fieldOptions, - placeholder, - onChange, -}: { - field?: EvaluationFieldOption - fieldOptions: EvaluationFieldOption[] - placeholder: string - onChange: (fieldId: string) => void -}) => { - return ( - - ) -} - -const ConditionOperatorSelect = ({ - field, - operator, - operators, - onChange, -}: { - field?: EvaluationFieldOption - operator: ComparisonOperator - operators: ComparisonOperator[] - onChange: (operator: ComparisonOperator) => void -}) => { - const { t } = useTranslation('evaluation') - - return ( - - ) -} - -const JudgeModelSelector = ({ - resourceId, - resourceType, -}: EvaluationProps) => { - const { data: modelList } = useModelList(ModelTypeEnum.textGeneration) - const resource = useEvaluationResource(resourceType, resourceId) - const setJudgeModel = useEvaluationStore(state => state.setJudgeModel) - const selectedModel = decodeModelSelection(resource.judgeModelId) - - useEffect(() => { - if (resource.judgeModelId || !modelList.length) - return - - const firstProvider = modelList[0] - const firstModel = firstProvider.models[0] - if (!firstProvider || !firstModel) - return - - setJudgeModel(resourceType, resourceId, encodeModelSelection(firstProvider.provider, firstModel.model)) - }, [modelList, resource.judgeModelId, resourceId, resourceType, setJudgeModel]) - - return ( - setJudgeModel(resourceType, resourceId, encodeModelSelection(model.provider, model.model))} - showDeprecatedWarnIcon - triggerClassName="h-11" - /> - ) -} - -const MetricSelector = ({ - resourceType, - resourceId, -}: EvaluationProps) => { - const { t } = useTranslation('evaluation') - const config = getEvaluationMockConfig(resourceType) - const metricGroupLabels = { - quality: t('metrics.groups.quality'), - operations: t('metrics.groups.operations'), - } - const metrics = useEvaluationResource(resourceType, resourceId).metrics - const addBuiltinMetric = useEvaluationStore(state => state.addBuiltinMetric) - const addCustomMetric = useEvaluationStore(state => state.addCustomMetric) - const [open, setOpen] = useState(false) - const [query, setQuery] = useState('') - const [showAll, setShowAll] = useState(false) - const [isLoading, setIsLoading] = useState(false) - const loadingTimerRef = useRef(null) - - const triggerLoading = () => { - if (loadingTimerRef.current) - window.clearTimeout(loadingTimerRef.current) - - setIsLoading(true) - loadingTimerRef.current = window.setTimeout(() => { - setIsLoading(false) - }, 180) - } - - const handleOpenChange = (nextOpen: boolean) => { - setOpen(nextOpen) - - if (nextOpen) { - triggerLoading() - return - } - - if (loadingTimerRef.current) - window.clearTimeout(loadingTimerRef.current) - setIsLoading(false) - } - - const handleQueryChange = (event: ChangeEvent) => { - setQuery(event.target.value) - if (open) - triggerLoading() - } - - useEffect(() => { - return () => { - if (loadingTimerRef.current) - window.clearTimeout(loadingTimerRef.current) - } - }, []) - - const filteredGroups = useMemo(() => { - const filteredMetrics = config.builtinMetrics.filter((metric) => { - const keyword = query.trim().toLowerCase() - if (!keyword) - return true - - return metric.label.toLowerCase().includes(keyword) || metric.description.toLowerCase().includes(keyword) - }) - - const grouped = filteredMetrics.reduce>((acc, metric) => { - acc[metric.group] = [...(acc[metric.group] ?? []), metric] - return acc - }, {}) - - return Object.entries(grouped) - }, [config.builtinMetrics, query]) - - return ( - - - - {t('metrics.add')} - - -
- -
- {isLoading && ( -
- {['metric-skeleton-1', 'metric-skeleton-2', 'metric-skeleton-3'].map(key => ( -
- ))} -
- )} - {!isLoading && filteredGroups.length === 0 && ( -
- {t('metrics.noResults')} -
- )} - {!isLoading && filteredGroups.map(([groupName, options]) => { - const shownOptions = showAll ? options : options.slice(0, 2) - return ( -
-
{metricGroupLabels[groupName as keyof typeof metricGroupLabels] ?? groupName}
-
- {shownOptions.map(option => ( - - ))} -
-
- ) - })} -
- {filteredGroups.some(([, options]) => options.length > 2) && ( - - )} -
- -
-
- - - ) -} - -const CustomMetricEditor = ({ - resourceType, - resourceId, - metric, -}: EvaluationProps & { metric: EvaluationMetric }) => { - const { t } = useTranslation('evaluation') - const config = getEvaluationMockConfig(resourceType) - const setCustomMetricWorkflow = useEvaluationStore(state => state.setCustomMetricWorkflow) - const addCustomMetricMapping = useEvaluationStore(state => state.addCustomMetricMapping) - const updateCustomMetricMapping = useEvaluationStore(state => state.updateCustomMetricMapping) - const removeCustomMetricMapping = useEvaluationStore(state => state.removeCustomMetricMapping) - const selectedWorkflow = config.workflowOptions.find(option => option.id === metric.customConfig?.workflowId) - const isConfigured = isCustomMetricConfigured(metric) - - if (!metric.customConfig) - return null - - return ( -
-
-
-
{t('metrics.custom.title')}
-
{t('metrics.custom.description')}
-
- {!isConfigured && {t('metrics.custom.warningBadge')}} -
-
-
-
{t('metrics.custom.workflowLabel')}
- - {selectedWorkflow &&
{selectedWorkflow.description}
} -
-
-
-
{t('metrics.custom.mappingTitle')}
- -
-
- {metric.customConfig.mappings.map(mapping => ( - updateCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id, patch)} - onRemove={() => removeCustomMetricMapping(resourceType, resourceId, metric.id, mapping.id)} - /> - ))} -
- {!isConfigured && ( -
- {t('metrics.custom.mappingWarning')} -
- )} -
-
-
- ) -} - -function MappingRow({ - resourceType, - mapping, - targetOptions, - onUpdate, - onRemove, -}: { - resourceType: EvaluationResourceType - mapping: CustomMetricMapping - targetOptions: Array<{ id: string, label: string }> - onUpdate: (patch: { sourceFieldId?: string | null, targetVariableId?: string | null }) => void - onRemove: () => void -}) { - const { t } = useTranslation('evaluation') - const config = getEvaluationMockConfig(resourceType) - - return ( -
- -
- -
- - -
- ) -} - -const ConditionGroup = ({ - resourceType, - resourceId, - group, - index, -}: EvaluationProps & { group: JudgmentConditionGroup, index: number }) => { - const { t } = useTranslation('evaluation') - const config = getEvaluationMockConfig(resourceType) - const logicalLabels = { - and: t('conditions.logical.and'), - or: t('conditions.logical.or'), - } - const removeConditionGroup = useEvaluationStore(state => state.removeConditionGroup) - const setConditionGroupOperator = useEvaluationStore(state => state.setConditionGroupOperator) - const addConditionItem = useEvaluationStore(state => state.addConditionItem) - const removeConditionItem = useEvaluationStore(state => state.removeConditionItem) - const updateConditionField = useEvaluationStore(state => state.updateConditionField) - const updateConditionOperator = useEvaluationStore(state => state.updateConditionOperator) - const updateConditionValue = useEvaluationStore(state => state.updateConditionValue) - - return ( -
-
-
- {t('conditions.groupLabel', { index: index + 1 })} -
- {(['and', 'or'] as const).map(operator => ( - - ))} -
-
-
- - -
-
-
- {group.items.map((item) => { - const field = config.fieldOptions.find(option => option.id === item.fieldId) - const allowedOperators = getAllowedOperators(resourceType, item.fieldId) - const showValue = !!field && requiresConditionValue(item.operator) - - return ( -
-
-
-
- updateConditionField(resourceType, resourceId, group.id, item.id, value)} - /> -
-
- updateConditionOperator(resourceType, resourceId, group.id, item.id, value)} - /> -
- {showValue && ( -
- updateConditionValue(resourceType, resourceId, group.id, item.id, value)} - /> -
- )} -
-
- -
-
- ) - })} -
-
- ) -} - -const BatchTestPanel = ({ - resourceType, - resourceId, -}: EvaluationProps) => { - const { t } = useTranslation('evaluation') - const config = getEvaluationMockConfig(resourceType) - const tabLabels = { - 'input-fields': t('batch.tabs.input-fields'), - 'history': t('batch.tabs.history'), - } - const statusLabels = { - running: t('batch.status.running'), - success: t('batch.status.success'), - failed: t('batch.status.failed'), - } - const resource = useEvaluationResource(resourceType, resourceId) - const setBatchTab = useEvaluationStore(state => state.setBatchTab) - const setUploadedFileName = useEvaluationStore(state => state.setUploadedFileName) - const runBatchTest = useEvaluationStore(state => state.runBatchTest) - const fileInputRef = useRef(null) - const isRunnable = isEvaluationRunnable(resource) - - const handleDownloadTemplate = () => { - const content = ['case_id,input,expected', '1,Example input,Example output'].join('\n') - const link = document.createElement('a') - link.href = `data:text/csv;charset=utf-8,${encodeURIComponent(content)}` - link.download = config.templateFileName - link.click() - } - - const handleRun = () => { - if (!isRunnable) { - Toast.notify({ - type: 'warning', - message: t('batch.validation'), - }) - return - } - - runBatchTest(resourceType, resourceId) - } - - return ( -
-
-
- - {t('batch.title')} -
-
-
{t('batch.noticeTitle')}
-
{t('batch.noticeDescription')}
-
-
- {(['input-fields', 'history'] as const).map(tab => ( - - ))} -
-
-
- {resource.activeBatchTab === 'input-fields' && ( -
-
-
{t('batch.requirementsTitle')}
-
- {config.batchRequirements.map(requirement => ( -
- - {requirement} -
- ))} -
-
-
- - { - const file = event.target.files?.[0] - setUploadedFileName(resourceType, resourceId, file?.name ?? null) - }} - /> - -
- {!isRunnable && ( -
- {t('batch.validation')} -
- )} - -
- )} - {resource.activeBatchTab === 'history' && ( -
- {resource.batchRecords.length === 0 && ( -
- {t('batch.emptyHistory')} -
- )} - {resource.batchRecords.map(record => ( -
-
-
-
{record.summary}
-
{record.fileName}
-
- - {record.status === 'running' - ? ( - - - {statusLabels.running} - - ) - : statusLabels[record.status]} - -
-
{record.startedAt}
-
- ))} -
- )} -
-
- ) -} +import BatchTestPanel from './components/batch-test-panel' +import ConditionsSection from './components/conditions-section' +import JudgeModelSelector from './components/judge-model-selector' +import MetricSection from './components/metric-section' +import SectionHeader from './components/section-header' +import { useEvaluationStore } from './store' const Evaluation = ({ resourceType, resourceId, -}: EvaluationProps) => { +}: EvaluationResourceProps) => { const { t } = useTranslation('evaluation') - const resource = useEvaluationResource(resourceType, resourceId) const ensureResource = useEvaluationStore(state => state.ensureResource) - const removeMetric = useEvaluationStore(state => state.removeMetric) - const addConditionGroup = useEvaluationStore(state => state.addConditionGroup) useEffect(() => { ensureResource(resourceType, resourceId) @@ -934,76 +32,8 @@ const Evaluation = ({
- -
- } - /> -
- {resource.metrics.map(metric => ( -
-
-
-
{metric.label}
-
{metric.description}
-
- {metric.badges.map(badge => ( - {badge} - ))} -
-
- -
- {metric.kind === 'custom-workflow' && ( - - )} -
- ))} -
-
- -
- addConditionGroup(resourceType, resourceId)}> - - {t('conditions.addGroup')} - - )} - /> -
- {resource.conditions.length === 0 && ( -
-
{t('conditions.emptyTitle')}
-
{t('conditions.emptyDescription')}
-
- )} - {resource.conditions.map((group, index) => ( - - ))} -
-
+ +
diff --git a/web/app/components/evaluation/store-utils.ts b/web/app/components/evaluation/store-utils.ts new file mode 100644 index 0000000000..9724619348 --- /dev/null +++ b/web/app/components/evaluation/store-utils.ts @@ -0,0 +1,183 @@ +import type { + BatchTestRecord, + ComparisonOperator, + CustomMetricMapping, + EvaluationFieldOption, + EvaluationMetric, + EvaluationResourceState, + EvaluationResourceType, + JudgmentConditionGroup, + MetricOption, +} from './types' +import { getComparisonOperators, getDefaultOperator, getEvaluationMockConfig } from './mock' + +export type EvaluationStoreResources = Record + +const createId = (prefix: string) => `${prefix}-${Math.random().toString(36).slice(2, 10)}` + +export const buildResourceKey = (resourceType: EvaluationResourceType, resourceId: string) => `${resourceType}:${resourceId}` + +export const conditionOperatorsWithoutValue: ComparisonOperator[] = ['is_empty', 'is_not_empty'] + +export const requiresConditionValue = (operator: ComparisonOperator) => !conditionOperatorsWithoutValue.includes(operator) + +export const getConditionValue = ( + field: EvaluationFieldOption | undefined, + operator: ComparisonOperator, + previousValue: string | number | boolean | null = null, +) => { + if (!field || !requiresConditionValue(operator)) + return null + + if (field.type === 'boolean') + return typeof previousValue === 'boolean' ? previousValue : null + + if (field.type === 'enum') + return typeof previousValue === 'string' ? previousValue : null + + if (field.type === 'number') + return typeof previousValue === 'number' ? previousValue : null + + return typeof previousValue === 'string' ? previousValue : null +} + +export const createBuiltinMetric = (metric: MetricOption): EvaluationMetric => ({ + id: createId('metric'), + optionId: metric.id, + kind: 'builtin', + label: metric.label, + description: metric.description, + badges: metric.badges, +}) + +export const createCustomMetricMapping = (): CustomMetricMapping => ({ + id: createId('mapping'), + sourceFieldId: null, + targetVariableId: null, +}) + +export const createCustomMetric = (): EvaluationMetric => ({ + id: createId('metric'), + optionId: createId('custom'), + kind: 'custom-workflow', + label: 'Custom Evaluator', + description: 'Map workflow variables to your evaluation inputs.', + badges: ['Workflow'], + customConfig: { + workflowId: null, + mappings: [createCustomMetricMapping()], + }, +}) + +export const buildConditionItem = (resourceType: EvaluationResourceType) => { + const field = getEvaluationMockConfig(resourceType).fieldOptions[0] + const operator = field ? getDefaultOperator(field.type) : 'contains' + + return { + id: createId('condition'), + fieldId: field?.id ?? null, + operator, + value: getConditionValue(field, operator), + } +} + +export const createConditionGroup = (resourceType: EvaluationResourceType): JudgmentConditionGroup => ({ + id: createId('group'), + logicalOperator: 'and', + items: [buildConditionItem(resourceType)], +}) + +export const buildInitialState = (resourceType: EvaluationResourceType): EvaluationResourceState => { + const config = getEvaluationMockConfig(resourceType) + const defaultMetric = config.builtinMetrics[0] + + return { + judgeModelId: null, + metrics: defaultMetric ? [createBuiltinMetric(defaultMetric)] : [], + conditions: [createConditionGroup(resourceType)], + activeBatchTab: 'input-fields', + uploadedFileName: null, + batchRecords: [], + } +} + +export const getResourceState = ( + resources: EvaluationStoreResources, + resourceType: EvaluationResourceType, + resourceId: string, +) => { + const resourceKey = buildResourceKey(resourceType, resourceId) + + return { + resourceKey, + resource: resources[resourceKey] ?? buildInitialState(resourceType), + } +} + +export const updateResourceState = ( + resources: EvaluationStoreResources, + resourceType: EvaluationResourceType, + resourceId: string, + updater: (resource: EvaluationResourceState) => EvaluationResourceState, +) => { + const { resource, resourceKey } = getResourceState(resources, resourceType, resourceId) + + return { + ...resources, + [resourceKey]: updater(resource), + } +} + +export const updateMetric = ( + metrics: EvaluationMetric[], + metricId: string, + updater: (metric: EvaluationMetric) => EvaluationMetric, +) => metrics.map(metric => metric.id === metricId ? updater(metric) : metric) + +export const updateConditionGroup = ( + groups: JudgmentConditionGroup[], + groupId: string, + updater: (group: JudgmentConditionGroup) => JudgmentConditionGroup, +) => groups.map(group => group.id === groupId ? updater(group) : group) + +export const createBatchTestRecord = ( + resourceType: EvaluationResourceType, + uploadedFileName: string | null | undefined, +): BatchTestRecord => { + const config = getEvaluationMockConfig(resourceType) + + return { + id: createId('batch'), + fileName: uploadedFileName ?? config.templateFileName, + status: 'running', + startedAt: new Date().toLocaleTimeString(), + summary: config.historySummaryLabel, + } +} + +export const isCustomMetricConfigured = (metric: EvaluationMetric) => { + if (metric.kind !== 'custom-workflow') + return true + + if (!metric.customConfig?.workflowId) + return false + + return metric.customConfig.mappings.length > 0 + && metric.customConfig.mappings.every(mapping => !!mapping.sourceFieldId && !!mapping.targetVariableId) +} + +export const isEvaluationRunnable = (state: EvaluationResourceState) => { + return !!state.judgeModelId + && state.metrics.length > 0 + && state.metrics.every(isCustomMetricConfigured) + && state.conditions.some(group => group.items.length > 0) +} + +export const getAllowedOperators = (resourceType: EvaluationResourceType, fieldId: string | null) => { + const field = getEvaluationMockConfig(resourceType).fieldOptions.find(option => option.id === fieldId) + + if (!field) + return ['contains'] as ComparisonOperator[] + + return getComparisonOperators(field.type) +} diff --git a/web/app/components/evaluation/store.ts b/web/app/components/evaluation/store.ts index 59b2b6d907..26e3fd9e79 100644 --- a/web/app/components/evaluation/store.ts +++ b/web/app/components/evaluation/store.ts @@ -1,14 +1,29 @@ import type { - BatchTestRecord, ComparisonOperator, - EvaluationFieldOption, - EvaluationMetric, EvaluationResourceState, EvaluationResourceType, - JudgmentConditionGroup, } from './types' import { create } from 'zustand' -import { getComparisonOperators, getDefaultOperator, getEvaluationMockConfig } from './mock' +import { getDefaultOperator, getEvaluationMockConfig } from './mock' +import { + buildConditionItem, + buildInitialState, + buildResourceKey, + createBatchTestRecord, + createBuiltinMetric, + createConditionGroup, + createCustomMetric, + createCustomMetricMapping, + getAllowedOperators as getAllowedOperatorsFromUtils, + getConditionValue, + getResourceState, + isCustomMetricConfigured as isCustomMetricConfiguredFromUtils, + isEvaluationRunnable as isEvaluationRunnableFromUtils, + requiresConditionValue as requiresConditionValueFromUtils, + updateConditionGroup, + updateMetric, + updateResourceState, +} from './store-utils' type EvaluationStore = { resources: Record @@ -46,117 +61,8 @@ type EvaluationStore = { runBatchTest: (resourceType: EvaluationResourceType, resourceId: string) => void } -const buildResourceKey = (resourceType: EvaluationResourceType, resourceId: string) => `${resourceType}:${resourceId}` const initialResourceCache: Record = {} -const createId = (prefix: string) => `${prefix}-${Math.random().toString(36).slice(2, 10)}` - -export const conditionOperatorsWithoutValue: ComparisonOperator[] = ['is_empty', 'is_not_empty'] - -export const requiresConditionValue = (operator: ComparisonOperator) => !conditionOperatorsWithoutValue.includes(operator) - -const getConditionValue = ( - field: EvaluationFieldOption | undefined, - operator: ComparisonOperator, - previousValue: string | number | boolean | null = null, -) => { - if (!field || !requiresConditionValue(operator)) - return null - - if (field.type === 'boolean') - return typeof previousValue === 'boolean' ? previousValue : null - - if (field.type === 'enum') - return typeof previousValue === 'string' ? previousValue : null - - if (field.type === 'number') - return typeof previousValue === 'number' ? previousValue : null - - return typeof previousValue === 'string' ? previousValue : null -} - -const buildConditionItem = (resourceType: EvaluationResourceType) => { - const field = getEvaluationMockConfig(resourceType).fieldOptions[0] - const operator = field ? getDefaultOperator(field.type) : 'contains' - - return { - id: createId('condition'), - fieldId: field?.id ?? null, - operator, - value: getConditionValue(field, operator), - } -} - -const buildInitialState = (resourceType: EvaluationResourceType): EvaluationResourceState => { - const config = getEvaluationMockConfig(resourceType) - const defaultMetric = config.builtinMetrics[0] - - return { - judgeModelId: null, - metrics: defaultMetric - ? [{ - id: createId('metric'), - optionId: defaultMetric.id, - kind: 'builtin', - label: defaultMetric.label, - description: defaultMetric.description, - badges: defaultMetric.badges, - }] - : [], - conditions: [{ - id: createId('group'), - logicalOperator: 'and', - items: [buildConditionItem(resourceType)], - }], - activeBatchTab: 'input-fields', - uploadedFileName: null, - batchRecords: [], - } -} - -const withResourceState = ( - resources: EvaluationStore['resources'], - resourceType: EvaluationResourceType, - resourceId: string, -) => { - const resourceKey = buildResourceKey(resourceType, resourceId) - - return { - resourceKey, - resource: resources[resourceKey] ?? buildInitialState(resourceType), - } -} - -const updateMetric = ( - metrics: EvaluationMetric[], - metricId: string, - updater: (metric: EvaluationMetric) => EvaluationMetric, -) => metrics.map(metric => metric.id === metricId ? updater(metric) : metric) - -const updateConditionGroup = ( - groups: JudgmentConditionGroup[], - groupId: string, - updater: (group: JudgmentConditionGroup) => JudgmentConditionGroup, -) => groups.map(group => group.id === groupId ? updater(group) : group) - -export const isCustomMetricConfigured = (metric: EvaluationMetric) => { - if (metric.kind !== 'custom-workflow') - return true - - if (!metric.customConfig?.workflowId) - return false - - return metric.customConfig.mappings.length > 0 - && metric.customConfig.mappings.every(mapping => !!mapping.sourceFieldId && !!mapping.targetVariableId) -} - -export const isEvaluationRunnable = (state: EvaluationResourceState) => { - return !!state.judgeModelId - && state.metrics.length > 0 - && state.metrics.every(isCustomMetricConfigured) - && state.conditions.some(group => group.items.length > 0) -} - export const useEvaluationStore = create((set, get) => ({ resources: {}, ensureResource: (resourceType, resourceId) => { @@ -172,19 +78,12 @@ export const useEvaluationStore = create((set, get) => ({ })) }, setJudgeModel: (resourceType, resourceId, judgeModelId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - judgeModelId, - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + judgeModelId, + })), + })) }, addBuiltinMetric: (resourceType, resourceId, optionId) => { const option = getEvaluationMockConfig(resourceType).builtinMetrics.find(metric => metric.id === optionId) @@ -192,430 +91,254 @@ export const useEvaluationStore = create((set, get) => ({ return set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) + const { resource } = getResourceState(state.resources, resourceType, resourceId) if (resource.metrics.some(metric => metric.optionId === optionId && metric.kind === 'builtin')) return state return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: [ - ...resource.metrics, - { - id: createId('metric'), - optionId: option.id, - kind: 'builtin', - label: option.label, - description: option.description, - badges: option.badges, - }, - ], - }, - }, + resources: updateResourceState(state.resources, resourceType, resourceId, currentResource => ({ + ...currentResource, + metrics: [...currentResource.metrics, createBuiltinMetric(option)], + })), } }) }, addCustomMetric: (resourceType, resourceId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: [ - ...resource.metrics, - { - id: createId('metric'), - optionId: createId('custom'), - kind: 'custom-workflow', - label: 'Custom Evaluator', - description: 'Map workflow variables to your evaluation inputs.', - badges: ['Workflow'], - customConfig: { - workflowId: null, - mappings: [{ - id: createId('mapping'), - sourceFieldId: null, - targetVariableId: null, - }], - }, - }, - ], - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + metrics: [...resource.metrics, createCustomMetric()], + })), + })) }, removeMetric: (resourceType, resourceId, metricId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: resource.metrics.filter(metric => metric.id !== metricId), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + metrics: resource.metrics.filter(metric => metric.id !== metricId), + })), + })) }, setCustomMetricWorkflow: (resourceType, resourceId, metricId, workflowId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: updateMetric(resource.metrics, metricId, metric => ({ - ...metric, - customConfig: metric.customConfig - ? { - ...metric.customConfig, - workflowId, - mappings: metric.customConfig.mappings.map(mapping => ({ - ...mapping, - targetVariableId: null, - })), - } - : metric.customConfig, - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + metrics: updateMetric(resource.metrics, metricId, metric => ({ + ...metric, + customConfig: metric.customConfig + ? { + ...metric.customConfig, + workflowId, + mappings: metric.customConfig.mappings.map(mapping => ({ + ...mapping, + targetVariableId: null, + })), + } + : metric.customConfig, + })), + })), + })) }, addCustomMetricMapping: (resourceType, resourceId, metricId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: updateMetric(resource.metrics, metricId, metric => ({ - ...metric, - customConfig: metric.customConfig - ? { - ...metric.customConfig, - mappings: [ - ...metric.customConfig.mappings, - { - id: createId('mapping'), - sourceFieldId: null, - targetVariableId: null, - }, - ], - } - : metric.customConfig, - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + metrics: updateMetric(resource.metrics, metricId, metric => ({ + ...metric, + customConfig: metric.customConfig + ? { + ...metric.customConfig, + mappings: [...metric.customConfig.mappings, createCustomMetricMapping()], + } + : metric.customConfig, + })), + })), + })) }, updateCustomMetricMapping: (resourceType, resourceId, metricId, mappingId, patch) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: updateMetric(resource.metrics, metricId, metric => ({ - ...metric, - customConfig: metric.customConfig - ? { - ...metric.customConfig, - mappings: metric.customConfig.mappings.map(mapping => mapping.id === mappingId ? { ...mapping, ...patch } : mapping), - } - : metric.customConfig, - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + metrics: updateMetric(resource.metrics, metricId, metric => ({ + ...metric, + customConfig: metric.customConfig + ? { + ...metric.customConfig, + mappings: metric.customConfig.mappings.map(mapping => mapping.id === mappingId ? { ...mapping, ...patch } : mapping), + } + : metric.customConfig, + })), + })), + })) }, removeCustomMetricMapping: (resourceType, resourceId, metricId, mappingId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - metrics: updateMetric(resource.metrics, metricId, metric => ({ - ...metric, - customConfig: metric.customConfig - ? { - ...metric.customConfig, - mappings: metric.customConfig.mappings.filter(mapping => mapping.id !== mappingId), - } - : metric.customConfig, - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + metrics: updateMetric(resource.metrics, metricId, metric => ({ + ...metric, + customConfig: metric.customConfig + ? { + ...metric.customConfig, + mappings: metric.customConfig.mappings.filter(mapping => mapping.id !== mappingId), + } + : metric.customConfig, + })), + })), + })) }, addConditionGroup: (resourceType, resourceId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: [ - ...resource.conditions, - { - id: createId('group'), - logicalOperator: 'and', - items: [buildConditionItem(resourceType)], - }, - ], - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: [...resource.conditions, createConditionGroup(resourceType)], + })), + })) }, removeConditionGroup: (resourceType, resourceId, groupId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: resource.conditions.filter(group => group.id !== groupId), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: resource.conditions.filter(group => group.id !== groupId), + })), + })) }, setConditionGroupOperator: (resourceType, resourceId, groupId, logicalOperator) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: updateConditionGroup(resource.conditions, groupId, group => ({ - ...group, - logicalOperator, - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: updateConditionGroup(resource.conditions, groupId, group => ({ + ...group, + logicalOperator, + })), + })), + })) }, addConditionItem: (resourceType, resourceId, groupId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: updateConditionGroup(resource.conditions, groupId, group => ({ - ...group, - items: [ - ...group.items, - buildConditionItem(resourceType), - ], - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: updateConditionGroup(resource.conditions, groupId, group => ({ + ...group, + items: [...group.items, buildConditionItem(resourceType)], + })), + })), + })) }, removeConditionItem: (resourceType, resourceId, groupId, itemId) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: updateConditionGroup(resource.conditions, groupId, group => ({ - ...group, - items: group.items.filter(item => item.id !== itemId), - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: updateConditionGroup(resource.conditions, groupId, group => ({ + ...group, + items: group.items.filter(item => item.id !== itemId), + })), + })), + })) }, updateConditionField: (resourceType, resourceId, groupId, itemId, fieldId) => { const field = getEvaluationMockConfig(resourceType).fieldOptions.find(option => option.id === fieldId) - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: updateConditionGroup(resource.conditions, groupId, group => ({ + ...group, + items: group.items.map((item) => { + if (item.id !== itemId) + return item - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: updateConditionGroup(resource.conditions, groupId, group => ({ - ...group, - items: group.items.map((item) => { - if (item.id !== itemId) - return item + const nextOperator = field ? getDefaultOperator(field.type) : item.operator - return { - ...item, - fieldId, - operator: field ? getDefaultOperator(field.type) : item.operator, - value: getConditionValue(field, field ? getDefaultOperator(field.type) : item.operator), - } - }), - })), - }, - }, - } - }) + return { + ...item, + fieldId, + operator: nextOperator, + value: getConditionValue(field, nextOperator), + } + }), + })), + })), + })) }, updateConditionOperator: (resourceType, resourceId, groupId, itemId, operator) => { set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) const fieldOptions = getEvaluationMockConfig(resourceType).fieldOptions return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: updateConditionGroup(resource.conditions, groupId, group => ({ - ...group, - items: group.items.map((item) => { - if (item.id !== itemId) - return item + resources: updateResourceState(state.resources, resourceType, resourceId, currentResource => ({ + ...currentResource, + conditions: updateConditionGroup(currentResource.conditions, groupId, group => ({ + ...group, + items: group.items.map((item) => { + if (item.id !== itemId) + return item - const field = fieldOptions.find(option => option.id === item.fieldId) + const field = fieldOptions.find(option => option.id === item.fieldId) - return { - ...item, - operator, - value: getConditionValue(field, operator, item.value), - } - }), - })), - }, - }, + return { + ...item, + operator, + value: getConditionValue(field, operator, item.value), + } + }), + })), + })), } }) }, updateConditionValue: (resourceType, resourceId, groupId, itemId, value) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - conditions: updateConditionGroup(resource.conditions, groupId, group => ({ - ...group, - items: group.items.map(item => item.id === itemId ? { ...item, value } : item), - })), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + conditions: updateConditionGroup(resource.conditions, groupId, group => ({ + ...group, + items: group.items.map(item => item.id === itemId ? { ...item, value } : item), + })), + })), + })) }, setBatchTab: (resourceType, resourceId, tab) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - activeBatchTab: tab, - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + activeBatchTab: tab, + })), + })) }, setUploadedFileName: (resourceType, resourceId, uploadedFileName) => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - uploadedFileName, - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + uploadedFileName, + })), + })) }, runBatchTest: (resourceType, resourceId) => { - const config = getEvaluationMockConfig(resourceType) - const recordId = createId('batch') - const nextRecord: BatchTestRecord = { - id: recordId, - fileName: get().resources[buildResourceKey(resourceType, resourceId)]?.uploadedFileName ?? config.templateFileName, - status: 'running', - startedAt: new Date().toLocaleTimeString(), - summary: config.historySummaryLabel, - } + const { uploadedFileName } = get().resources[buildResourceKey(resourceType, resourceId)] ?? buildInitialState(resourceType) + const nextRecord = createBatchTestRecord(resourceType, uploadedFileName) - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - activeBatchTab: 'history', - batchRecords: [nextRecord, ...resource.batchRecords], - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + activeBatchTab: 'history', + batchRecords: [nextRecord, ...resource.batchRecords], + })), + })) window.setTimeout(() => { - set((state) => { - const { resource, resourceKey } = withResourceState(state.resources, resourceType, resourceId) - - return { - resources: { - ...state.resources, - [resourceKey]: { - ...resource, - batchRecords: resource.batchRecords.map(record => record.id === recordId - ? { - ...record, - status: resource.metrics.length > 1 ? 'success' : 'failed', - } - : record), - }, - }, - } - }) + set(state => ({ + resources: updateResourceState(state.resources, resourceType, resourceId, resource => ({ + ...resource, + batchRecords: resource.batchRecords.map(record => record.id === nextRecord.id + ? { + ...record, + status: resource.metrics.length > 1 ? 'success' : 'failed', + } + : record), + })), + })) }, 1200) }, })) @@ -626,10 +349,17 @@ export const useEvaluationResource = (resourceType: EvaluationResourceType, reso } export const getAllowedOperators = (resourceType: EvaluationResourceType, fieldId: string | null) => { - const field = getEvaluationMockConfig(resourceType).fieldOptions.find(option => option.id === fieldId) - - if (!field) - return ['contains'] as ComparisonOperator[] - - return getComparisonOperators(field.type) + return getAllowedOperatorsFromUtils(resourceType, fieldId) +} + +export const isCustomMetricConfigured = (metric: EvaluationResourceState['metrics'][number]) => { + return isCustomMetricConfiguredFromUtils(metric) +} + +export const isEvaluationRunnable = (state: EvaluationResourceState) => { + return isEvaluationRunnableFromUtils(state) +} + +export const requiresConditionValue = (operator: ComparisonOperator) => { + return requiresConditionValueFromUtils(operator) } diff --git a/web/app/components/evaluation/types.ts b/web/app/components/evaluation/types.ts index 9b9cfda31c..dfda0e2f6c 100644 --- a/web/app/components/evaluation/types.ts +++ b/web/app/components/evaluation/types.ts @@ -1,5 +1,10 @@ export type EvaluationResourceType = 'workflow' | 'pipeline' | 'snippet' +export type EvaluationResourceProps = { + resourceType: EvaluationResourceType + resourceId: string +} + export type MetricKind = 'builtin' | 'custom-workflow' export type BatchTestTab = 'input-fields' | 'history' diff --git a/web/app/components/evaluation/utils.ts b/web/app/components/evaluation/utils.ts new file mode 100644 index 0000000000..99723ad1de --- /dev/null +++ b/web/app/components/evaluation/utils.ts @@ -0,0 +1,60 @@ +import type { TFunction } from 'i18next' +import type { ComparisonOperator, EvaluationFieldOption } from './types' + +export const TAB_CLASS_NAME = 'flex-1 rounded-lg px-3 py-2 text-left system-sm-medium' + +const compactOperatorLabels: Partial> = { + is: '=', + is_not: '!=', + greater_than: '>', + less_than: '<', + greater_or_equal: '>=', + less_or_equal: '<=', +} + +export const encodeModelSelection = (provider: string, model: string) => `${provider}::${model}` + +export const decodeModelSelection = (judgeModelId: string | null) => { + if (!judgeModelId) + return undefined + + const [provider, model] = judgeModelId.split('::') + if (!provider || !model) + return undefined + + return { provider, model } +} + +export const groupFieldOptions = (fieldOptions: EvaluationFieldOption[]) => { + return Object.entries(fieldOptions.reduce>((acc, field) => { + acc[field.group] = [...(acc[field.group] ?? []), field] + return acc + }, {})) +} + +export const getOperatorLabel = ( + operator: ComparisonOperator, + fieldType: EvaluationFieldOption['type'] | undefined, + t: TFunction<'evaluation'>, +) => { + if (fieldType === 'number' && compactOperatorLabels[operator]) + return compactOperatorLabels[operator] as string + + return t(`conditions.operators.${operator}` as const) +} + +export const getFieldTypeIconClassName = (fieldType: EvaluationFieldOption['type']) => { + if (fieldType === 'number') + return 'i-ri-hashtag' + + if (fieldType === 'boolean') + return 'i-ri-checkbox-circle-line' + + if (fieldType === 'enum') + return 'i-ri-list-check-2' + + if (fieldType === 'time') + return 'i-ri-time-line' + + return 'i-ri-text' +} diff --git a/web/app/components/snippets/components/hooks/__tests__/use-snippet-publish.spec.ts b/web/app/components/snippets/components/hooks/__tests__/use-snippet-publish.spec.ts index 8ffe98182e..9d29d80906 100644 --- a/web/app/components/snippets/components/hooks/__tests__/use-snippet-publish.spec.ts +++ b/web/app/components/snippets/components/hooks/__tests__/use-snippet-publish.spec.ts @@ -54,7 +54,6 @@ describe('useSnippetPublish', () => { it('should publish the snippet, close the menu, and show success feedback', async () => { const { result } = renderHook(() => useSnippetPublish({ snippetId: 'snippet-1', - section: 'orchestrate', })) await act(async () => { @@ -73,7 +72,6 @@ describe('useSnippetPublish', () => { const { result } = renderHook(() => useSnippetPublish({ snippetId: 'snippet-1', - section: 'orchestrate', })) await act(async () => { @@ -89,7 +87,6 @@ describe('useSnippetPublish', () => { it('should trigger publish on ctrl+shift+p in the orchestrate section', async () => { renderHook(() => useSnippetPublish({ snippetId: 'snippet-1', - section: 'orchestrate', })) const event = new KeyboardEvent('keydown') @@ -110,7 +107,6 @@ describe('useSnippetPublish', () => { it('should ignore the shortcut outside the orchestrate section', () => { renderHook(() => useSnippetPublish({ snippetId: 'snippet-1', - section: 'evaluation', })) const event = new KeyboardEvent('keydown')