fix(llm): show all enabled chat models in multimodal sidecar selector

The vision/video sidecar dropdown filtered candidates through the built-in
capability heuristics, so provider-compatible models whose custom names match
no known prefix (and carry no declared modalities) were hidden and could not be
selected as a sidecar — even when they natively support the modality.

- listByType now returns every enabled chat model, annotating each row with a
  transient modalityCapable flag and sorting known-capable rows first, instead
  of hard-filtering recognized models only.
- MultimodalRouter honours an explicitly configured sidecar model instead of
  dropping it when the heuristics don't recognize it; a wrong pick degrades
  gracefully through the caption path rather than silently disabling routing.
- ModelPicker gains an optional capability badge; the sidecar UI tags
  recognized vision models while keeping every enabled model selectable.
This commit is contained in:
matevip 2026-06-09 10:30:37 +08:00
parent f699746d65
commit a9cf3cbd55
7 changed files with 71 additions and 16 deletions

View File

@ -77,6 +77,16 @@ public class ModelConfigEntity {
*/
private String modalities;
/**
* Transient, request-scoped flag set by {@link vip.mate.llm.service.ModelConfigService#listByType}
* when a modality filter is supplied: {@code true} when this row's declared or
* heuristically-resolved capabilities cover the requested modality. Lets the
* sidecar selector list every enabled chat model while still highlighting the
* ones already known to support the modality. Never persisted.
*/
@TableField(exist = false)
private Boolean modalityCapable;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;

View File

@ -125,11 +125,19 @@ public class MultimodalRouter {
}
/**
* Resolve the configured sidecar model for a modality. Returns null when:
* Resolve the configured sidecar model for a modality. Returns null only when:
* - the setting is empty / blank;
* - the referenced row no longer exists or has been disabled;
* - the row's resolved capability set does not actually contain the modality.
* - the referenced row no longer exists or has been disabled.
* The caller treats null as "ask the user to configure one."
* <p>
* An explicit sidecar selection is treated as the user's own capability
* declaration: a provider-compatible model can be vision-capable in practice
* even when the built-in heuristics don't recognize its name and it carries no
* declared {@code modalities}. Rejecting such a model here made it impossible to
* use a perfectly good compatible-mode vision model as the sidecar. We therefore
* honour the explicit choice and only emit a diagnostic when the heuristics
* can't confirm it a wrong pick degrades gracefully (the caption call fails and
* the attachment is reported as un-processed) rather than being silently ignored.
*/
private ModelConfigEntity resolveSidecar(Modality modality) {
SystemSettingsDTO settings = systemSettingService.getSettings();
@ -148,9 +156,9 @@ public class MultimodalRouter {
}
if (model == null || !Boolean.TRUE.equals(model.getEnabled())) return null;
if (!capabilityService.supports(model.getModelName(), model.getModalities(), modality)) {
log.warn("Configured sidecar model {}/{} does not actually support {} — ignoring",
model.getProvider(), model.getModelName(), modality);
return null;
log.info("Configured sidecar model {}/{} is not recognized as {}-capable by the "
+ "built-in heuristics; honouring the explicit selection anyway",
model.getProvider(), model.getModelName(), modality.name().toLowerCase());
}
return model;
}

View File

@ -11,6 +11,7 @@ import vip.mate.llm.event.ModelConfigChangedEvent;
import vip.mate.llm.model.ModelConfigEntity;
import vip.mate.llm.repository.ModelConfigMapper;
import java.util.Comparator;
import java.util.List;
import org.springframework.context.ApplicationEventPublisher;
@ -73,9 +74,18 @@ public class ModelConfigService {
/**
* Optional modality filter (case-insensitive: {@code "vision" / "video" / "audio"}).
* When non-null, only enabled rows whose resolved capability set contains the
* requested modality survive used by the multimodal sidecar settings UI to
* populate "default vision model" / "default video model" dropdowns.
* Used by the multimodal sidecar settings UI to populate "default vision model" /
* "default video model" dropdowns.
* <p>
* The filter does <b>not</b> hide models the built-in heuristics fail to recognize:
* a provider-compatible model (e.g. a DashScope OpenAI-compatible vision model with
* a custom name) is vision-capable in practice even though its name matches no
* built-in prefix and it carries no declared {@code modalities}. Hard-filtering
* those out left them un-selectable as a sidecar. Instead every <em>enabled</em>
* chat model is returned; each row's transient {@link ModelConfigEntity#getModalityCapable()}
* flag records whether its declared / heuristic capabilities already cover the
* requested modality, and known-capable rows are sorted to the top so the UI can
* highlight them while still letting the user pick any model.
*/
public List<ModelConfigEntity> listByType(String modelType, String modality) {
List<ModelConfigEntity> rows;
@ -100,7 +110,12 @@ public class ModelConfigService {
}
return rows.stream()
.filter(m -> Boolean.TRUE.equals(m.getEnabled()))
.filter(m -> modelCapabilityService.supports(m.getModelName(), m.getModalities(), required))
.peek(m -> m.setModalityCapable(
modelCapabilityService.supports(m.getModelName(), m.getModalities(), required)))
// Known-capable models first; preserve the existing default-then-name
// order within each group.
.sorted(Comparator.comparing(
(ModelConfigEntity m) -> Boolean.TRUE.equals(m.getModalityCapable())).reversed())
.toList();
}

View File

@ -68,6 +68,7 @@
>
<span class="model-picker__item-name">{{ m.modelName }}</span>
<span v-if="m.name && m.name !== m.modelName" class="model-picker__item-alias">{{ m.name }}</span>
<span v-if="badgeText && m.modalityCapable" class="model-picker__item-badge">{{ badgeText }}</span>
<svg
v-if="String(m.id) === String(modelValue)"
class="model-picker__check"
@ -97,6 +98,8 @@ interface ModelOption {
name?: string
provider: string
modelName: string
/** When true and `badgeText` is set, the item renders a small capability badge. */
modalityCapable?: boolean
}
const props = withDefaults(defineProps<{
@ -116,6 +119,8 @@ const props = withDefaults(defineProps<{
clearable?: boolean
/** Render trigger in disabled state (no popover, no clear). */
disabled?: boolean
/** When set, items whose `modalityCapable` is true show this short badge label. */
badgeText?: string
}>(), {
searchable: true,
clearable: true,
@ -387,6 +392,17 @@ onBeforeUnmount(() => {
color: var(--mc-text-tertiary);
font-family: inherit;
}
.model-picker__item-badge {
flex-shrink: 0;
font-size: 10px;
font-weight: 600;
padding: 1px 6px;
border-radius: 999px;
font-family: inherit;
letter-spacing: 0.02em;
background: color-mix(in srgb, var(--mc-primary) 12%, transparent);
color: var(--mc-primary);
}
.model-picker__check {
flex-shrink: 0;
color: var(--mc-primary);

View File

@ -622,15 +622,16 @@ export default {
notConfigured: 'Not configured (skip attachment)',
idle: 'Off',
reserved: 'Reserved',
capable: 'Vision',
vision: {
label: 'Vision sidecar model',
desc: 'Invoked once per uploaded image to produce a structured description. The primary chat model is unchanged.',
empty: 'No vision-capable model is enabled yet. Add one under "Cloud Models" above.',
desc: 'Invoked once per uploaded image to produce a structured description. The primary chat model is unchanged. Any enabled model can be selected; the "Vision" tag marks models already recognized as supporting image input.',
empty: 'No chat model is enabled yet. Add and enable one under "Cloud Models" above.',
},
video: {
label: 'Video sidecar model (reserved)',
desc: 'Reserved for a future iteration: when the user uploads a video, this model will sample frames and describe them. Not yet wired in v1.',
empty: 'No video-capable model is enabled yet.',
empty: 'No chat model is enabled yet.',
},
},
},

View File

@ -502,15 +502,16 @@ export default {
notConfigured: '未配置(不路由附件)',
idle: '未启用',
reserved: '预留',
capable: '视觉',
vision: {
label: '视觉旁路模型',
desc: '用户上传图片时调用一次,把图片转成结构化描述。主对话模型不变。',
empty: '尚未发现支持图片输入的模型。请到上方"云端模型"中先添加一个视觉模型。',
desc: '用户上传图片时调用一次,把图片转成结构化描述。主对话模型不变。可选择任意已启用模型;带"视觉"标记的为已识别支持图片输入的模型。',
empty: '尚未发现可用的对话模型。请到上方"云端模型"中先添加并启用一个模型。',
},
video: {
label: '视频旁路模型(预留)',
desc: '用于未来版本:当用户上传视频时,由该模型负责拆帧和描述。当前版本暂不接入路由。',
empty: '尚未发现支持视频输入的模型。',
empty: '尚未发现可用的对话模型。',
},
},
},

View File

@ -39,6 +39,7 @@
:models="visionModels"
:placeholder="t('settings.models.sidecar.notConfigured')"
:empty-text="t('settings.models.sidecar.vision.empty')"
:badge-text="t('settings.models.sidecar.capable')"
:disabled="visionModels.length === 0"
/>
</div>
@ -75,6 +76,7 @@
:models="videoModels"
:placeholder="t('settings.models.sidecar.notConfigured')"
:empty-text="t('settings.models.sidecar.video.empty')"
:badge-text="t('settings.models.sidecar.capable')"
:disabled="videoModels.length === 0"
/>
</div>
@ -102,6 +104,8 @@ interface ModelOption {
name: string
provider: string
modelName: string
/** Backend flag: declared/heuristic capabilities already cover this modality. */
modalityCapable?: boolean
}
const { t } = useI18n()