diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 83348804..6a33abab 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -101,6 +101,8 @@ export const agentApi = { chat: (id: string | number, data: any) => http.post(`/agents/${id}/chat`, data), execute: (id: string | number, data: any) => http.post(`/agents/${id}/execute`, data), getState: (id: string | number) => http.get(`/agents/${id}/state`), + /** Lightweight capability snapshot used by the chat console attachment hint. */ + getCapabilities: (id: string | number) => http.get(`/agents/${id}/capabilities`), } // ==================== Templates ==================== @@ -444,8 +446,8 @@ export const modelApi = { disableProvider: (providerId: string) => http.post(`/models/${providerId}/disable`), // ==================== Embedding Model (RFC Embedding UI) ==================== - listByType: (modelType: 'chat' | 'embedding') => - http.get('/models/by-type', { params: { modelType } }), + listByType: (modelType: 'chat' | 'embedding', modality?: 'vision' | 'video' | 'audio') => + http.get('/models/by-type', { params: { modelType, modality } }), testEmbedding: (modelId: string | number) => http.post(`/models/embedding/${modelId}/test`), getDefaultEmbedding: () => http.get('/models/embedding/default'), diff --git a/mateclaw-ui/src/components/chat/MessageBubble.vue b/mateclaw-ui/src/components/chat/MessageBubble.vue index 17697c86..cd2f835b 100644 --- a/mateclaw-ui/src/components/chat/MessageBubble.vue +++ b/mateclaw-ui/src/components/chat/MessageBubble.vue @@ -343,6 +343,12 @@ class="action-model" :title="replyModelTitle" >{{ replyModel }} + + 🔀 {{ routingBadge.label }} {{ formattedTime }} @@ -726,6 +732,30 @@ const replyModelTitle = computed(() => { return provider ? `${base} (${provider})` : base }) +// Multimodal routing badge: rendered only when a sidecar actually fired this +// turn (strategy=sidecar, sidecarModel populated). Skipped for the legacy +// "primary handled it natively" case so non-routed turns stay clean. +const routingBadge = computed(() => { + const r = props.message.metadata?.routing + if (!r || r.strategy !== 'sidecar' || !r.sidecarModel) return null + // Count routed attachments by required modality. We summarize as "1 image" + // / "2 images" rather than naming each file to keep the chip compact. + const required = r.requiredModalities || [] + const kind = required.includes('VISION') + ? t('chat.routing.kind.image') + : required.includes('VIDEO') + ? t('chat.routing.kind.video') + : t('chat.routing.kind.media') + const label = `${r.sidecarModel} (${kind})` + const tooltip = t('chat.routing.tooltip', { + primary: replyModel.value || '?', + sidecar: r.sidecarModel, + sidecarProvider: r.sidecarProvider || '', + kind, + }) + return { label, tooltip } +}) + const formatFileSize = (size: number) => { if (size < 1024) return `${size} B` if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB` @@ -1503,6 +1533,19 @@ watch(isGenerating, (generating) => { white-space: nowrap; } +.action-routing { + font-size: 11px; + color: var(--mc-primary, #d96d46); + margin-left: 4px; + padding: 1px 6px; + border-radius: 4px; + background: var(--mc-primary-bg, rgba(217, 109, 70, 0.1)); + font-family: var(--mc-mono-font, ui-monospace, "SF Mono", Menlo, monospace); + user-select: text; + white-space: nowrap; + font-weight: 500; +} + /* ==================== 主内容区域 ==================== */ .msg-content { position: relative; diff --git a/mateclaw-ui/src/components/chat/MultimodalRoutingHint.vue b/mateclaw-ui/src/components/chat/MultimodalRoutingHint.vue new file mode 100644 index 00000000..a9446a0e --- /dev/null +++ b/mateclaw-ui/src/components/chat/MultimodalRoutingHint.vue @@ -0,0 +1,160 @@ + + + + + diff --git a/mateclaw-ui/src/components/common/ModelPicker.vue b/mateclaw-ui/src/components/common/ModelPicker.vue new file mode 100644 index 00000000..3c97abac --- /dev/null +++ b/mateclaw-ui/src/components/common/ModelPicker.vue @@ -0,0 +1,408 @@ + + + + + diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index ff02e473..c09c1172 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -4,6 +4,8 @@ export default { }, common: { save: 'Save', + saving: 'Saving...', + saved: 'Saved', cancel: 'Cancel', reset: 'Reset', edit: 'Edit', @@ -22,6 +24,8 @@ export default { close: 'Close', add: 'Add', search: 'Search', + noResults: 'No matches', + noOptions: 'No options', clear: 'Clear', expandSidebar: 'Expand sidebar', collapseSidebar: 'Collapse sidebar', @@ -149,6 +153,22 @@ export default { copied: 'Copied', regenerate: 'Regenerate', replyModel: 'Reply model: {model}', + routing: { + kind: { + image: 'image', + video: 'video', + media: 'media', + }, + tooltip: 'This turn was answered by {primary}; image attachments were captioned via {sidecar} ({sidecarProvider}) before reaching the primary model.', + hint: { + willRoute: 'Detected {kind} attachment. The current model "{primary}" does not support {kind} input — it will be captioned by "{sidecar}" first (sidecar mode, primary chat unchanged).', + notConfigured: 'Detected {kind} attachment. The current model does not support {kind} input and no sidecar model has been configured.', + videoReserved: 'Detected video attachment. Video sidecar is reserved (pre-configured "{sidecar}") but not yet wired in this version. Switch to a video-capable primary model instead.', + action: { + gotoSettings: 'Configure', + }, + }, + }, ttsPlay: 'Read Aloud', ttsStop: 'Stop Reading', conversations: 'Conversations', @@ -506,6 +526,25 @@ export default { about: 'About', advanced: 'Advanced', }, + models: { + sidecar: { + title: 'Multimodal sidecar', + hint: "When the primary model can't handle images or video, the sidecar model captions the attachment to text first.", + notConfigured: 'Not configured (skip attachment)', + idle: 'Off', + reserved: 'Reserved', + vision: { + label: 'Vision sidecar model', + desc: 'Invoked once per uploaded image to produce a structured description. The primary chat model is unchanged.', + empty: 'No vision-capable model is enabled yet. Add one under "Cloud Models" above.', + }, + video: { + label: 'Video sidecar model (reserved)', + desc: 'Reserved for a future iteration: when the user uploads a video, this model will sample frames and describe them. Not yet wired in v1.', + empty: 'No video-capable model is enabled yet.', + }, + }, + }, featureFlags: { title: 'Feature Flags', description: 'Runtime-toggleable feature switches. Edits apply immediately on the writing instance and propagate to peers within one cache refresh tick (≈30 s); no server restart required.', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index fe3d7f61..3e461d7b 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -4,6 +4,8 @@ export default { }, common: { save: '保存', + saving: '保存中...', + saved: '已保存', cancel: '取消', reset: '重置', edit: '编辑', @@ -22,6 +24,8 @@ export default { close: '关闭', add: '添加', search: '搜索', + noResults: '没有匹配项', + noOptions: '暂无可选项', clear: '清除', expandSidebar: '展开侧边栏', collapseSidebar: '折叠侧边栏', @@ -149,6 +153,22 @@ export default { copied: '已复制', regenerate: '重新生成', replyModel: '本条回复模型: {model}', + routing: { + kind: { + image: '图片', + video: '视频', + media: '附件', + }, + tooltip: '本轮对话由「{primary}」处理；图片附件先经「{sidecar}」（{sidecarProvider}）转成文字后再交给主模型。', + hint: { + willRoute: '检测到{kind}附件。当前 Agent 模型「{primary}」不支持{kind}输入，将自动调用「{sidecar}」识别（旁路模式，不影响主模型）。', + notConfigured: '检测到{kind}附件。当前 Agent 模型不支持{kind}输入，且尚未配置旁路模型。', + videoReserved: '检测到视频附件。当前版本暂未接入视频旁路（已预留「{sidecar}」），建议切换到具备视频能力的多模态主模型。', + action: { + gotoSettings: '前往配置', + }, + }, + }, ttsPlay: '朗读', ttsStop: '停止朗读', conversations: '会话列表', @@ -392,6 +412,25 @@ export default { about: '关于', advanced: '高级', }, + models: { + sidecar: { + title: '多模态旁路', + hint: '当主模型不支持图片/视频时，自动用下面配置的模型先把附件转成文字，再交给主模型回答。', + notConfigured: '未配置（不路由附件）', + idle: '未启用', + reserved: '预留', + vision: { + label: '视觉旁路模型', + desc: '用户上传图片时调用一次，把图片转成结构化描述。主对话模型不变。', + empty: '尚未发现支持图片输入的模型。请到上方"云端模型"中先添加一个视觉模型。', + }, + video: { + label: '视频旁路模型（预留）', + desc: '用于未来版本：当用户上传视频时，由该模型负责拆帧和描述。当前版本暂不接入路由。', + empty: '尚未发现支持视频输入的模型。', + }, + }, + }, featureFlags: { title: '功能开关', description: '运行时可切换的功能开关。修改后立即在所有节点生效（每实例 30 秒内同步缓存）；不需要重启服务。', diff --git a/mateclaw-ui/src/types/index.ts b/mateclaw-ui/src/types/index.ts index ee3d2de6..567838b4 100644 --- a/mateclaw-ui/src/types/index.ts +++ b/mateclaw-ui/src/types/index.ts @@ -200,6 +200,36 @@ export interface MessageMetadata { durationMs: number timestamp: number }> + /** + * Multimodal sidecar routing snapshot for this turn — written by the backend + * when the user uploaded an image / video the primary model couldn't handle + * natively. The chat bubble renders a "primary 🔀 sidecar" badge from this. + */ + routing?: RoutingMeta +} + +export interface RoutingMeta { + /** "none" | "sidecar" | "native" — lowercased on the wire to keep the JSON small. */ + strategy: 'none' | 'sidecar' | 'native' + sidecarModelId?: number + sidecarModel?: string + sidecarProvider?: string + /** Modality names like ["VISION"] / ["VIDEO"] / ... — uppercase to match the backend enum. */ + requiredModalities?: string[] + primaryMissing?: string[] + skipped?: Array<{ type: string; fileName?: string; reason: string }> +} + +export interface AgentCapabilities { + agentId: number + modelName: string + providerId: string + /** Modality enum values: TEXT / VISION / VIDEO / AUDIO. */ + modalities: string[] + defaultVisionModelId?: number | null + defaultVisionModelLabel?: string | null + defaultVideoModelId?: number | null + defaultVideoModelLabel?: string | null } export interface MessageContentPart { diff --git a/mateclaw-ui/src/views/ChatConsole.vue b/mateclaw-ui/src/views/ChatConsole.vue index f3301e57..a29eed38 100644 --- a/mateclaw-ui/src/views/ChatConsole.vue +++ b/mateclaw-ui/src/views/ChatConsole.vue @@ -286,6 +286,13 @@ :lifecycle-stage="lifecycleStage" /> + + + (null) const pendingAttachments = ref([]) const uploadingAttachment = ref(false) +// Per-agent capability snapshot for the multimodal routing hint above the +// input box. Refetched whenever the active agent changes; cached locally to +// avoid an extra request per attachment change. +const agentCapabilities = ref(null) + // 思考模式：只有两个状态 — 开或关 const thinkingEnabled = ref(localStorage.getItem('mateclaw_thinking') !== 'off') const thinkingLevel = computed(() => thinkingEnabled.value ? 'high' : 'off') @@ -1134,6 +1147,19 @@ watch([selectedAgentId, currentConversationId], () => { syncRouteState() }) +// Refetch agent capabilities (modalities + sidecar config) on agent change so +// the multimodal routing hint above the input box can react synchronously when +// the user attaches an image / video. +watch(selectedAgentId, async (id) => { + if (!id) { agentCapabilities.value = null; return } + try { + const res: any = await agentApi.getCapabilities(id) + agentCapabilities.value = res.data || null + } catch { + agentCapabilities.value = null + } +}, { immediate: true }) + // ============ 方法 ============ async function loadAgents() { try { diff --git a/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue b/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue new file mode 100644 index 00000000..7f9c7ed9 --- /dev/null +++ b/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue @@ -0,0 +1,359 @@ + + + + + diff --git a/mateclaw-ui/src/views/Settings/Models/index.vue b/mateclaw-ui/src/views/Settings/Models/index.vue index 28f8f1ea..c16bc397 100644 --- a/mateclaw-ui/src/views/Settings/Models/index.vue +++ b/mateclaw-ui/src/views/Settings/Models/index.vue @@ -116,6 +116,10 @@ + + +
{{ savedTip }}
@@ -194,6 +198,7 @@ import type { ProviderInfo, ProviderModelInfo } from '@/types' import { useProviders } from './useProviders' import ProviderCard from './ProviderCard.vue' import EmbeddingModelsSection from './EmbeddingModelsSection.vue' +import MultimodalSidecarSection from './MultimodalSidecarSection.vue' // RFC-074 PR-1: defer modal JS until the user actually opens one — same // pattern as ChannelEditModal in commit 9300559b. Drops ~30KB from the // initial Settings/Models route chunk.