From 47bdb97a3a480fc9ff11068a149a5301b76e88e5 Mon Sep 17 00:00:00 2001 From: matevip Date: Thu, 30 Apr 2026 17:25:51 +0800 Subject: [PATCH] fix(agent): per-model multimodal capability resolution (issue #44) --- .../vip/mate/agent/AgentGraphBuilder.java | 3 + .../main/java/vip/mate/agent/BaseAgent.java | 40 +++- .../vip/mate/llm/model/ModelConfigEntity.java | 7 + .../llm/service/ModelCapabilityService.java | 205 ++++++++++++++++++ .../h2/V66__add_model_capabilities.sql | 4 + .../mysql/V66__add_model_capabilities.sql | 5 + 6 files changed, 255 insertions(+), 9 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/service/ModelCapabilityService.java create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V66__add_model_capabilities.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V66__add_model_capabilities.sql diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index ea3f3b4b..3748a3a9 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -100,6 +100,7 @@ public class AgentGraphBuilder { private final ConversationService conversationService; private final ModelConfigService modelConfigService; private final ModelProviderService modelProviderService; + private final vip.mate.llm.service.ModelCapabilityService modelCapabilityService; private final PlanningService planningService; private final ToolGuardService toolGuardService; private final vip.mate.tool.guard.service.ToolGuardConfigService toolGuardConfigService; @@ -235,6 +236,8 @@ public class AgentGraphBuilder { agent.systemPrompt = enhancedPrompt; agent.maxIterations = maxIter; agent.modelName = runtimeModel.getModelName(); + agent.modelCapabilities = modelCapabilityService.resolve( + runtimeModel.getModelName(), runtimeModel.getModalities()); agent.runtimeProviderId = provider != null ? provider.getProviderId() : ""; agent.temperature = runtimeModel.getTemperature(); agent.maxTokens = runtimeModel.getMaxTokens(); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java index cac7ce46..450984dd 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java @@ -10,6 +10,7 @@ import org.springframework.core.io.FileSystemResource; import org.springframework.util.MimeType; import reactor.core.publisher.Flux; import vip.mate.approval.ApprovalPlaceholderUtil; +import vip.mate.llm.service.ModelCapabilityService; import vip.mate.workspace.conversation.ConversationService; import vip.mate.workspace.conversation.model.MessageContentPart; import vip.mate.workspace.conversation.model.MessageEntity; @@ -18,7 +19,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; /** @@ -57,6 +60,13 @@ public abstract class BaseAgent { /** 模型名称 */ protected String modelName; + /** + * Modalities the chat model can natively consume (resolved at agent build time + * by {@link vip.mate.llm.service.ModelCapabilityService}). Empty set = unknown model, + * fall back to text-only behavior. See issue #44. + */ + protected Set modelCapabilities = EnumSet.noneOf(ModelCapabilityService.Modality.class); + /** 采样温度 */ protected Double temperature; @@ -483,15 +493,11 @@ public abstract class BaseAgent { /** * 判断当前模型是否支持视频输入。 - * 仅已知支持视频分析的视觉模型（Qwen-VL、GPT-4o、Gemini 等）才注入视频 Media。 + * 由 {@link ModelCapabilityService} 在 agent 构建时解析并注入到 + * {@link #modelCapabilities}，per-model 粒度（区分如 glm-4v vs glm-4v-plus）。 */ private boolean modelSupportsVideo() { - if (modelName == null) return false; - String n = modelName.toLowerCase(); - return (n.contains("qwen") && n.contains("vl")) - || n.contains("gpt-4o") - || n.contains("gemini") - || (n.contains("glm") && n.contains("v")); + return modelCapabilities.contains(ModelCapabilityService.Modality.VIDEO); } /** @@ -501,6 +507,10 @@ public abstract class BaseAgent { protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { List parts = conversationService.parseMessageParts(message); List mediaList = new ArrayList<>(); + // Reasons for attachments that the model cannot consume — surfaced to the agent + // via the user message text so it does not hallucinate a tool call to read them. + // See issue #44. + List skippedAttachments = new ArrayList<>(); boolean videoSupported = modelSupportsVideo(); for (MessageContentPart part : parts) { @@ -522,6 +532,7 @@ public abstract class BaseAgent { if (isImage && contentType.contains("svg")) { log.debug("[{}] Skipping SVG attachment (not supported by multimodal API): {}", agentName, part.getFileName()); + skippedAttachments.add(part.getFileName() + "(SVG 格式，多模态 API 不支持)"); continue; } @@ -529,6 +540,7 @@ public abstract class BaseAgent { if (isVideo && !videoSupported) { log.debug("[{}] Skipping video attachment (model '{}' does not support video): {}", agentName, modelName, part.getFileName()); + skippedAttachments.add(part.getFileName() + "(当前模型未声明视频能力)"); continue; } @@ -536,6 +548,7 @@ public abstract class BaseAgent { if (isVideo && part.getFileSize() != null && part.getFileSize() > MAX_VIDEO_SIZE_BYTES) { log.warn("[{}] Skipping oversized video attachment ({}MB > 20MB): {}", agentName, part.getFileSize() / (1024 * 1024), part.getFileName()); + skippedAttachments.add(part.getFileName() + "(视频超过 20MB 大小限制)"); continue; } @@ -547,6 +560,7 @@ public abstract class BaseAgent { if (mediaPath == null) { log.warn("[{}] {} file not found for attachment: {}, path: {}, mediaId: {}", agentName, isVideo ? "Video" : "Image", part.getFileName(), part.getPath(), part.getMediaId()); + skippedAttachments.add(part.getFileName() + "(文件未找到)"); continue; } try { @@ -558,14 +572,22 @@ public abstract class BaseAgent { } catch (Exception e) { log.warn("[{}] Failed to create Media for {} {}: {}", agentName, isVideo ? "video" : "image", part.getFileName(), e.getMessage()); + skippedAttachments.add(part.getFileName() + "(媒体加载失败)"); } } + String finalText = renderedContent; + if (!skippedAttachments.isEmpty()) { + finalText = (renderedContent == null ? "" : renderedContent) + + "\n\n[系统提示] 以下附件未传入模型：" + String.join("、", skippedAttachments) + + "。请直接告知用户具体原因，不要尝试用其他工具读取这些文件。"; + } + if (mediaList.isEmpty()) { - return new UserMessage(renderedContent); + return new UserMessage(finalText); } return UserMessage.builder() - .text(renderedContent) + .text(finalText) .media(mediaList) .build(); } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ModelConfigEntity.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ModelConfigEntity.java index 139fe2c2..c2eac724 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ModelConfigEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ModelConfigEntity.java @@ -55,6 +55,13 @@ public class ModelConfigEntity { */ private String modelType; + /** + * Declared modalities the chat model can natively consume, JSON array of lowercase + * names, e.g. {@code ["vision","video","audio"]}. {@code null} or blank → defer to + * {@link vip.mate.llm.service.ModelCapabilityService} built-in heuristics. + */ + private String modalities; + @TableField(fill = FieldFill.INSERT) private LocalDateTime createTime; diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelCapabilityService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelCapabilityService.java new file mode 100644 index 00000000..23f87841 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelCapabilityService.java @@ -0,0 +1,205 @@ +package vip.mate.llm.service; + +import cn.hutool.core.util.StrUtil; +import cn.hutool.json.JSONUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Resolves which modalities (vision / video / audio) a chat model can natively consume. + *

+ * Two-layer lookup: + *

    + *
  1. Explicit DB declaration on {@code mate_model_config.modalities} (JSON array of + * lowercase modality names) — user opt-in, takes precedence.
    2. + *
  2. Built-in heuristics keyed by lowercase model-name prefix; longest match wins. + * Granularity is per-model (e.g. {@code glm-4v-plus} supports video but + * {@code glm-4v} does not), not per-family.
    3. + *
+ *

+ * Resolved at agent build time and cached per agent — see + * {@link vip.mate.agent.AgentGraphBuilder#buildAgent}. + */ +@Slf4j +@Service +public class ModelCapabilityService { + + public enum Modality { + TEXT, VISION, VIDEO, AUDIO + } + + private static final Map> BUILTIN; + + static { + // Capability data current as of 2026-04. Keys are lowercase model-name prefixes; + // longest match wins. When a vendor ships a new model, prefer extending this + // table over relying on DB overrides — overrides are the per-deployment safety + // valve, but new models should "just work" out of the box. + Map> m = new LinkedHashMap<>(); + + // ===== Zhipu GLM ===== + // Granularity matters: 4v (no video) vs 4v-plus (video). 4.1v / 4.5v / 4.6v / 5v + // lines are all video-capable across variants (turbo / flash / thinking). + m.put("glm-5v", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("glm-4.6v", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("glm-4.5v", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("glm-4.1v", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("glm-4v-plus", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("glm-4v-flash", EnumSet.of(Modality.VISION)); + m.put("glm-4v", EnumSet.of(Modality.VISION)); + + // ===== Alibaba Qwen-VL / Omni ===== + // Qwen3-VL (Sept 2025+, all sizes 2B/4B/8B/32B/30B-A3B/235B-A22B) and Qwen3.5-Omni + // (Mar 2026) natively handle video. Older qwen-vl-plus / qwen-vl base are image-only. + m.put("qwen3.5-omni", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO)); + m.put("qwen3-omni", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO)); + m.put("qwen2.5-omni", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO)); + m.put("qwen3-vl", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("qwen2.5-vl", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("qwen2-vl", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("qwen-vl-max", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("qwen-vl-plus", EnumSet.of(Modality.VISION)); + m.put("qwen-vl", EnumSet.of(Modality.VISION)); + + // ===== OpenAI ===== + // IMPORTANT: as of 2026-04 the OpenAI Chat Completions / Responses APIs do NOT + // accept video files natively for any model — including gpt-4o and gpt-5.x. The + // recommended workflow is still client-side frame extraction. So vision yes, + // video no, regardless of marketing copy that says "multimodal video". + m.put("gpt-5", EnumSet.of(Modality.VISION)); + m.put("gpt-4.1", EnumSet.of(Modality.VISION)); + m.put("gpt-4o", EnumSet.of(Modality.VISION)); + m.put("gpt-4-vision", EnumSet.of(Modality.VISION)); + + // ===== Google Gemini ===== + // Native multimodal across the 1.5/2/2.5 lines, including pro/flash/flash-lite. + m.put("gemini-2.5", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO)); + m.put("gemini-2", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO)); + m.put("gemini-1.5", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO)); + + // ===== Anthropic Claude ===== + // Vision yes (image), native video no — Anthropic's API only accepts images. + m.put("claude-4.7", EnumSet.of(Modality.VISION)); + m.put("claude-4.5", EnumSet.of(Modality.VISION)); + m.put("claude-4", EnumSet.of(Modality.VISION)); + m.put("claude-3.7", EnumSet.of(Modality.VISION)); + m.put("claude-3.5", EnumSet.of(Modality.VISION)); + m.put("claude-opus", EnumSet.of(Modality.VISION)); + m.put("claude-sonnet", EnumSet.of(Modality.VISION)); + m.put("claude-haiku", EnumSet.of(Modality.VISION)); + + // ===== DeepSeek ===== + // V4 (Apr 2026) is the first DeepSeek line with native multimodal — image + video. + // V3 and earlier are text-only (no entry → defaults to text only). + m.put("deepseek-v4", EnumSet.of(Modality.VISION, Modality.VIDEO)); + + // ===== ByteDance Doubao / Seed ===== + // Seed 2.0 Pro (Feb 2026) handles hour-long videos. Seed1.5-VL also supports video. + m.put("doubao-seed-2", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("seed-1.5-vl", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("doubao-vision", EnumSet.of(Modality.VISION)); + + // ===== Moonshot Kimi ===== + // K2.6 (Apr 2026) added video; K2.5 was image-only. + m.put("kimi-k2.6", EnumSet.of(Modality.VISION, Modality.VIDEO)); + m.put("kimi-k2.5", EnumSet.of(Modality.VISION)); + + // ===== MiniMax ===== + // MiniMax-VL-01 / abab-vision are vision multimodal. Native video INPUT is not + // documented in the platform API — Hailuo / video-01 are generation models, not input. + m.put("minimax-vl", EnumSet.of(Modality.VISION)); + m.put("abab-vision", EnumSet.of(Modality.VISION)); + + // ===== Tencent Hunyuan ===== + // Hunyuan-Vision-1.5 / Hunyuan-Large-Vision are image-only vision LLMs. + // HunyuanVideo / HunyuanCustom are video GENERATION (output), not input. + m.put("hunyuan-large-vision", EnumSet.of(Modality.VISION)); + m.put("hunyuan-vision", EnumSet.of(Modality.VISION)); + + // ===== xAI Grok ===== + // Grok 2/3/4 accept image input. Grok Imagine is video generation, not video input. + m.put("grok", EnumSet.of(Modality.VISION)); + + // ===== Mistral ===== + // Pixtral / Mistral Small 4 take images via the Pixtral vision stack. No native video. + m.put("pixtral", EnumSet.of(Modality.VISION)); + m.put("mistral-small-4", EnumSet.of(Modality.VISION)); + + // ===== Meta Llama ===== + // Llama 4 (Scout / Maverick, Apr 2026) is the first Llama line natively trained on + // text + image + video. Llama 3.x was image-only via separate adapters. + m.put("llama-4", EnumSet.of(Modality.VISION, Modality.VIDEO)); + + BUILTIN = Collections.unmodifiableMap(m); + } + + /** + * Resolve the full capability set for a model. + * + * @param modelName the chat model identifier (e.g. {@code glm-4v-plus}) + * @param modalitiesJson optional JSON array of declared modality names (case-insensitive), + * e.g. {@code ["vision","video"]}; when present and parseable, takes + * precedence over the heuristic table; pass {@code null} or blank to + * defer entirely to heuristics + * @return an {@link EnumSet} of modalities the model can consume; always non-null, + * {@link Modality#TEXT} is implicit and always included + */ + public EnumSet resolve(String modelName, String modalitiesJson) { + EnumSet result = EnumSet.of(Modality.TEXT); + + if (StrUtil.isNotBlank(modalitiesJson)) { + EnumSet declared = parseDeclared(modalitiesJson, modelName); + if (declared != null) { + result.addAll(declared); + return result; + } + // parse failed → fall through to heuristics + } + + if (StrUtil.isBlank(modelName)) { + return result; + } + + String lowered = modelName.toLowerCase(); + BUILTIN.entrySet().stream() + .filter(e -> lowered.startsWith(e.getKey())) + .max(Comparator.comparingInt(e -> e.getKey().length())) + .map(Map.Entry::getValue) + .ifPresent(result::addAll); + return result; + } + + /** + * @return {@code true} if the model supports the given modality + */ + public boolean supports(String modelName, String modalitiesJson, Modality modality) { + return resolve(modelName, modalitiesJson).contains(modality); + } + + private EnumSet parseDeclared(String json, String modelName) { + try { + List declared = JSONUtil.toList(json, String.class); + EnumSet set = EnumSet.noneOf(Modality.class); + for (String name : declared) { + if (StrUtil.isBlank(name)) continue; + try { + set.add(Modality.valueOf(name.trim().toUpperCase())); + } catch (IllegalArgumentException ignore) { + log.warn("Unknown modality '{}' declared on model '{}'", name, modelName); + } + } + return set; + } catch (Exception e) { + log.warn("Invalid modalities JSON for model '{}': {} — falling back to heuristics", + modelName, json); + return null; + } + } +} diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V66__add_model_capabilities.sql b/mateclaw-server/src/main/resources/db/migration/h2/V66__add_model_capabilities.sql new file mode 100644 index 00000000..178b9009 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V66__add_model_capabilities.sql @@ -0,0 +1,4 @@ +-- V66: Per-model capability declaration (issue #44) +-- Optional JSON array such as ["vision","video","audio"]; NULL falls back to +-- ModelCapabilityService built-in heuristics. +ALTER TABLE mate_model_config ADD COLUMN IF NOT EXISTS modalities VARCHAR(512) DEFAULT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V66__add_model_capabilities.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V66__add_model_capabilities.sql new file mode 100644 index 00000000..cab4acb9 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V66__add_model_capabilities.sql @@ -0,0 +1,5 @@ +-- V66: Per-model capability declaration (issue #44) +-- MySQL lacks ADD COLUMN IF NOT EXISTS; use INFORMATION_SCHEMA guard instead. +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_model_config' AND COLUMN_NAME = 'modalities'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_model_config ADD COLUMN modalities VARCHAR(512) DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;