mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(agent): per-model multimodal capability resolution (issue #44)
This commit is contained in:
matevip
parent
101aa3209e
commit
47bdb97a3a
@ -100,6 +100,7 @@ public class AgentGraphBuilder {
|
||||
private final ConversationService conversationService;
|
||||
private final ModelConfigService modelConfigService;
|
||||
private final ModelProviderService modelProviderService;
|
||||
private final vip.mate.llm.service.ModelCapabilityService modelCapabilityService;
|
||||
private final PlanningService planningService;
|
||||
private final ToolGuardService toolGuardService;
|
||||
private final vip.mate.tool.guard.service.ToolGuardConfigService toolGuardConfigService;
|
||||
@ -235,6 +236,8 @@ public class AgentGraphBuilder {
|
||||
agent.systemPrompt = enhancedPrompt;
|
||||
agent.maxIterations = maxIter;
|
||||
agent.modelName = runtimeModel.getModelName();
|
||||
agent.modelCapabilities = modelCapabilityService.resolve(
|
||||
runtimeModel.getModelName(), runtimeModel.getModalities());
|
||||
agent.runtimeProviderId = provider != null ? provider.getProviderId() : "";
|
||||
agent.temperature = runtimeModel.getTemperature();
|
||||
agent.maxTokens = runtimeModel.getMaxTokens();
|
||||
|
||||
@ -10,6 +10,7 @@ import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.util.MimeType;
|
||||
import reactor.core.publisher.Flux;
|
||||
import vip.mate.approval.ApprovalPlaceholderUtil;
|
||||
import vip.mate.llm.service.ModelCapabilityService;
|
||||
import vip.mate.workspace.conversation.ConversationService;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||
@ -18,7 +19,9 @@ import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
@ -57,6 +60,13 @@ public abstract class BaseAgent {
|
||||
/** 模型名称 */
|
||||
protected String modelName;
|
||||
|
||||
/**
|
||||
* Modalities the chat model can natively consume (resolved at agent build time
|
||||
* by {@link vip.mate.llm.service.ModelCapabilityService}). Empty set = unknown model,
|
||||
* fall back to text-only behavior. See issue #44.
|
||||
*/
|
||||
protected Set<ModelCapabilityService.Modality> modelCapabilities = EnumSet.noneOf(ModelCapabilityService.Modality.class);
|
||||
|
||||
/** 采样温度 */
|
||||
protected Double temperature;
|
||||
|
||||
@ -483,15 +493,11 @@ public abstract class BaseAgent {
|
||||
|
||||
/**
|
||||
* 判断当前模型是否支持视频输入。
|
||||
* 仅已知支持视频分析的视觉模型（Qwen-VL、GPT-4o、Gemini 等）才注入视频 Media。
|
||||
* 由 {@link ModelCapabilityService} 在 agent 构建时解析并注入到
|
||||
* {@link #modelCapabilities}，per-model 粒度（区分如 glm-4v vs glm-4v-plus）。
|
||||
*/
|
||||
private boolean modelSupportsVideo() {
|
||||
if (modelName == null) return false;
|
||||
String n = modelName.toLowerCase();
|
||||
return (n.contains("qwen") && n.contains("vl"))
|
||||
|| n.contains("gpt-4o")
|
||||
|| n.contains("gemini")
|
||||
|| (n.contains("glm") && n.contains("v"));
|
||||
return modelCapabilities.contains(ModelCapabilityService.Modality.VIDEO);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -501,6 +507,10 @@ public abstract class BaseAgent {
|
||||
protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) {
|
||||
List<MessageContentPart> parts = conversationService.parseMessageParts(message);
|
||||
List<Media> mediaList = new ArrayList<>();
|
||||
// Reasons for attachments that the model cannot consume — surfaced to the agent
|
||||
// via the user message text so it does not hallucinate a tool call to read them.
|
||||
// See issue #44.
|
||||
List<String> skippedAttachments = new ArrayList<>();
|
||||
boolean videoSupported = modelSupportsVideo();
|
||||
|
||||
for (MessageContentPart part : parts) {
|
||||
@ -522,6 +532,7 @@ public abstract class BaseAgent {
|
||||
if (isImage && contentType.contains("svg")) {
|
||||
log.debug("[{}] Skipping SVG attachment (not supported by multimodal API): {}",
|
||||
agentName, part.getFileName());
|
||||
skippedAttachments.add(part.getFileName() + "(SVG 格式，多模态 API 不支持)");
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -529,6 +540,7 @@ public abstract class BaseAgent {
|
||||
if (isVideo && !videoSupported) {
|
||||
log.debug("[{}] Skipping video attachment (model '{}' does not support video): {}",
|
||||
agentName, modelName, part.getFileName());
|
||||
skippedAttachments.add(part.getFileName() + "(当前模型未声明视频能力)");
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -536,6 +548,7 @@ public abstract class BaseAgent {
|
||||
if (isVideo && part.getFileSize() != null && part.getFileSize() > MAX_VIDEO_SIZE_BYTES) {
|
||||
log.warn("[{}] Skipping oversized video attachment ({}MB > 20MB): {}",
|
||||
agentName, part.getFileSize() / (1024 * 1024), part.getFileName());
|
||||
skippedAttachments.add(part.getFileName() + "(视频超过 20MB 大小限制)");
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -547,6 +560,7 @@ public abstract class BaseAgent {
|
||||
if (mediaPath == null) {
|
||||
log.warn("[{}] {} file not found for attachment: {}, path: {}, mediaId: {}",
|
||||
agentName, isVideo ? "Video" : "Image", part.getFileName(), part.getPath(), part.getMediaId());
|
||||
skippedAttachments.add(part.getFileName() + "(文件未找到)");
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
@ -558,14 +572,22 @@ public abstract class BaseAgent {
|
||||
} catch (Exception e) {
|
||||
log.warn("[{}] Failed to create Media for {} {}: {}",
|
||||
agentName, isVideo ? "video" : "image", part.getFileName(), e.getMessage());
|
||||
skippedAttachments.add(part.getFileName() + "(媒体加载失败)");
|
||||
}
|
||||
}
|
||||
|
||||
String finalText = renderedContent;
|
||||
if (!skippedAttachments.isEmpty()) {
|
||||
finalText = (renderedContent == null ? "" : renderedContent)
|
||||
+ "\n\n[系统提示] 以下附件未传入模型：" + String.join("、", skippedAttachments)
|
||||
+ "。请直接告知用户具体原因，不要尝试用其他工具读取这些文件。";
|
||||
}
|
||||
|
||||
if (mediaList.isEmpty()) {
|
||||
return new UserMessage(renderedContent);
|
||||
return new UserMessage(finalText);
|
||||
}
|
||||
return UserMessage.builder()
|
||||
.text(renderedContent)
|
||||
.text(finalText)
|
||||
.media(mediaList)
|
||||
.build();
|
||||
}
|
||||
|
||||
@ -55,6 +55,13 @@ public class ModelConfigEntity {
|
||||
*/
|
||||
private String modelType;
|
||||
|
||||
/**
|
||||
* Declared modalities the chat model can natively consume, JSON array of lowercase
|
||||
* names, e.g. {@code ["vision","video","audio"]}. {@code null} or blank → defer to
|
||||
* {@link vip.mate.llm.service.ModelCapabilityService} built-in heuristics.
|
||||
*/
|
||||
private String modalities;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT)
|
||||
private LocalDateTime createTime;
|
||||
|
||||
|
||||
@ -0,0 +1,205 @@
|
||||
package vip.mate.llm.service;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.EnumSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Resolves which modalities (vision / video / audio) a chat model can natively consume.
|
||||
* <p>
|
||||
* Two-layer lookup:
|
||||
* <ol>
|
||||
* <li>Explicit DB declaration on {@code mate_model_config.modalities} (JSON array of
|
||||
* lowercase modality names) — user opt-in, takes precedence.</li>
|
||||
* <li>Built-in heuristics keyed by lowercase model-name prefix; longest match wins.
|
||||
* Granularity is per-model (e.g. {@code glm-4v-plus} supports video but
|
||||
* {@code glm-4v} does not), not per-family.</li>
|
||||
* </ol>
|
||||
* <p>
|
||||
* Resolved at agent build time and cached per agent — see
|
||||
* {@link vip.mate.agent.AgentGraphBuilder#buildAgent}.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class ModelCapabilityService {
|
||||
|
||||
public enum Modality {
|
||||
TEXT, VISION, VIDEO, AUDIO
|
||||
}
|
||||
|
||||
private static final Map<String, EnumSet<Modality>> BUILTIN;
|
||||
|
||||
static {
|
||||
// Capability data current as of 2026-04. Keys are lowercase model-name prefixes;
|
||||
// longest match wins. When a vendor ships a new model, prefer extending this
|
||||
// table over relying on DB overrides — overrides are the per-deployment safety
|
||||
// valve, but new models should "just work" out of the box.
|
||||
Map<String, EnumSet<Modality>> m = new LinkedHashMap<>();
|
||||
|
||||
// ===== Zhipu GLM =====
|
||||
// Granularity matters: 4v (no video) vs 4v-plus (video). 4.1v / 4.5v / 4.6v / 5v
|
||||
// lines are all video-capable across variants (turbo / flash / thinking).
|
||||
m.put("glm-5v", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("glm-4.6v", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("glm-4.5v", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("glm-4.1v", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("glm-4v-plus", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("glm-4v-flash", EnumSet.of(Modality.VISION));
|
||||
m.put("glm-4v", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== Alibaba Qwen-VL / Omni =====
|
||||
// Qwen3-VL (Sept 2025+, all sizes 2B/4B/8B/32B/30B-A3B/235B-A22B) and Qwen3.5-Omni
|
||||
// (Mar 2026) natively handle video. Older qwen-vl-plus / qwen-vl base are image-only.
|
||||
m.put("qwen3.5-omni", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO));
|
||||
m.put("qwen3-omni", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO));
|
||||
m.put("qwen2.5-omni", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO));
|
||||
m.put("qwen3-vl", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("qwen2.5-vl", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("qwen2-vl", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("qwen-vl-max", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("qwen-vl-plus", EnumSet.of(Modality.VISION));
|
||||
m.put("qwen-vl", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== OpenAI =====
|
||||
// IMPORTANT: as of 2026-04 the OpenAI Chat Completions / Responses APIs do NOT
|
||||
// accept video files natively for any model — including gpt-4o and gpt-5.x. The
|
||||
// recommended workflow is still client-side frame extraction. So vision yes,
|
||||
// video no, regardless of marketing copy that says "multimodal video".
|
||||
m.put("gpt-5", EnumSet.of(Modality.VISION));
|
||||
m.put("gpt-4.1", EnumSet.of(Modality.VISION));
|
||||
m.put("gpt-4o", EnumSet.of(Modality.VISION));
|
||||
m.put("gpt-4-vision", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== Google Gemini =====
|
||||
// Native multimodal across the 1.5/2/2.5 lines, including pro/flash/flash-lite.
|
||||
m.put("gemini-2.5", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO));
|
||||
m.put("gemini-2", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO));
|
||||
m.put("gemini-1.5", EnumSet.of(Modality.VISION, Modality.VIDEO, Modality.AUDIO));
|
||||
|
||||
// ===== Anthropic Claude =====
|
||||
// Vision yes (image), native video no — Anthropic's API only accepts images.
|
||||
m.put("claude-4.7", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-4.5", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-4", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-3.7", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-3.5", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-opus", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-sonnet", EnumSet.of(Modality.VISION));
|
||||
m.put("claude-haiku", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== DeepSeek =====
|
||||
// V4 (Apr 2026) is the first DeepSeek line with native multimodal — image + video.
|
||||
// V3 and earlier are text-only (no entry → defaults to text only).
|
||||
m.put("deepseek-v4", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
|
||||
// ===== ByteDance Doubao / Seed =====
|
||||
// Seed 2.0 Pro (Feb 2026) handles hour-long videos. Seed1.5-VL also supports video.
|
||||
m.put("doubao-seed-2", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("seed-1.5-vl", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("doubao-vision", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== Moonshot Kimi =====
|
||||
// K2.6 (Apr 2026) added video; K2.5 was image-only.
|
||||
m.put("kimi-k2.6", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
m.put("kimi-k2.5", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== MiniMax =====
|
||||
// MiniMax-VL-01 / abab-vision are vision multimodal. Native video INPUT is not
|
||||
// documented in the platform API — Hailuo / video-01 are generation models, not input.
|
||||
m.put("minimax-vl", EnumSet.of(Modality.VISION));
|
||||
m.put("abab-vision", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== Tencent Hunyuan =====
|
||||
// Hunyuan-Vision-1.5 / Hunyuan-Large-Vision are image-only vision LLMs.
|
||||
// HunyuanVideo / HunyuanCustom are video GENERATION (output), not input.
|
||||
m.put("hunyuan-large-vision", EnumSet.of(Modality.VISION));
|
||||
m.put("hunyuan-vision", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== xAI Grok =====
|
||||
// Grok 2/3/4 accept image input. Grok Imagine is video generation, not video input.
|
||||
m.put("grok", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== Mistral =====
|
||||
// Pixtral / Mistral Small 4 take images via the Pixtral vision stack. No native video.
|
||||
m.put("pixtral", EnumSet.of(Modality.VISION));
|
||||
m.put("mistral-small-4", EnumSet.of(Modality.VISION));
|
||||
|
||||
// ===== Meta Llama =====
|
||||
// Llama 4 (Scout / Maverick, Apr 2026) is the first Llama line natively trained on
|
||||
// text + image + video. Llama 3.x was image-only via separate adapters.
|
||||
m.put("llama-4", EnumSet.of(Modality.VISION, Modality.VIDEO));
|
||||
|
||||
BUILTIN = Collections.unmodifiableMap(m);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the full capability set for a model.
|
||||
*
|
||||
* @param modelName the chat model identifier (e.g. {@code glm-4v-plus})
|
||||
* @param modalitiesJson optional JSON array of declared modality names (case-insensitive),
|
||||
* e.g. {@code ["vision","video"]}; when present and parseable, takes
|
||||
* precedence over the heuristic table; pass {@code null} or blank to
|
||||
* defer entirely to heuristics
|
||||
* @return an {@link EnumSet} of modalities the model can consume; always non-null,
|
||||
* {@link Modality#TEXT} is implicit and always included
|
||||
*/
|
||||
public EnumSet<Modality> resolve(String modelName, String modalitiesJson) {
|
||||
EnumSet<Modality> result = EnumSet.of(Modality.TEXT);
|
||||
|
||||
if (StrUtil.isNotBlank(modalitiesJson)) {
|
||||
EnumSet<Modality> declared = parseDeclared(modalitiesJson, modelName);
|
||||
if (declared != null) {
|
||||
result.addAll(declared);
|
||||
return result;
|
||||
}
|
||||
// parse failed → fall through to heuristics
|
||||
}
|
||||
|
||||
if (StrUtil.isBlank(modelName)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
String lowered = modelName.toLowerCase();
|
||||
BUILTIN.entrySet().stream()
|
||||
.filter(e -> lowered.startsWith(e.getKey()))
|
||||
.max(Comparator.comparingInt(e -> e.getKey().length()))
|
||||
.map(Map.Entry::getValue)
|
||||
.ifPresent(result::addAll);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@code true} if the model supports the given modality
|
||||
*/
|
||||
public boolean supports(String modelName, String modalitiesJson, Modality modality) {
|
||||
return resolve(modelName, modalitiesJson).contains(modality);
|
||||
}
|
||||
|
||||
private EnumSet<Modality> parseDeclared(String json, String modelName) {
|
||||
try {
|
||||
List<String> declared = JSONUtil.toList(json, String.class);
|
||||
EnumSet<Modality> set = EnumSet.noneOf(Modality.class);
|
||||
for (String name : declared) {
|
||||
if (StrUtil.isBlank(name)) continue;
|
||||
try {
|
||||
set.add(Modality.valueOf(name.trim().toUpperCase()));
|
||||
} catch (IllegalArgumentException ignore) {
|
||||
log.warn("Unknown modality '{}' declared on model '{}'", name, modelName);
|
||||
}
|
||||
}
|
||||
return set;
|
||||
} catch (Exception e) {
|
||||
log.warn("Invalid modalities JSON for model '{}': {} — falling back to heuristics",
|
||||
modelName, json);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,4 @@
|
||||
-- V66: Per-model capability declaration (issue #44)
|
||||
-- Optional JSON array such as ["vision","video","audio"]; NULL falls back to
|
||||
-- ModelCapabilityService built-in heuristics.
|
||||
ALTER TABLE mate_model_config ADD COLUMN IF NOT EXISTS modalities VARCHAR(512) DEFAULT NULL;
|
||||
@ -0,0 +1,5 @@
|
||||
-- V66: Per-model capability declaration (issue #44)
|
||||
-- MySQL lacks ADD COLUMN IF NOT EXISTS; use INFORMATION_SCHEMA guard instead.
|
||||
SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_model_config' AND COLUMN_NAME = 'modalities');
|
||||
SET @s := IF(@c = 0, 'ALTER TABLE mate_model_config ADD COLUMN modalities VARCHAR(512) DEFAULT NULL', 'SELECT 1');
|
||||
PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
Loading…
Reference in New Issue
Block a user