From eb780327525863d5c5704bb142063f577a36cde7 Mon Sep 17 00:00:00 2001 From: matevip Date: Mon, 6 Apr 2026 09:42:54 +0800 Subject: [PATCH] feat(video): support video upload, preview, and multimodal analysis --- .../vip/mate/agent/AgentGraphBuilder.java | 117 ++++++++++++++++++ .../main/java/vip/mate/agent/BaseAgent.java | 70 ++++++++--- .../vip/mate/channel/web/ChatController.java | 19 +++ mateclaw-ui/src/components/chat/ChatInput.vue | 12 +- .../src/components/chat/MessageBubble.vue | 46 ++++++- .../composables/useAuthenticatedAttachment.ts | 14 +++ mateclaw-ui/src/types/index.ts | 2 +- mateclaw-ui/src/views/ChatConsole.vue | 17 ++- 8 files changed, 269 insertions(+), 28 deletions(-) diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 27e7a3c2..158f4f95 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -782,6 +782,7 @@ public class AgentGraphBuilder { MultiValueMap additionalHttpHeader) { chatRequest = patchReasoningContent(chatRequest); chatRequest = stripReasoningEffortIfIncompatible(chatRequest); + chatRequest = patchVideoMediaContent(chatRequest); if (kimiSearchEnabled) { chatRequest = injectKimiWebSearch(chatRequest); } @@ -800,6 +801,7 @@ public class AgentGraphBuilder { MultiValueMap additionalHttpHeader) { chatRequest = patchReasoningContent(chatRequest); chatRequest = stripReasoningEffortIfIncompatible(chatRequest); + chatRequest = patchVideoMediaContent(chatRequest); if (kimiSearchEnabled) { chatRequest = injectKimiWebSearch(chatRequest); } @@ -1378,6 +1380,121 @@ public class AgentGraphBuilder { return family.isThinking(); } + /** + * 将 Spring AI 错误地序列化为 image_url 的视频内容块转换为 video_url 格式。 + *

+ * Spring AI 1.x 的 MediaContent 没有 video_url 类型，所有非 audio/pdf 的 Media + * 都被序列化为 image_url。智谱 GLM-5V 等模型要求视频使用 video_url 格式， + * 否则会报"图片输入格式/解析错误"。 + *

+ * 此方法遍历 user 消息的 rawContent，将 data:video/* 前缀的 image_url 替换为 video_url。 + */ + @SuppressWarnings("unchecked") + private static OpenAiApi.ChatCompletionRequest patchVideoMediaContent(OpenAiApi.ChatCompletionRequest request) { + if (request.messages() == null || request.messages().isEmpty()) { + return request; + } + + boolean needsPatch = false; + for (var msg : request.messages()) { + if (msg.role() == OpenAiApi.ChatCompletionMessage.Role.USER) { + Object raw = msg.rawContent(); + if (raw instanceof List parts) { + for (Object part : parts) { + // 检查是否为 MediaContent record + if (part instanceof OpenAiApi.ChatCompletionMessage.MediaContent mc + && "image_url".equals(mc.type()) + && mc.imageUrl() != null + && mc.imageUrl().url() != null + && mc.imageUrl().url().startsWith("data:video/")) { + needsPatch = true; + break; + } + // 检查是否为 Map（Spring AI 内部用 LinkedHashMap 表示 content parts） + if (part instanceof java.util.Map map) { + Object type = map.get("type"); + if ("image_url".equals(type)) { + Object imgUrlObj = map.get("image_url"); + if (imgUrlObj instanceof java.util.Map imgUrl) { + Object url = imgUrl.get("url"); + if (url instanceof String urlStr && urlStr.startsWith("data:video/")) { + needsPatch = true; + break; + } + } + } + } + } + } + } + if (needsPatch) break; + } + if (!needsPatch) { + return request; + } + + List patched = request.messages().stream().map(msg -> { + if (msg.role() != OpenAiApi.ChatCompletionMessage.Role.USER || !(msg.rawContent() instanceof List parts)) { + return msg; + } + List newParts = new ArrayList<>(); + for (Object part : parts) { + String videoDataUrl = null; + + // 场景 1：MediaContent record（Spring AI 原生构建） + if (part instanceof OpenAiApi.ChatCompletionMessage.MediaContent mc + && "image_url".equals(mc.type()) + && mc.imageUrl() != null && mc.imageUrl().url() != null + && mc.imageUrl().url().startsWith("data:video/")) { + videoDataUrl = mc.imageUrl().url(); + } + // 场景 2：Map（Jackson 反序列化或 Spring AI 内部用 Map 表示） + if (videoDataUrl == null && part instanceof java.util.Map map + && "image_url".equals(map.get("type"))) { + Object imgUrlObj = map.get("image_url"); + if (imgUrlObj instanceof java.util.Map imgUrl) { + Object url = imgUrl.get("url"); + if (url instanceof String urlStr && urlStr.startsWith("data:video/")) { + videoDataUrl = urlStr; + } + } + } + + if (videoDataUrl != null) { + // 替换为 video_url 格式 + newParts.add(Map.of( + "type", "video_url", + "video_url", Map.of("url", videoDataUrl) + )); + } else { + newParts.add(part); + } + } + return new OpenAiApi.ChatCompletionMessage( + newParts, msg.role(), msg.name(), msg.toolCallId(), + msg.toolCalls(), msg.refusal(), msg.audioOutput(), + msg.annotations(), msg.reasoningContent()); + }).toList(); + + return new OpenAiApi.ChatCompletionRequest( + patched, + request.model(), request.store(), request.metadata(), + request.frequencyPenalty(), request.logitBias(), + request.logprobs(), request.topLogprobs(), + request.maxTokens(), request.maxCompletionTokens(), + request.n(), request.outputModalities(), request.audioParameters(), + request.presencePenalty(), request.responseFormat(), + request.seed(), request.serviceTier(), request.stop(), + request.stream(), request.streamOptions(), + request.temperature(), request.topP(), + request.tools(), request.toolChoice(), request.parallelToolCalls(), + request.user(), request.reasoningEffort(), + request.webSearchOptions(), request.verbosity(), + request.promptCacheKey(), request.safetyIdentifier(), + request.extraBody() + ); + } + private void logOpenAiRequest(ModelProviderEntity provider, OpenAiApi.ChatCompletionRequest chatRequest) { try { log.info("OpenAI-compatible request: provider={}, body={}", diff --git a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java index f95fd2df..4d3d7562 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java @@ -225,42 +225,78 @@ public abstract class BaseAgent { }; } + private static final long MAX_VIDEO_SIZE_BYTES = 20 * 1024 * 1024; // 20MB + /** - * 构建 UserMessage，支持 multimodal：如果消息包含图片附件，直接注入 Spring AI Media 对象， - * 让模型在 prompt 中直接看到图片，不需要再调 MCP read_media_file 工具。 + * 判断当前模型是否支持视频输入。 + * 仅已知支持视频分析的视觉模型（Qwen-VL、GPT-4o、Gemini 等）才注入视频 Media。 + */ + private boolean modelSupportsVideo() { + if (modelName == null) return false; + String n = modelName.toLowerCase(); + return (n.contains("qwen") && n.contains("vl")) + || n.contains("gpt-4o") + || n.contains("gemini") + || (n.contains("glm") && n.contains("v")); + } + + /** + * 构建 UserMessage，支持 multimodal：如果消息包含图片/视频附件，直接注入 Spring AI Media 对象， + * 让模型在 prompt 中直接看到媒体内容，不需要再调 MCP read_media_file 工具。 */ protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { List parts = conversationService.parseMessageParts(message); List mediaList = new ArrayList<>(); + boolean videoSupported = modelSupportsVideo(); for (MessageContentPart part : parts) { - if (part == null || !"file".equals(part.getType())) { - continue; - } + if (part == null) continue; + String partType = part.getType(); String contentType = part.getContentType(); - if (contentType == null || !contentType.startsWith("image/")) { - continue; - } + if (contentType == null) continue; + + boolean isImage = "file".equals(partType) && contentType.startsWith("image/"); + boolean isVideo = ("video".equals(partType) || "file".equals(partType)) && contentType.startsWith("video/"); + + if (!isImage && !isVideo) continue; + // SVG 是 XML 文本，不是光栅图片，LLM multimodal API 不支持 - if (contentType.contains("svg")) { + if (isImage && contentType.contains("svg")) { log.debug("[{}] Skipping SVG attachment (not supported by multimodal API): {}", agentName, part.getFileName()); continue; } - // 解析图片文件路径：先尝试原始 path，再尝试拼接工作目录 - Path imagePath = resolveImagePath(part.getPath()); - if (imagePath == null) { - log.warn("[{}] Image file not found for attachment: {}, path: {}", - agentName, part.getFileName(), part.getPath()); + + // 视频仅在模型支持时注入，否则跳过（避免发送给非视觉模型导致 400 错误） + if (isVideo && !videoSupported) { + log.debug("[{}] Skipping video attachment (model '{}' does not support video): {}", + agentName, modelName, part.getFileName()); + continue; + } + + // 视频文件大小保护 + if (isVideo && part.getFileSize() != null && part.getFileSize() > MAX_VIDEO_SIZE_BYTES) { + log.warn("[{}] Skipping oversized video attachment ({}MB > 20MB): {}", + agentName, part.getFileSize() / (1024 * 1024), part.getFileName()); + continue; + } + + // 解析媒体文件路径：先尝试原始 path，再尝试拼接工作目录 + Path mediaPath = resolveImagePath(part.getPath()); + if (mediaPath == null) { + log.warn("[{}] {} file not found for attachment: {}, path: {}", + agentName, isVideo ? "Video" : "Image", part.getFileName(), part.getPath()); continue; } try { MimeType mimeType = MimeType.valueOf(contentType); - Media media = new Media(mimeType, new FileSystemResource(imagePath)); + Media media = new Media(mimeType, new FileSystemResource(mediaPath)); mediaList.add(media); - log.debug("[{}] Injected image into prompt: {} ({})", agentName, part.getFileName(), imagePath); + log.debug("[{}] Injected {} into prompt: {} ({})", + agentName, isVideo ? "video" : "image", part.getFileName(), mediaPath); } catch (Exception e) { - log.warn("[{}] Failed to create Media for image {}: {}", agentName, part.getFileName(), e.getMessage()); + log.warn("[{}] Failed to create Media for {} {}: {}", + agentName, isVideo ? "video" : "image", part.getFileName(), e.getMessage()); } } diff --git a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java index 4f71a0bd..d6566c63 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java @@ -867,6 +867,10 @@ public class ChatController { Resource resource = new FileSystemResource(filePath); String contentType = Files.probeContentType(filePath); + // probeContentType 在部分平台不识别视频格式，通过扩展名 fallback + if (contentType == null) { + contentType = guessContentTypeByExtension(filePath.getFileName().toString()); + } MediaType mediaType = MediaType.APPLICATION_OCTET_STREAM; if (contentType != null) { try { @@ -1107,6 +1111,8 @@ public class ChatController { switch (part.getType()) { case "text", "thinking" -> appendPromptLine(builder, part.getText()); case "file" -> appendPromptLine(builder, "附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")"); + case "image" -> appendPromptLine(builder, "图片附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")"); + case "video" -> appendPromptLine(builder, "视频附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")"); default -> appendPromptLine(builder, part.getText()); } } @@ -1127,6 +1133,19 @@ public class ChatController { return text == null ? "" : text; } + private static final java.util.Map MEDIA_CONTENT_TYPES = java.util.Map.of( + "mp4", "video/mp4", "webm", "video/webm", "mov", "video/quicktime", + "avi", "video/x-msvideo", "mkv", "video/x-matroska", "mpeg", "video/mpeg", + "mp3", "audio/mpeg", "wav", "audio/wav", "ogg", "audio/ogg" + ); + + private static String guessContentTypeByExtension(String fileName) { + if (fileName == null) return null; + int dot = fileName.lastIndexOf('.'); + if (dot < 0 || dot == fileName.length() - 1) return null; + return MEDIA_CONTENT_TYPES.get(fileName.substring(dot + 1).toLowerCase()); + } + /** * 注册 SseEmitter 的完整生命周期回调 */ diff --git a/mateclaw-ui/src/components/chat/ChatInput.vue b/mateclaw-ui/src/components/chat/ChatInput.vue index 6e6d7c8e..4414bcb7 100644 --- a/mateclaw-ui/src/components/chat/ChatInput.vue +++ b/mateclaw-ui/src/components/chat/ChatInput.vue @@ -17,6 +17,7 @@ :class="{ 'attachment-chip--dir': attachment.contentType === 'inode/directory', 'attachment-chip--image': attachment.contentType?.startsWith('image/'), + 'attachment-chip--video': attachment.contentType?.startsWith('video/'), }" > @@ -27,6 +28,14 @@ class="attachment-chip__thumbnail" loading="lazy" /> + +