feat(video): support video upload, preview, and multimodal analysis

mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java

@@ -782,6 +782,7 @@ public class AgentGraphBuilder {
                     MultiValueMap<String, String> additionalHttpHeader) {
                 chatRequest = patchReasoningContent(chatRequest);
                 chatRequest = stripReasoningEffortIfIncompatible(chatRequest);
+                chatRequest = patchVideoMediaContent(chatRequest);
                 if (kimiSearchEnabled) {
                     chatRequest = injectKimiWebSearch(chatRequest);
                 }
@@ -800,6 +801,7 @@ public class AgentGraphBuilder {
                     MultiValueMap<String, String> additionalHttpHeader) {
                 chatRequest = patchReasoningContent(chatRequest);
                 chatRequest = stripReasoningEffortIfIncompatible(chatRequest);
+                chatRequest = patchVideoMediaContent(chatRequest);
                 if (kimiSearchEnabled) {
                     chatRequest = injectKimiWebSearch(chatRequest);
                 }
@@ -1378,6 +1380,121 @@ public class AgentGraphBuilder {
         return family.isThinking();
     }
 
+    /**
+     * 将 Spring AI 错误地序列化为 image_url 的视频内容块转换为 video_url 格式。
+     * <p>
+     * Spring AI 1.x 的 MediaContent 没有 video_url 类型，所有非 audio/pdf 的 Media
+     * 都被序列化为 image_url。智谱 GLM-5V 等模型要求视频使用 video_url 格式，
+     * 否则会报"图片输入格式/解析错误"。
+     * <p>
+     * 此方法遍历 user 消息的 rawContent，将 data:video/* 前缀的 image_url 替换为 video_url。
+     */
+    @SuppressWarnings("unchecked")
+    private static OpenAiApi.ChatCompletionRequest patchVideoMediaContent(OpenAiApi.ChatCompletionRequest request) {
+        if (request.messages() == null || request.messages().isEmpty()) {
+            return request;
+        }
+
+        boolean needsPatch = false;
+        for (var msg : request.messages()) {
+            if (msg.role() == OpenAiApi.ChatCompletionMessage.Role.USER) {
+                Object raw = msg.rawContent();
+                if (raw instanceof List<?> parts) {
+                    for (Object part : parts) {
+                        // 检查是否为 MediaContent record
+                        if (part instanceof OpenAiApi.ChatCompletionMessage.MediaContent mc
+                                && "image_url".equals(mc.type())
+                                && mc.imageUrl() != null
+                                && mc.imageUrl().url() != null
+                                && mc.imageUrl().url().startsWith("data:video/")) {
+                            needsPatch = true;
+                            break;
+                        }
+                        // 检查是否为 Map（Spring AI 内部用 LinkedHashMap 表示 content parts）
+                        if (part instanceof java.util.Map<?,?> map) {
+                            Object type = map.get("type");
+                            if ("image_url".equals(type)) {
+                                Object imgUrlObj = map.get("image_url");
+                                if (imgUrlObj instanceof java.util.Map<?,?> imgUrl) {
+                                    Object url = imgUrl.get("url");
+                                    if (url instanceof String urlStr && urlStr.startsWith("data:video/")) {
+                                        needsPatch = true;
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if (needsPatch) break;
+        }
+        if (!needsPatch) {
+            return request;
+        }
+
+        List<OpenAiApi.ChatCompletionMessage> patched = request.messages().stream().map(msg -> {
+            if (msg.role() != OpenAiApi.ChatCompletionMessage.Role.USER || !(msg.rawContent() instanceof List<?> parts)) {
+                return msg;
+            }
+            List<Object> newParts = new ArrayList<>();
+            for (Object part : parts) {
+                String videoDataUrl = null;
+
+                // 场景 1：MediaContent record（Spring AI 原生构建）
+                if (part instanceof OpenAiApi.ChatCompletionMessage.MediaContent mc
+                        && "image_url".equals(mc.type())
+                        && mc.imageUrl() != null && mc.imageUrl().url() != null
+                        && mc.imageUrl().url().startsWith("data:video/")) {
+                    videoDataUrl = mc.imageUrl().url();
+                }
+                // 场景 2：Map（Jackson 反序列化或 Spring AI 内部用 Map 表示）
+                if (videoDataUrl == null && part instanceof java.util.Map<?,?> map
+                        && "image_url".equals(map.get("type"))) {
+                    Object imgUrlObj = map.get("image_url");
+                    if (imgUrlObj instanceof java.util.Map<?,?> imgUrl) {
+                        Object url = imgUrl.get("url");
+                        if (url instanceof String urlStr && urlStr.startsWith("data:video/")) {
+                            videoDataUrl = urlStr;
+                        }
+                    }
+                }
+
+                if (videoDataUrl != null) {
+                    // 替换为 video_url 格式
+                    newParts.add(Map.of(
+                            "type", "video_url",
+                            "video_url", Map.of("url", videoDataUrl)
+                    ));
+                } else {
+                    newParts.add(part);
+                }
+            }
+            return new OpenAiApi.ChatCompletionMessage(
+                    newParts, msg.role(), msg.name(), msg.toolCallId(),
+                    msg.toolCalls(), msg.refusal(), msg.audioOutput(),
+                    msg.annotations(), msg.reasoningContent());
+        }).toList();
+
+        return new OpenAiApi.ChatCompletionRequest(
+                patched,
+                request.model(), request.store(), request.metadata(),
+                request.frequencyPenalty(), request.logitBias(),
+                request.logprobs(), request.topLogprobs(),
+                request.maxTokens(), request.maxCompletionTokens(),
+                request.n(), request.outputModalities(), request.audioParameters(),
+                request.presencePenalty(), request.responseFormat(),
+                request.seed(), request.serviceTier(), request.stop(),
+                request.stream(), request.streamOptions(),
+                request.temperature(), request.topP(),
+                request.tools(), request.toolChoice(), request.parallelToolCalls(),
+                request.user(), request.reasoningEffort(),
+                request.webSearchOptions(), request.verbosity(),
+                request.promptCacheKey(), request.safetyIdentifier(),
+                request.extraBody()
+        );
+    }
+
     private void logOpenAiRequest(ModelProviderEntity provider, OpenAiApi.ChatCompletionRequest chatRequest) {
         try {
             log.info("OpenAI-compatible request: provider={}, body={}",

mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java

@@ -225,42 +225,78 @@ public abstract class BaseAgent {
         };
     }
 
+    private static final long MAX_VIDEO_SIZE_BYTES = 20 * 1024 * 1024; // 20MB
+
     /**
-     * 构建 UserMessage，支持 multimodal：如果消息包含图片附件，直接注入 Spring AI Media 对象，
+     * 判断当前模型是否支持视频输入。
-     * 让模型在 prompt 中直接看到图片，不需要再调 MCP read_media_file 工具。
+     * 仅已知支持视频分析的视觉模型（Qwen-VL、GPT-4o、Gemini 等）才注入视频 Media。
+     */
+    private boolean modelSupportsVideo() {
+        if (modelName == null) return false;
+        String n = modelName.toLowerCase();
+        return (n.contains("qwen") && n.contains("vl"))
+                || n.contains("gpt-4o")
+                || n.contains("gemini")
+                || (n.contains("glm") && n.contains("v"));
+    }
+
+    /**
+     * 构建 UserMessage，支持 multimodal：如果消息包含图片/视频附件，直接注入 Spring AI Media 对象，
+     * 让模型在 prompt 中直接看到媒体内容，不需要再调 MCP read_media_file 工具。
      */
     protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) {
         List<MessageContentPart> parts = conversationService.parseMessageParts(message);
         List<Media> mediaList = new ArrayList<>();
+        boolean videoSupported = modelSupportsVideo();
 
         for (MessageContentPart part : parts) {
-            if (part == null || !"file".equals(part.getType())) {
+            if (part == null) continue;
-                continue;
+            String partType = part.getType();
-            }
             String contentType = part.getContentType();
-            if (contentType == null || !contentType.startsWith("image/")) {
+            if (contentType == null) continue;
-                continue;
+
-            }
+            boolean isImage = "file".equals(partType) && contentType.startsWith("image/");
+            boolean isVideo = ("video".equals(partType) || "file".equals(partType)) && contentType.startsWith("video/");
+
+            if (!isImage && !isVideo) continue;
+
             // SVG 是 XML 文本，不是光栅图片，LLM multimodal API 不支持
-            if (contentType.contains("svg")) {
+            if (isImage && contentType.contains("svg")) {
                 log.debug("[{}] Skipping SVG attachment (not supported by multimodal API): {}",
                         agentName, part.getFileName());
                 continue;
             }
-            // 解析图片文件路径：先尝试原始 path，再尝试拼接工作目录
+
-            Path imagePath = resolveImagePath(part.getPath());
+            // 视频仅在模型支持时注入，否则跳过（避免发送给非视觉模型导致 400 错误）
-            if (imagePath == null) {
+            if (isVideo && !videoSupported) {
-                log.warn("[{}] Image file not found for attachment: {}, path: {}",
+                log.debug("[{}] Skipping video attachment (model '{}' does not support video): {}",
-                        agentName, part.getFileName(), part.getPath());
+                        agentName, modelName, part.getFileName());
+                continue;
+            }
+
+            // 视频文件大小保护
+            if (isVideo && part.getFileSize() != null && part.getFileSize() > MAX_VIDEO_SIZE_BYTES) {
+                log.warn("[{}] Skipping oversized video attachment ({}MB > 20MB): {}",
+                        agentName, part.getFileSize() / (1024 * 1024), part.getFileName());
+                continue;
+            }
+
+            // 解析媒体文件路径：先尝试原始 path，再尝试拼接工作目录
+            Path mediaPath = resolveImagePath(part.getPath());
+            if (mediaPath == null) {
+                log.warn("[{}] {} file not found for attachment: {}, path: {}",
+                        agentName, isVideo ? "Video" : "Image", part.getFileName(), part.getPath());
                 continue;
             }
             try {
                 MimeType mimeType = MimeType.valueOf(contentType);
-                Media media = new Media(mimeType, new FileSystemResource(imagePath));
+                Media media = new Media(mimeType, new FileSystemResource(mediaPath));
                 mediaList.add(media);
-                log.debug("[{}] Injected image into prompt: {} ({})", agentName, part.getFileName(), imagePath);
+                log.debug("[{}] Injected {} into prompt: {} ({})",
+                        agentName, isVideo ? "video" : "image", part.getFileName(), mediaPath);
             } catch (Exception e) {
-                log.warn("[{}] Failed to create Media for image {}: {}", agentName, part.getFileName(), e.getMessage());
+                log.warn("[{}] Failed to create Media for {} {}: {}",
+                        agentName, isVideo ? "video" : "image", part.getFileName(), e.getMessage());
             }
         }
 

mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java

@@ -867,6 +867,10 @@ public class ChatController {
 
         Resource resource = new FileSystemResource(filePath);
         String contentType = Files.probeContentType(filePath);
+        // probeContentType 在部分平台不识别视频格式，通过扩展名 fallback
+        if (contentType == null) {
+            contentType = guessContentTypeByExtension(filePath.getFileName().toString());
+        }
         MediaType mediaType = MediaType.APPLICATION_OCTET_STREAM;
         if (contentType != null) {
             try {
@@ -1107,6 +1111,8 @@ public class ChatController {
             switch (part.getType()) {
                 case "text", "thinking" -> appendPromptLine(builder, part.getText());
                 case "file" -> appendPromptLine(builder, "附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")");
+                case "image" -> appendPromptLine(builder, "图片附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")");
+                case "video" -> appendPromptLine(builder, "视频附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")");
                 default -> appendPromptLine(builder, part.getText());
             }
         }
@@ -1127,6 +1133,19 @@ public class ChatController {
         return text == null ? "" : text;
     }
 
+    private static final java.util.Map<String, String> MEDIA_CONTENT_TYPES = java.util.Map.of(
+            "mp4", "video/mp4", "webm", "video/webm", "mov", "video/quicktime",
+            "avi", "video/x-msvideo", "mkv", "video/x-matroska", "mpeg", "video/mpeg",
+            "mp3", "audio/mpeg", "wav", "audio/wav", "ogg", "audio/ogg"
+    );
+
+    private static String guessContentTypeByExtension(String fileName) {
+        if (fileName == null) return null;
+        int dot = fileName.lastIndexOf('.');
+        if (dot < 0 || dot == fileName.length() - 1) return null;
+        return MEDIA_CONTENT_TYPES.get(fileName.substring(dot + 1).toLowerCase());
+    }
+
     /**
      * 注册 SseEmitter 的完整生命周期回调
      */

mateclaw-ui/src/components/chat/ChatInput.vue

@@ -17,6 +17,7 @@
         :class="{
           'attachment-chip--dir': attachment.contentType === 'inode/directory',
           'attachment-chip--image': attachment.contentType?.startsWith('image/'),
+          'attachment-chip--video': attachment.contentType?.startsWith('video/'),
         }"
       >
         <!-- 图片缩略图预览（优先用本地 previewUrl，避免 JWT 认证问题） -->
@@ -27,6 +28,14 @@
           class="attachment-chip__thumbnail"
           loading="lazy"
         />
+        <!-- 视频缩略图预览 -->
+        <video
+          v-else-if="attachment.contentType?.startsWith('video/') && (attachment.previewUrl || attachment.url)"
+          :src="attachment.previewUrl || attachment.url"
+          class="attachment-chip__thumbnail"
+          preload="metadata"
+          muted
+        />
         <component
           :is="attachment.url ? 'a' : 'span'"
           :href="attachment.url || undefined"
@@ -416,7 +425,8 @@ defineExpose({
   padding: 6px 8px 6px 12px;
 }
 
-.attachment-chip--image {
+.attachment-chip--image,
+.attachment-chip--video {
   padding: 4px 6px;
 }
 

mateclaw-ui/src/components/chat/MessageBubble.vue

@@ -221,6 +221,19 @@
             />
             <span class="message-attachment-image__name">{{ attachment.name }}</span>
           </div>
+          <div
+            v-for="attachment in videoAttachments"
+            :key="attachment.storedName"
+            class="message-attachment-video"
+          >
+            <video
+              :src="getDisplayUrl(attachment)"
+              controls
+              preload="metadata"
+              playsinline
+            />
+            <span class="message-attachment-video__name">{{ attachment.name }}</span>
+          </div>
           <button
             v-for="attachment in fileAttachments"
             :key="attachment.storedName"
@@ -293,7 +306,7 @@ import type { ChatErrorInfo } from '@/types/chatError'
 
 const { renderMarkdown } = useMarkdownRenderer()
 const { t } = useI18n()
-const { blobUrls, loadAllImages, downloadFile, openImage, getDisplayUrl, revokeAll } = useAuthenticatedAttachment()
+const { blobUrls, loadAllImages, loadAllVideos, downloadFile, openImage, getDisplayUrl, revokeAll } = useAuthenticatedAttachment()
 
 interface Props {
   message: Message
@@ -462,12 +475,18 @@ onBeforeUnmount(() => {
 // --- 附件 ---
 const attachments = computed(() => props.message.attachments || [])
 const imageAttachments = computed(() => attachments.value.filter(a => a.contentType?.startsWith('image/')))
-const fileAttachments = computed(() => attachments.value.filter(a => !a.contentType?.startsWith('image/')))
+const videoAttachments = computed(() => attachments.value.filter(a => a.contentType?.startsWith('video/')))
+const fileAttachments = computed(() => attachments.value.filter(a =>
+  !a.contentType?.startsWith('image/') && !a.contentType?.startsWith('video/')
+))
 
-// 增量加载图片附件的鉴权 blob URL（watch 覆盖首次 + 后续变化）
+// 增量加载图片/视频附件的鉴权 blob URL（watch 覆盖首次 + 后续变化）
 watch(imageAttachments, (atts) => {
   if (atts.length > 0) loadAllImages(atts)
 }, { immediate: true })
+watch(videoAttachments, (atts) => {
+  if (atts.length > 0) loadAllVideos(atts)
+}, { immediate: true })
 
 // --- 时间 ---
 const formattedTime = computed(() => {
@@ -1325,6 +1344,27 @@ watch(isGenerating, (generating) => {
   white-space: nowrap;
 }
 
+.message-attachment-video {
+  border-radius: 12px;
+  overflow: hidden;
+}
+
+.message-attachment-video video {
+  max-width: 400px;
+  max-height: 280px;
+  border-radius: 12px;
+}
+
+.message-attachment-video__name {
+  display: block;
+  margin-top: 4px;
+  font-size: 12px;
+  opacity: 0.76;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
 .message-attachment {
   display: flex;
   align-items: center;

mateclaw-ui/src/composables/useAuthenticatedAttachment.ts

@@ -58,6 +58,19 @@ export function useAuthenticatedAttachment() {
     }
   }
 
+  /**
+   * 批量加载所有视频附件的 blob URL
+   */
+  async function loadAllVideos(attachments: ChatAttachment[]) {
+    const videoAtts = attachments.filter(a => a.contentType?.startsWith('video/'))
+    for (const att of videoAtts) {
+      const key = att.storedName || att.url
+      if (!att.previewUrl && att.url && !blobUrls.value[key]) {
+        await loadBlobUrl(att.url, key)
+      }
+    }
+  }
+
   /**
    * 鉴权下载文件：fetch blob → 创建临时 <a download> → 触发点击
    */
@@ -126,6 +139,7 @@ export function useAuthenticatedAttachment() {
     blobUrls,
     loadBlobUrl,
     loadAllImages,
+    loadAllVideos,
     downloadFile,
     openImage,
     getDisplayUrl,

mateclaw-ui/src/types/index.ts

@@ -143,7 +143,7 @@ export interface MessageMetadata {
 }
 
 export interface MessageContentPart {
-  type: 'text' | 'thinking' | 'file' | 'tool_call'
+  type: 'text' | 'thinking' | 'image' | 'file' | 'audio' | 'video' | 'tool_call'
   text?: string
   fileUrl?: string
   fileName?: string

mateclaw-ui/src/views/ChatConsole.vue

@@ -915,9 +915,10 @@ async function handleFileSelect(files: File[]) {
     for (const file of files) {
       const res: any = await chatApi.uploadFile(currentConversationId.value, file)
       const data = res.data || {}
-      // 图片使用本地 ObjectURL 预览（避免 /api/v1/chat/files/ 需要 JWT 认证导致 <img> 加载失败）
+      // 图片/视频使用本地 ObjectURL 预览（避免 /api/v1/chat/files/ 需要 JWT 认证导致加载失败）
-      const isImage = (data.contentType || file.type || '').startsWith('image/')
+      const ct = data.contentType || file.type || ''
-      const previewUrl = isImage ? URL.createObjectURL(file) : data.url
+      const isPreviewable = ct.startsWith('image/') || ct.startsWith('video/')
+      const previewUrl = isPreviewable ? URL.createObjectURL(file) : data.url
       pendingAttachments.value.push({
         name: data.fileName || file.name,
         size: data.size || file.size,
@@ -959,8 +960,12 @@ function buildOutgoingParts(text: string, attachments: ChatAttachment[]): Messag
   const parts: MessageContentPart[] = []
   if (text) parts.push({ type: 'text', text })
   for (const attachment of attachments) {
+    const ct = attachment.contentType || ''
+    const partType: MessageContentPart['type'] = ct.startsWith('video/') ? 'video'
+      : ct.startsWith('image/') ? 'image'
+      : 'file'
     parts.push({
-      type: 'file',
+      type: partType,
       fileUrl: attachment.url,
       fileName: attachment.name,
       storedName: attachment.storedName,
@@ -1029,8 +1034,8 @@ function normalizeMessage(raw: Message): Message {
   // interrupted 是合法的历史状态（interrupt-with-followup），不映射为 stopped
   if (!msg.status) msg.status = 'completed'
 
-  // 从 file-type contentParts 恢复 attachments（历史消息 API 不返回单独的 attachments 字段）
+  // 从 file/image/video contentParts 恢复 attachments（历史消息 API 不返回单独的 attachments 字段）
-  const fileParts = msg.contentParts.filter(p => p.type === 'file' && p.fileUrl)
+  const fileParts = msg.contentParts.filter(p => (p.type === 'file' || p.type === 'image' || p.type === 'video') && p.fileUrl)
   if (fileParts.length > 0 && (!msg.attachments || msg.attachments.length === 0)) {
     msg.attachments = fileParts.map(p => ({
       name: p.fileName || 'unknown',