mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(video): support video upload, preview, and multimodal analysis
This commit is contained in:
matevip
parent
49aae91e90
commit
eb78032752
@ -782,6 +782,7 @@ public class AgentGraphBuilder {
|
||||
MultiValueMap<String, String> additionalHttpHeader) {
|
||||
chatRequest = patchReasoningContent(chatRequest);
|
||||
chatRequest = stripReasoningEffortIfIncompatible(chatRequest);
|
||||
chatRequest = patchVideoMediaContent(chatRequest);
|
||||
if (kimiSearchEnabled) {
|
||||
chatRequest = injectKimiWebSearch(chatRequest);
|
||||
}
|
||||
@ -800,6 +801,7 @@ public class AgentGraphBuilder {
|
||||
MultiValueMap<String, String> additionalHttpHeader) {
|
||||
chatRequest = patchReasoningContent(chatRequest);
|
||||
chatRequest = stripReasoningEffortIfIncompatible(chatRequest);
|
||||
chatRequest = patchVideoMediaContent(chatRequest);
|
||||
if (kimiSearchEnabled) {
|
||||
chatRequest = injectKimiWebSearch(chatRequest);
|
||||
}
|
||||
@ -1378,6 +1380,121 @@ public class AgentGraphBuilder {
|
||||
return family.isThinking();
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 Spring AI 错误地序列化为 image_url 的视频内容块转换为 video_url 格式。
|
||||
* <p>
|
||||
* Spring AI 1.x 的 MediaContent 没有 video_url 类型，所有非 audio/pdf 的 Media
|
||||
* 都被序列化为 image_url。智谱 GLM-5V 等模型要求视频使用 video_url 格式，
|
||||
* 否则会报"图片输入格式/解析错误"。
|
||||
* <p>
|
||||
* 此方法遍历 user 消息的 rawContent，将 data:video/* 前缀的 image_url 替换为 video_url。
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static OpenAiApi.ChatCompletionRequest patchVideoMediaContent(OpenAiApi.ChatCompletionRequest request) {
|
||||
if (request.messages() == null || request.messages().isEmpty()) {
|
||||
return request;
|
||||
}
|
||||
|
||||
boolean needsPatch = false;
|
||||
for (var msg : request.messages()) {
|
||||
if (msg.role() == OpenAiApi.ChatCompletionMessage.Role.USER) {
|
||||
Object raw = msg.rawContent();
|
||||
if (raw instanceof List<?> parts) {
|
||||
for (Object part : parts) {
|
||||
// 检查是否为 MediaContent record
|
||||
if (part instanceof OpenAiApi.ChatCompletionMessage.MediaContent mc
|
||||
&& "image_url".equals(mc.type())
|
||||
&& mc.imageUrl() != null
|
||||
&& mc.imageUrl().url() != null
|
||||
&& mc.imageUrl().url().startsWith("data:video/")) {
|
||||
needsPatch = true;
|
||||
break;
|
||||
}
|
||||
// 检查是否为 Map（Spring AI 内部用 LinkedHashMap 表示 content parts）
|
||||
if (part instanceof java.util.Map<?,?> map) {
|
||||
Object type = map.get("type");
|
||||
if ("image_url".equals(type)) {
|
||||
Object imgUrlObj = map.get("image_url");
|
||||
if (imgUrlObj instanceof java.util.Map<?,?> imgUrl) {
|
||||
Object url = imgUrl.get("url");
|
||||
if (url instanceof String urlStr && urlStr.startsWith("data:video/")) {
|
||||
needsPatch = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (needsPatch) break;
|
||||
}
|
||||
if (!needsPatch) {
|
||||
return request;
|
||||
}
|
||||
|
||||
List<OpenAiApi.ChatCompletionMessage> patched = request.messages().stream().map(msg -> {
|
||||
if (msg.role() != OpenAiApi.ChatCompletionMessage.Role.USER || !(msg.rawContent() instanceof List<?> parts)) {
|
||||
return msg;
|
||||
}
|
||||
List<Object> newParts = new ArrayList<>();
|
||||
for (Object part : parts) {
|
||||
String videoDataUrl = null;
|
||||
|
||||
// 场景 1：MediaContent record（Spring AI 原生构建）
|
||||
if (part instanceof OpenAiApi.ChatCompletionMessage.MediaContent mc
|
||||
&& "image_url".equals(mc.type())
|
||||
&& mc.imageUrl() != null && mc.imageUrl().url() != null
|
||||
&& mc.imageUrl().url().startsWith("data:video/")) {
|
||||
videoDataUrl = mc.imageUrl().url();
|
||||
}
|
||||
// 场景 2：Map（Jackson 反序列化或 Spring AI 内部用 Map 表示）
|
||||
if (videoDataUrl == null && part instanceof java.util.Map<?,?> map
|
||||
&& "image_url".equals(map.get("type"))) {
|
||||
Object imgUrlObj = map.get("image_url");
|
||||
if (imgUrlObj instanceof java.util.Map<?,?> imgUrl) {
|
||||
Object url = imgUrl.get("url");
|
||||
if (url instanceof String urlStr && urlStr.startsWith("data:video/")) {
|
||||
videoDataUrl = urlStr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (videoDataUrl != null) {
|
||||
// 替换为 video_url 格式
|
||||
newParts.add(Map.of(
|
||||
"type", "video_url",
|
||||
"video_url", Map.of("url", videoDataUrl)
|
||||
));
|
||||
} else {
|
||||
newParts.add(part);
|
||||
}
|
||||
}
|
||||
return new OpenAiApi.ChatCompletionMessage(
|
||||
newParts, msg.role(), msg.name(), msg.toolCallId(),
|
||||
msg.toolCalls(), msg.refusal(), msg.audioOutput(),
|
||||
msg.annotations(), msg.reasoningContent());
|
||||
}).toList();
|
||||
|
||||
return new OpenAiApi.ChatCompletionRequest(
|
||||
patched,
|
||||
request.model(), request.store(), request.metadata(),
|
||||
request.frequencyPenalty(), request.logitBias(),
|
||||
request.logprobs(), request.topLogprobs(),
|
||||
request.maxTokens(), request.maxCompletionTokens(),
|
||||
request.n(), request.outputModalities(), request.audioParameters(),
|
||||
request.presencePenalty(), request.responseFormat(),
|
||||
request.seed(), request.serviceTier(), request.stop(),
|
||||
request.stream(), request.streamOptions(),
|
||||
request.temperature(), request.topP(),
|
||||
request.tools(), request.toolChoice(), request.parallelToolCalls(),
|
||||
request.user(), request.reasoningEffort(),
|
||||
request.webSearchOptions(), request.verbosity(),
|
||||
request.promptCacheKey(), request.safetyIdentifier(),
|
||||
request.extraBody()
|
||||
);
|
||||
}
|
||||
|
||||
private void logOpenAiRequest(ModelProviderEntity provider, OpenAiApi.ChatCompletionRequest chatRequest) {
|
||||
try {
|
||||
log.info("OpenAI-compatible request: provider={}, body={}",
|
||||
|
||||
@ -225,42 +225,78 @@ public abstract class BaseAgent {
|
||||
};
|
||||
}
|
||||
|
||||
private static final long MAX_VIDEO_SIZE_BYTES = 20 * 1024 * 1024; // 20MB
|
||||
|
||||
/**
|
||||
* 构建 UserMessage，支持 multimodal：如果消息包含图片附件，直接注入 Spring AI Media 对象，
|
||||
* 让模型在 prompt 中直接看到图片，不需要再调 MCP read_media_file 工具。
|
||||
* 判断当前模型是否支持视频输入。
|
||||
* 仅已知支持视频分析的视觉模型（Qwen-VL、GPT-4o、Gemini 等）才注入视频 Media。
|
||||
*/
|
||||
private boolean modelSupportsVideo() {
|
||||
if (modelName == null) return false;
|
||||
String n = modelName.toLowerCase();
|
||||
return (n.contains("qwen") && n.contains("vl"))
|
||||
|| n.contains("gpt-4o")
|
||||
|| n.contains("gemini")
|
||||
|| (n.contains("glm") && n.contains("v"));
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 UserMessage，支持 multimodal：如果消息包含图片/视频附件，直接注入 Spring AI Media 对象，
|
||||
* 让模型在 prompt 中直接看到媒体内容，不需要再调 MCP read_media_file 工具。
|
||||
*/
|
||||
protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) {
|
||||
List<MessageContentPart> parts = conversationService.parseMessageParts(message);
|
||||
List<Media> mediaList = new ArrayList<>();
|
||||
boolean videoSupported = modelSupportsVideo();
|
||||
|
||||
for (MessageContentPart part : parts) {
|
||||
if (part == null || !"file".equals(part.getType())) {
|
||||
continue;
|
||||
}
|
||||
if (part == null) continue;
|
||||
String partType = part.getType();
|
||||
String contentType = part.getContentType();
|
||||
if (contentType == null || !contentType.startsWith("image/")) {
|
||||
continue;
|
||||
}
|
||||
if (contentType == null) continue;
|
||||
|
||||
boolean isImage = "file".equals(partType) && contentType.startsWith("image/");
|
||||
boolean isVideo = ("video".equals(partType) || "file".equals(partType)) && contentType.startsWith("video/");
|
||||
|
||||
if (!isImage && !isVideo) continue;
|
||||
|
||||
// SVG 是 XML 文本，不是光栅图片，LLM multimodal API 不支持
|
||||
if (contentType.contains("svg")) {
|
||||
if (isImage && contentType.contains("svg")) {
|
||||
log.debug("[{}] Skipping SVG attachment (not supported by multimodal API): {}",
|
||||
agentName, part.getFileName());
|
||||
continue;
|
||||
}
|
||||
// 解析图片文件路径：先尝试原始 path，再尝试拼接工作目录
|
||||
Path imagePath = resolveImagePath(part.getPath());
|
||||
if (imagePath == null) {
|
||||
log.warn("[{}] Image file not found for attachment: {}, path: {}",
|
||||
agentName, part.getFileName(), part.getPath());
|
||||
|
||||
// 视频仅在模型支持时注入，否则跳过（避免发送给非视觉模型导致 400 错误）
|
||||
if (isVideo && !videoSupported) {
|
||||
log.debug("[{}] Skipping video attachment (model '{}' does not support video): {}",
|
||||
agentName, modelName, part.getFileName());
|
||||
continue;
|
||||
}
|
||||
|
||||
// 视频文件大小保护
|
||||
if (isVideo && part.getFileSize() != null && part.getFileSize() > MAX_VIDEO_SIZE_BYTES) {
|
||||
log.warn("[{}] Skipping oversized video attachment ({}MB > 20MB): {}",
|
||||
agentName, part.getFileSize() / (1024 * 1024), part.getFileName());
|
||||
continue;
|
||||
}
|
||||
|
||||
// 解析媒体文件路径：先尝试原始 path，再尝试拼接工作目录
|
||||
Path mediaPath = resolveImagePath(part.getPath());
|
||||
if (mediaPath == null) {
|
||||
log.warn("[{}] {} file not found for attachment: {}, path: {}",
|
||||
agentName, isVideo ? "Video" : "Image", part.getFileName(), part.getPath());
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
MimeType mimeType = MimeType.valueOf(contentType);
|
||||
Media media = new Media(mimeType, new FileSystemResource(imagePath));
|
||||
Media media = new Media(mimeType, new FileSystemResource(mediaPath));
|
||||
mediaList.add(media);
|
||||
log.debug("[{}] Injected image into prompt: {} ({})", agentName, part.getFileName(), imagePath);
|
||||
log.debug("[{}] Injected {} into prompt: {} ({})",
|
||||
agentName, isVideo ? "video" : "image", part.getFileName(), mediaPath);
|
||||
} catch (Exception e) {
|
||||
log.warn("[{}] Failed to create Media for image {}: {}", agentName, part.getFileName(), e.getMessage());
|
||||
log.warn("[{}] Failed to create Media for {} {}: {}",
|
||||
agentName, isVideo ? "video" : "image", part.getFileName(), e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -867,6 +867,10 @@ public class ChatController {
|
||||
|
||||
Resource resource = new FileSystemResource(filePath);
|
||||
String contentType = Files.probeContentType(filePath);
|
||||
// probeContentType 在部分平台不识别视频格式，通过扩展名 fallback
|
||||
if (contentType == null) {
|
||||
contentType = guessContentTypeByExtension(filePath.getFileName().toString());
|
||||
}
|
||||
MediaType mediaType = MediaType.APPLICATION_OCTET_STREAM;
|
||||
if (contentType != null) {
|
||||
try {
|
||||
@ -1107,6 +1111,8 @@ public class ChatController {
|
||||
switch (part.getType()) {
|
||||
case "text", "thinking" -> appendPromptLine(builder, part.getText());
|
||||
case "file" -> appendPromptLine(builder, "附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")");
|
||||
case "image" -> appendPromptLine(builder, "图片附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")");
|
||||
case "video" -> appendPromptLine(builder, "视频附件: " + safe(part.getFileName()) + " (" + safe(part.getPath()) + ")");
|
||||
default -> appendPromptLine(builder, part.getText());
|
||||
}
|
||||
}
|
||||
@ -1127,6 +1133,19 @@ public class ChatController {
|
||||
return text == null ? "" : text;
|
||||
}
|
||||
|
||||
private static final java.util.Map<String, String> MEDIA_CONTENT_TYPES = java.util.Map.of(
|
||||
"mp4", "video/mp4", "webm", "video/webm", "mov", "video/quicktime",
|
||||
"avi", "video/x-msvideo", "mkv", "video/x-matroska", "mpeg", "video/mpeg",
|
||||
"mp3", "audio/mpeg", "wav", "audio/wav", "ogg", "audio/ogg"
|
||||
);
|
||||
|
||||
private static String guessContentTypeByExtension(String fileName) {
|
||||
if (fileName == null) return null;
|
||||
int dot = fileName.lastIndexOf('.');
|
||||
if (dot < 0 || dot == fileName.length() - 1) return null;
|
||||
return MEDIA_CONTENT_TYPES.get(fileName.substring(dot + 1).toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* 注册 SseEmitter 的完整生命周期回调
|
||||
*/
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
:class="{
|
||||
'attachment-chip--dir': attachment.contentType === 'inode/directory',
|
||||
'attachment-chip--image': attachment.contentType?.startsWith('image/'),
|
||||
'attachment-chip--video': attachment.contentType?.startsWith('video/'),
|
||||
}"
|
||||
>
|
||||
<!-- 图片缩略图预览（优先用本地 previewUrl，避免 JWT 认证问题） -->
|
||||
@ -27,6 +28,14 @@
|
||||
class="attachment-chip__thumbnail"
|
||||
loading="lazy"
|
||||
/>
|
||||
<!-- 视频缩略图预览 -->
|
||||
<video
|
||||
v-else-if="attachment.contentType?.startsWith('video/') && (attachment.previewUrl || attachment.url)"
|
||||
:src="attachment.previewUrl || attachment.url"
|
||||
class="attachment-chip__thumbnail"
|
||||
preload="metadata"
|
||||
muted
|
||||
/>
|
||||
<component
|
||||
:is="attachment.url ? 'a' : 'span'"
|
||||
:href="attachment.url || undefined"
|
||||
@ -416,7 +425,8 @@ defineExpose({
|
||||
padding: 6px 8px 6px 12px;
|
||||
}
|
||||
|
||||
.attachment-chip--image {
|
||||
.attachment-chip--image,
|
||||
.attachment-chip--video {
|
||||
padding: 4px 6px;
|
||||
}
|
||||
|
||||
|
||||
@ -221,6 +221,19 @@
|
||||
/>
|
||||
<span class="message-attachment-image__name">{{ attachment.name }}</span>
|
||||
</div>
|
||||
<div
|
||||
v-for="attachment in videoAttachments"
|
||||
:key="attachment.storedName"
|
||||
class="message-attachment-video"
|
||||
>
|
||||
<video
|
||||
:src="getDisplayUrl(attachment)"
|
||||
controls
|
||||
preload="metadata"
|
||||
playsinline
|
||||
/>
|
||||
<span class="message-attachment-video__name">{{ attachment.name }}</span>
|
||||
</div>
|
||||
<button
|
||||
v-for="attachment in fileAttachments"
|
||||
:key="attachment.storedName"
|
||||
@ -293,7 +306,7 @@ import type { ChatErrorInfo } from '@/types/chatError'
|
||||
|
||||
const { renderMarkdown } = useMarkdownRenderer()
|
||||
const { t } = useI18n()
|
||||
const { blobUrls, loadAllImages, downloadFile, openImage, getDisplayUrl, revokeAll } = useAuthenticatedAttachment()
|
||||
const { blobUrls, loadAllImages, loadAllVideos, downloadFile, openImage, getDisplayUrl, revokeAll } = useAuthenticatedAttachment()
|
||||
|
||||
interface Props {
|
||||
message: Message
|
||||
@ -462,12 +475,18 @@ onBeforeUnmount(() => {
|
||||
// --- 附件 ---
|
||||
const attachments = computed(() => props.message.attachments || [])
|
||||
const imageAttachments = computed(() => attachments.value.filter(a => a.contentType?.startsWith('image/')))
|
||||
const fileAttachments = computed(() => attachments.value.filter(a => !a.contentType?.startsWith('image/')))
|
||||
const videoAttachments = computed(() => attachments.value.filter(a => a.contentType?.startsWith('video/')))
|
||||
const fileAttachments = computed(() => attachments.value.filter(a =>
|
||||
!a.contentType?.startsWith('image/') && !a.contentType?.startsWith('video/')
|
||||
))
|
||||
|
||||
// 增量加载图片附件的鉴权 blob URL（watch 覆盖首次 + 后续变化）
|
||||
// 增量加载图片/视频附件的鉴权 blob URL（watch 覆盖首次 + 后续变化）
|
||||
watch(imageAttachments, (atts) => {
|
||||
if (atts.length > 0) loadAllImages(atts)
|
||||
}, { immediate: true })
|
||||
watch(videoAttachments, (atts) => {
|
||||
if (atts.length > 0) loadAllVideos(atts)
|
||||
}, { immediate: true })
|
||||
|
||||
// --- 时间 ---
|
||||
const formattedTime = computed(() => {
|
||||
@ -1325,6 +1344,27 @@ watch(isGenerating, (generating) => {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.message-attachment-video {
|
||||
border-radius: 12px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.message-attachment-video video {
|
||||
max-width: 400px;
|
||||
max-height: 280px;
|
||||
border-radius: 12px;
|
||||
}
|
||||
|
||||
.message-attachment-video__name {
|
||||
display: block;
|
||||
margin-top: 4px;
|
||||
font-size: 12px;
|
||||
opacity: 0.76;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.message-attachment {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
@ -58,6 +58,19 @@ export function useAuthenticatedAttachment() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量加载所有视频附件的 blob URL
|
||||
*/
|
||||
async function loadAllVideos(attachments: ChatAttachment[]) {
|
||||
const videoAtts = attachments.filter(a => a.contentType?.startsWith('video/'))
|
||||
for (const att of videoAtts) {
|
||||
const key = att.storedName || att.url
|
||||
if (!att.previewUrl && att.url && !blobUrls.value[key]) {
|
||||
await loadBlobUrl(att.url, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 鉴权下载文件：fetch blob → 创建临时 <a download> → 触发点击
|
||||
*/
|
||||
@ -126,6 +139,7 @@ export function useAuthenticatedAttachment() {
|
||||
blobUrls,
|
||||
loadBlobUrl,
|
||||
loadAllImages,
|
||||
loadAllVideos,
|
||||
downloadFile,
|
||||
openImage,
|
||||
getDisplayUrl,
|
||||
|
||||
@ -143,7 +143,7 @@ export interface MessageMetadata {
|
||||
}
|
||||
|
||||
export interface MessageContentPart {
|
||||
type: 'text' | 'thinking' | 'file' | 'tool_call'
|
||||
type: 'text' | 'thinking' | 'image' | 'file' | 'audio' | 'video' | 'tool_call'
|
||||
text?: string
|
||||
fileUrl?: string
|
||||
fileName?: string
|
||||
|
||||
@ -915,9 +915,10 @@ async function handleFileSelect(files: File[]) {
|
||||
for (const file of files) {
|
||||
const res: any = await chatApi.uploadFile(currentConversationId.value, file)
|
||||
const data = res.data || {}
|
||||
// 图片使用本地 ObjectURL 预览（避免 /api/v1/chat/files/ 需要 JWT 认证导致 <img> 加载失败）
|
||||
const isImage = (data.contentType || file.type || '').startsWith('image/')
|
||||
const previewUrl = isImage ? URL.createObjectURL(file) : data.url
|
||||
// 图片/视频使用本地 ObjectURL 预览（避免 /api/v1/chat/files/ 需要 JWT 认证导致加载失败）
|
||||
const ct = data.contentType || file.type || ''
|
||||
const isPreviewable = ct.startsWith('image/') || ct.startsWith('video/')
|
||||
const previewUrl = isPreviewable ? URL.createObjectURL(file) : data.url
|
||||
pendingAttachments.value.push({
|
||||
name: data.fileName || file.name,
|
||||
size: data.size || file.size,
|
||||
@ -959,8 +960,12 @@ function buildOutgoingParts(text: string, attachments: ChatAttachment[]): Messag
|
||||
const parts: MessageContentPart[] = []
|
||||
if (text) parts.push({ type: 'text', text })
|
||||
for (const attachment of attachments) {
|
||||
const ct = attachment.contentType || ''
|
||||
const partType: MessageContentPart['type'] = ct.startsWith('video/') ? 'video'
|
||||
: ct.startsWith('image/') ? 'image'
|
||||
: 'file'
|
||||
parts.push({
|
||||
type: 'file',
|
||||
type: partType,
|
||||
fileUrl: attachment.url,
|
||||
fileName: attachment.name,
|
||||
storedName: attachment.storedName,
|
||||
@ -1029,8 +1034,8 @@ function normalizeMessage(raw: Message): Message {
|
||||
// interrupted 是合法的历史状态（interrupt-with-followup），不映射为 stopped
|
||||
if (!msg.status) msg.status = 'completed'
|
||||
|
||||
// 从 file-type contentParts 恢复 attachments（历史消息 API 不返回单独的 attachments 字段）
|
||||
const fileParts = msg.contentParts.filter(p => p.type === 'file' && p.fileUrl)
|
||||
// 从 file/image/video contentParts 恢复 attachments（历史消息 API 不返回单独的 attachments 字段）
|
||||
const fileParts = msg.contentParts.filter(p => (p.type === 'file' || p.type === 'image' || p.type === 'video') && p.fileUrl)
|
||||
if (fileParts.length > 0 && (!msg.attachments || msg.attachments.length === 0)) {
|
||||
msg.attachments = fileParts.map(p => ({
|
||||
name: p.fileName || 'unknown',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user