From 84a205509b5f41da6cba030999fcecf7f950a28c Mon Sep 17 00:00:00 2001 From: matevip Date: Sun, 5 Apr 2026 23:58:42 +0800 Subject: [PATCH] feat(chat): multimodal image injection and upload UX improvements --- .../ai/chat/messages/UserMessage.java | 138 ++++++++++++++++++ .../main/java/vip/mate/agent/BaseAgent.java | 103 ++++++++++++- .../agent/graph/StateGraphReActAgent.java | 3 +- .../plan/StateGraphPlanExecuteAgent.java | 2 +- .../vip/mate/channel/web/ChatController.java | 20 +++ .../conversation/ConversationService.java | 9 ++ mateclaw-ui/src/components/chat/ChatInput.vue | 25 +++- mateclaw-ui/src/types/index.ts | 2 + mateclaw-ui/src/views/ChatConsole.vue | 4 + 9 files changed, 302 insertions(+), 4 deletions(-) create mode 100644 mateclaw-server/org/springframework/ai/chat/messages/UserMessage.java diff --git a/mateclaw-server/org/springframework/ai/chat/messages/UserMessage.java b/mateclaw-server/org/springframework/ai/chat/messages/UserMessage.java new file mode 100644 index 00000000..a75d5f65 --- /dev/null +++ b/mateclaw-server/org/springframework/ai/chat/messages/UserMessage.java @@ -0,0 +1,138 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.chat.messages; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.ai.content.Media; +import org.springframework.ai.content.MediaContent; +import org.springframework.core.io.Resource; +import org.springframework.lang.NonNull; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; + +/** + * A message of the type 'user' passed as input Messages with the user role are from the + * end-user or developer. They represent questions, prompts, or any input that you want + * the generative to respond to. + */ +public class UserMessage extends AbstractMessage implements MediaContent { + + protected final List media; + + public UserMessage(String textContent) { + this(textContent, new ArrayList<>(), Map.of()); + } + + private UserMessage(String textContent, Collection media, Map metadata) { + super(MessageType.USER, textContent, metadata); + Assert.notNull(media, "media cannot be null"); + Assert.noNullElements(media, "media cannot have null elements"); + this.media = new ArrayList<>(media); + } + + public UserMessage(Resource resource) { + this(MessageUtils.readResource(resource)); + } + + @Override + public String toString() { + return "UserMessage{" + "content='" + getText() + '\'' + ", metadata=" + this.metadata + ", messageType=" + + this.messageType + '}'; + } + + @Override + @NonNull + public String getText() { + return this.textContent; + } + + @Override + public List getMedia() { + return this.media; + } + + public UserMessage copy() { + return new Builder().text(getText()).media(List.copyOf(getMedia())).metadata(Map.copyOf(getMetadata())).build(); + } + + public Builder mutate() { + return new Builder().text(getText()).media(List.copyOf(getMedia())).metadata(Map.copyOf(getMetadata())); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + @Nullable + private String textContent; + + @Nullable + private Resource resource; + + private List media = new ArrayList<>(); + + private Map metadata = new HashMap<>(); + + public Builder text(String textContent) { + this.textContent = textContent; + return this; + } + + public Builder text(Resource resource) { + this.resource = resource; + return this; + } + + public Builder media(List media) { + this.media = media; + return this; + } + + public Builder media(@Nullable Media... media) { + if (media != null) { + this.media = Arrays.asList(media); + } + return this; + } + + public Builder metadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public UserMessage build() { + if (StringUtils.hasText(this.textContent) && this.resource != null) { + throw new IllegalArgumentException("textContent and resource cannot be set at the same time"); + } + else if (this.resource != null) { + this.textContent = MessageUtils.readResource(this.resource); + } + return new UserMessage(this.textContent, this.media, this.metadata); + } + + } + +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java index b6536ed2..09c49f29 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java @@ -5,11 +5,18 @@ import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.content.Media; +import org.springframework.core.io.FileSystemResource; +import org.springframework.util.MimeType; import reactor.core.publisher.Flux; import vip.mate.approval.ApprovalPlaceholderUtil; import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; import vip.mate.workspace.conversation.model.MessageEntity; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -213,9 +220,103 @@ public abstract class BaseAgent { return switch (message.getRole()) { case "assistant" -> new AssistantMessage(renderedContent); case "system" -> new SystemMessage(renderedContent); - case "user" -> new UserMessage(renderedContent); + case "user" -> buildUserMessage(message, renderedContent); default -> null; }; } + /** + * 构建 UserMessage，支持 multimodal：如果消息包含图片附件，直接注入 Spring AI Media 对象， + * 让模型在 prompt 中直接看到图片，不需要再调 MCP read_media_file 工具。 + */ + protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { + List parts = conversationService.parseMessageParts(message); + List mediaList = new ArrayList<>(); + + for (MessageContentPart part : parts) { + if (part == null || !"file".equals(part.getType())) { + continue; + } + String contentType = part.getContentType(); + if (contentType == null || !contentType.startsWith("image/")) { + continue; + } + // 解析图片文件路径：先尝试原始 path，再尝试拼接工作目录 + Path imagePath = resolveImagePath(part.getPath()); + if (imagePath == null) { + log.warn("[{}] Image file not found for attachment: {}, path: {}", + agentName, part.getFileName(), part.getPath()); + continue; + } + try { + MimeType mimeType = MimeType.valueOf(contentType); + Media media = new Media(mimeType, new FileSystemResource(imagePath)); + mediaList.add(media); + log.debug("[{}] Injected image into prompt: {} ({})", agentName, part.getFileName(), imagePath); + } catch (Exception e) { + log.warn("[{}] Failed to create Media for image {}: {}", agentName, part.getFileName(), e.getMessage()); + } + } + + if (mediaList.isEmpty()) { + return new UserMessage(renderedContent); + } + return UserMessage.builder() + .text(renderedContent) + .media(mediaList) + .build(); + } + + /** + * 解析图片文件的绝对路径。 + *

+ * 上传文件存储在 data/chat-uploads/ 下，是相对于 Spring Boot 工作目录的路径。 + * MCP 工具的工作目录可能不同，所以这里直接解析为绝对路径。 + */ + /** + * 构建当前用户消息的 UserMessage（含 multimodal 图片注入）。 + *

+ * 从 DB 读取最新的 user 消息的 contentParts，提取图片附件并注入 Media。 + * 用于 StateGraphReActAgent.buildInitialState 等需要构建当前消息的场景。 + * + * @param conversationId 会话 ID + * @param userMessageText 用户消息文本 + * @return 带图片 Media 的 UserMessage（如果有图片附件），否则纯文本 UserMessage + */ + protected UserMessage buildCurrentUserMessage(String conversationId, String userMessageText) { + try { + List history = conversationService.listMessages(conversationId); + // 倒序找最新的 user 消息（内容匹配） + for (int i = history.size() - 1; i >= 0; i--) { + MessageEntity msg = history.get(i); + if ("user".equals(msg.getRole()) && userMessageText.equals(msg.getContent())) { + return buildUserMessage(msg, userMessageText); + } + } + } catch (Exception e) { + log.debug("[{}] Failed to load current user message parts for multimodal: {}", + agentName, e.getMessage()); + } + return new UserMessage(userMessageText); + } + + protected Path resolveImagePath(String relativePath) { + if (relativePath == null || relativePath.isBlank()) { + return null; + } + // 1. 如果已经是绝对路径且存在，直接用 + Path path = Paths.get(relativePath); + if (path.isAbsolute() && Files.exists(path)) { + return path; + } + // 2. 相对于 Spring Boot 工作目录解析 + Path resolved = Paths.get(System.getProperty("user.dir")).resolve(relativePath); + if (Files.exists(resolved)) { + return resolved; + } + // 3. 都找不到 + log.debug("[{}] Image path not found: tried {} and {}", agentName, path, resolved); + return null; + } + } diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java index 079dacd1..463d1f76 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java @@ -351,7 +351,8 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC } List messages = new ArrayList<>(historyMessages); - messages.add(new UserMessage(userMessage)); + // 构建当前用户消息：支持 multimodal（如果有图片附件，直接注入 Media） + messages.add(buildCurrentUserMessage(conversationId, userMessage)); Map inputs = new HashMap<>(); // 输入 diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java index af3dc67d..94965af7 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java @@ -255,7 +255,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS } List messages = new ArrayList<>(historyMessages); - messages.add(new UserMessage(userMessage)); + messages.add(buildCurrentUserMessage(conversationId, userMessage)); // 构建 working context：对历史消息做受控长度摘要 String workingContext = buildWorkingContext(historyMessages, List.of()); diff --git a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java index c0e9a34f..8d7d394b 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java @@ -922,6 +922,26 @@ public class ChatController { return; } + // Rate Limit 防护：如果上一轮以 rate limit 错误结束，不立即续跑排队消息（必然再次 429）。 + // 改为持久化用户消息 + 通知前端"稍后重试"，避免连锁 429 浪费配额。 + String lastMessage = conversationService.getLastMessage(conversationId); + if (lastMessage != null && (lastMessage.contains("频率过高") || lastMessage.contains("rate_limit") + || lastMessage.contains("429") || lastMessage.contains("速率限制"))) { + log.warn("Skipping queued message after rate limit error: conversationId={}, lastMessage={}", + conversationId, lastMessage.substring(0, Math.min(50, lastMessage.length()))); + // 持久化用户消息不丢失 + if (preConsumedInput.message() != null && !preConsumedInput.message().isBlank() + && !preConsumedInput.persisted()) { + conversationService.saveMessage(conversationId, "user", preConsumedInput.message()); + } + broadcastEvent(conversationId, "warning", Map.of( + "message", "上一轮请求触发了频率限制，排队消息已保存，请稍后重新发送")); + broadcastEvent(conversationId, "done", Map.of("status", "rate_limited")); + conversationService.updateStreamStatus(conversationId, "idle"); + completeEmitterQuietly(emitter, emitterDone); + return; + } + String queuedMessage = preConsumedInput.message(); Long agentId = preConsumedInput.agentId() != null ? preConsumedInput.agentId() : 1L; log.info("Starting queued message: conversationId={}, agentId={}, message={}", diff --git a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java index 89f1c72f..166d92d3 100644 --- a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java +++ b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java @@ -245,6 +245,15 @@ public class ConversationService { } } + /** + * 获取会话最后一条消息内容（用于 rate limit 防护等场景） + */ + public String getLastMessage(String conversationId) { + ConversationEntity conv = conversationMapper.selectOne(new LambdaQueryWrapper() + .eq(ConversationEntity::getConversationId, conversationId)); + return conv != null ? conv.getLastMessage() : null; + } + /** * 获取会话的消息数量 */ diff --git a/mateclaw-ui/src/components/chat/ChatInput.vue b/mateclaw-ui/src/components/chat/ChatInput.vue index abe1bc5f..6e6d7c8e 100644 --- a/mateclaw-ui/src/components/chat/ChatInput.vue +++ b/mateclaw-ui/src/components/chat/ChatInput.vue @@ -14,8 +14,19 @@ v-for="attachment in attachments" :key="attachment.storedName || attachment.path" class="attachment-chip" - :class="{ 'attachment-chip--dir': attachment.contentType === 'inode/directory' }" + :class="{ + 'attachment-chip--dir': attachment.contentType === 'inode/directory', + 'attachment-chip--image': attachment.contentType?.startsWith('image/'), + }" > + + 加载失败） + const isImage = (data.contentType || file.type || '').startsWith('image/') + const previewUrl = isImage ? URL.createObjectURL(file) : data.url pendingAttachments.value.push({ name: data.fileName || file.name, size: data.size || file.size, @@ -912,6 +915,7 @@ async function handleFileSelect(files: File[]) { storedName: data.storedName, path: data.path, contentType: data.contentType || file.type, + previewUrl, }) } } catch (e) {