opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 11:13:43 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(chat): multimodal image injection and upload UX improvements

Browse Source
This commit is contained in:
matevip 2026-04-05 23:58:42 +08:00
parent 99befe0d25
commit 84a205509b
9 changed files with 302 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
mateclaw-server/org/springframework/ai/chat/messages/UserMessage.java Normal file
View File

@ -0,0 +1,138 @@
/*
* Copyright 2023-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.ai.chat.messages;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.ai.content.Media;
import org.springframework.ai.content.MediaContent;
import org.springframework.core.io.Resource;
import org.springframework.lang.NonNull;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
/**
* A message of the type 'user' passed as input Messages with the user role are from the
* end-user or developer. They represent questions, prompts, or any input that you want
* the generative to respond to.
*/
public class UserMessage extends AbstractMessage implements MediaContent {
protected final List<Media> media;
public UserMessage(String textContent) {
this(textContent, new ArrayList<>(), Map.of());
}
private UserMessage(String textContent, Collection<Media> media, Map<String, Object> metadata) {
super(MessageType.USER, textContent, metadata);
Assert.notNull(media, "media cannot be null");
Assert.noNullElements(media, "media cannot have null elements");
this.media = new ArrayList<>(media);
}
public UserMessage(Resource resource) {
this(MessageUtils.readResource(resource));
}
@Override
public String toString() {
return "UserMessage{" + "content='" + getText() + '\'' + ", metadata=" + this.metadata + ", messageType="
+ this.messageType + '}';
}
@Override
@NonNull
public String getText() {
return this.textContent;
}
@Override
public List<Media> getMedia() {
return this.media;
}
public UserMessage copy() {
return new Builder().text(getText()).media(List.copyOf(getMedia())).metadata(Map.copyOf(getMetadata())).build();
}
public Builder mutate() {
return new Builder().text(getText()).media(List.copyOf(getMedia())).metadata(Map.copyOf(getMetadata()));
}
public static Builder builder() {
return new Builder();
}
public static final class Builder {
@Nullable
private String textContent;
@Nullable
private Resource resource;
private List<Media> media = new ArrayList<>();
private Map<String, Object> metadata = new HashMap<>();
public Builder text(String textContent) {
this.textContent = textContent;
return this;
}
public Builder text(Resource resource) {
this.resource = resource;
return this;
}
public Builder media(List<Media> media) {
this.media = media;
return this;
}
public Builder media(@Nullable Media... media) {
if (media != null) {
this.media = Arrays.asList(media);
}
return this;
}
public Builder metadata(Map<String, Object> metadata) {
this.metadata = metadata;
return this;
}
public UserMessage build() {
if (StringUtils.hasText(this.textContent) && this.resource != null) {
throw new IllegalArgumentException("textContent and resource cannot be set at the same time");
}
else if (this.resource != null) {
this.textContent = MessageUtils.readResource(this.resource);
}
return new UserMessage(this.textContent, this.media, this.metadata);
}
}
}

103
mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java
View File

@ -5,11 +5,18 @@ import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.content.Media;
import org.springframework.core.io.FileSystemResource;
import org.springframework.util.MimeType;
import reactor.core.publisher.Flux;
import vip.mate.approval.ApprovalPlaceholderUtil;
import vip.mate.workspace.conversation.ConversationService;
import vip.mate.workspace.conversation.model.MessageContentPart;
import vip.mate.workspace.conversation.model.MessageEntity;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
@ -213,9 +220,103 @@ public abstract class BaseAgent {
return switch (message.getRole()) {
case "assistant" -> new AssistantMessage(renderedContent);
case "system" -> new SystemMessage(renderedContent);
case "user" -> new UserMessage(renderedContent);
case "user" -> buildUserMessage(message, renderedContent);
default -> null;
};
}
/**
* 构建 UserMessage，支持 multimodal：如果消息包含图片附件，直接注入 Spring AI Media 对象，
* 让模型在 prompt 中直接看到图片，不需要再调 MCP read_media_file 工具。
*/
protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) {
List<MessageContentPart> parts = conversationService.parseMessageParts(message);
List<Media> mediaList = new ArrayList<>();
for (MessageContentPart part : parts) {
if (part == null || !"file".equals(part.getType())) {
continue;
}
String contentType = part.getContentType();
if (contentType == null || !contentType.startsWith("image/")) {
continue;
}
// 解析图片文件路径：先尝试原始 path，再尝试拼接工作目录
Path imagePath = resolveImagePath(part.getPath());
if (imagePath == null) {
log.warn("[{}] Image file not found for attachment: {}, path: {}",
agentName, part.getFileName(), part.getPath());
continue;
}
try {
MimeType mimeType = MimeType.valueOf(contentType);
Media media = new Media(mimeType, new FileSystemResource(imagePath));
mediaList.add(media);
log.debug("[{}] Injected image into prompt: {} ({})", agentName, part.getFileName(), imagePath);
} catch (Exception e) {
log.warn("[{}] Failed to create Media for image {}: {}", agentName, part.getFileName(), e.getMessage());
}
}
if (mediaList.isEmpty()) {
return new UserMessage(renderedContent);
}
return UserMessage.builder()
.text(renderedContent)
.media(mediaList)
.build();
}
/**
* 解析图片文件的绝对路径。
* <p>
* 上传文件存储在 data/chat-uploads/ 下，是相对于 Spring Boot 工作目录的路径。
* MCP 工具的工作目录可能不同，所以这里直接解析为绝对路径。
*/
/**
* 构建当前用户消息的 UserMessage（含 multimodal 图片注入）。
* <p>
* 从 DB 读取最新的 user 消息的 contentParts，提取图片附件并注入 Media。
* 用于 StateGraphReActAgent.buildInitialState 等需要构建当前消息的场景。
*
* @param conversationId 会话 ID
* @param userMessageText 用户消息文本
* @return 带图片 Media 的 UserMessage（如果有图片附件），否则纯文本 UserMessage
*/
protected UserMessage buildCurrentUserMessage(String conversationId, String userMessageText) {
try {
List<MessageEntity> history = conversationService.listMessages(conversationId);
// 倒序找最新的 user 消息（内容匹配）
for (int i = history.size() - 1; i >= 0; i--) {
MessageEntity msg = history.get(i);
if ("user".equals(msg.getRole()) && userMessageText.equals(msg.getContent())) {
return buildUserMessage(msg, userMessageText);
}
}
} catch (Exception e) {
log.debug("[{}] Failed to load current user message parts for multimodal: {}",
agentName, e.getMessage());
}
return new UserMessage(userMessageText);
}
protected Path resolveImagePath(String relativePath) {
if (relativePath == null || relativePath.isBlank()) {
return null;
}
// 1. 如果已经是绝对路径且存在，直接用
Path path = Paths.get(relativePath);
if (path.isAbsolute() && Files.exists(path)) {
return path;
}
// 2. 相对于 Spring Boot 工作目录解析
Path resolved = Paths.get(System.getProperty("user.dir")).resolve(relativePath);
if (Files.exists(resolved)) {
return resolved;
}
// 3. 都找不到
log.debug("[{}] Image path not found: tried {} and {}", agentName, path, resolved);
return null;
}
}

3
mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java
View File

@ -351,7 +351,8 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
}
List<Message> messages = new ArrayList<>(historyMessages);
messages.add(new UserMessage(userMessage));
// 构建当前用户消息：支持 multimodal（如果有图片附件，直接注入 Media）
messages.add(buildCurrentUserMessage(conversationId, userMessage));
Map<String, Object> inputs = new HashMap<>();
// 输入

2
mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java
View File

@ -255,7 +255,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS
}
List<Message> messages = new ArrayList<>(historyMessages);
messages.add(new UserMessage(userMessage));
messages.add(buildCurrentUserMessage(conversationId, userMessage));
// 构建 working context：对历史消息做受控长度摘要
String workingContext = buildWorkingContext(historyMessages, List.of());

20
mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java
View File

@ -922,6 +922,26 @@ public class ChatController {
return;
}
// Rate Limit 防护：如果上一轮以 rate limit 错误结束，不立即续跑排队消息（必然再次 429）。
// 改为持久化用户消息 + 通知前端"稍后重试"，避免连锁 429 浪费配额。
String lastMessage = conversationService.getLastMessage(conversationId);
if (lastMessage != null && (lastMessage.contains("频率过高") || lastMessage.contains("rate_limit")
|| lastMessage.contains("429") || lastMessage.contains("速率限制"))) {
log.warn("Skipping queued message after rate limit error: conversationId={}, lastMessage={}",
conversationId, lastMessage.substring(0, Math.min(50, lastMessage.length())));
// 持久化用户消息不丢失
if (preConsumedInput.message() != null && !preConsumedInput.message().isBlank()
&& !preConsumedInput.persisted()) {
conversationService.saveMessage(conversationId, "user", preConsumedInput.message());
}
broadcastEvent(conversationId, "warning", Map.of(
"message", "上一轮请求触发了频率限制，排队消息已保存，请稍后重新发送"));
broadcastEvent(conversationId, "done", Map.of("status", "rate_limited"));
conversationService.updateStreamStatus(conversationId, "idle");
completeEmitterQuietly(emitter, emitterDone);
return;
}
String queuedMessage = preConsumedInput.message();
Long agentId = preConsumedInput.agentId() != null ? preConsumedInput.agentId() : 1L;
log.info("Starting queued message: conversationId={}, agentId={}, message={}",

9
mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java
View File

@ -245,6 +245,15 @@ public class ConversationService {
}
}
/**
* 获取会话最后一条消息内容（用于 rate limit 防护等场景）
*/
public String getLastMessage(String conversationId) {
ConversationEntity conv = conversationMapper.selectOne(new LambdaQueryWrapper<ConversationEntity>()
.eq(ConversationEntity::getConversationId, conversationId));
return conv != null ? conv.getLastMessage() : null;
}
/**
* 获取会话的消息数量
*/

25
mateclaw-ui/src/components/chat/ChatInput.vue
View File

@ -14,8 +14,19 @@
v-for="attachment in attachments"
:key="attachment.storedName || attachment.path"
class="attachment-chip"
:class="{ 'attachment-chip--dir': attachment.contentType === 'inode/directory' }"
:class="{
'attachment-chip--dir': attachment.contentType === 'inode/directory',
'attachment-chip--image': attachment.contentType?.startsWith('image/'),
}"
>
<!-- 图片缩略图预览（优先用本地 previewUrl，避免 JWT 认证问题） -->
<img
v-if="attachment.contentType?.startsWith('image/') && (attachment.previewUrl || attachment.url)"
:src="attachment.previewUrl || attachment.url"
:alt="attachment.name"
class="attachment-chip__thumbnail"
loading="lazy"
/>
<component
:is="attachment.url ? 'a' : 'span'"
:href="attachment.url || undefined"
@ -405,6 +416,18 @@ defineExpose({
padding: 6px 8px 6px 12px;
}
.attachment-chip--image {
padding: 4px 6px;
}
.attachment-chip__thumbnail {
width: 36px;
height: 36px;
object-fit: cover;
border-radius: 4px;
flex-shrink: 0;
}
.attachment-chip__label {
display: inline-flex;
gap: 8px;

2
mateclaw-ui/src/types/index.ts
View File

@ -99,6 +99,8 @@ export interface ChatAttachment {
storedName: string
path: string
contentType?: string
/** 本地预览 URL（ObjectURL），图片附件用于避免 JWT 认证问题 */
previewUrl?: string
}
export interface ToolCallMeta {

4
mateclaw-ui/src/views/ChatConsole.vue
View File

@ -905,6 +905,9 @@ async function handleFileSelect(files: File[]) {
for (const file of files) {
const res: any = await chatApi.uploadFile(currentConversationId.value, file)
const data = res.data || {}
// 图片使用本地 ObjectURL 预览（避免 /api/v1/chat/files/ 需要 JWT 认证导致 <img> 加载失败）
const isImage = (data.contentType || file.type || '').startsWith('image/')
const previewUrl = isImage ? URL.createObjectURL(file) : data.url
pendingAttachments.value.push({
name: data.fileName || file.name,
size: data.size || file.size,
@ -912,6 +915,7 @@ async function handleFileSelect(files: File[]) {
storedName: data.storedName,
path: data.path,
contentType: data.contentType || file.type,
previewUrl,
})
}
} catch (e) {