From bad216d686928effabffda9af00ce117675ab7c9 Mon Sep 17 00:00:00 2001 From: matevip Date: Thu, 11 Jun 2026 13:53:56 +0800 Subject: [PATCH] feat(tool): add image_analyze for on-demand image re-analysis (#303) --- .../mate/llm/routing/MultimodalRouter.java | 10 + .../mate/tool/builtin/ImageAnalyzeTool.java | 150 +++++++++++++++ .../tool/builtin/ImageAnalyzeToolTest.java | 174 ++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java index b4ca2273..be41dda6 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java @@ -124,6 +124,16 @@ public class MultimodalRouter { .toList(); } + /** + * Resolve the configured default vision model, or {@code null} if none is set + * or the referenced model is missing/disabled. Exposed so tools (e.g. an + * on-demand image-analysis tool) can reuse the exact same model the automatic + * sidecar uses, keeping behaviour consistent across the auto and tool paths. + */ + public ModelConfigEntity resolveVisionSidecar() { + return resolveSidecar(Modality.VISION); + } + /** * Resolve the configured sidecar model for a modality. Returns null only when: * - the setting is empty / blank; diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java new file mode 100644 index 00000000..bedf8f02 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java @@ -0,0 +1,150 @@ +package vip.mate.tool.builtin; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Component; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.MediaCaptionService; +import vip.mate.llm.routing.MultimodalRouter; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.List; + +/** + * On-demand image analysis tool. + * + *

When the agent's primary model is text-only, images uploaded earlier in the + * conversation are only captioned generically once (by the automatic vision + * sidecar) and that caption is frozen into history. This tool lets the agent + * re-examine a previously uploaded image against the user's actual follow-up + * question — passing the question to the configured vision model so the answer is + * tailored rather than a stale generic description. + * + *

It resolves the target image from the current conversation: an explicit + * filename/path reference, or the most recent image when none is given. It reuses + * the same vision model the automatic sidecar uses, so behaviour stays consistent + * across the automatic and on-demand paths. + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ImageAnalyzeTool { + + private final ConversationService conversationService; + private final MultimodalRouter multimodalRouter; + private final MediaCaptionService mediaCaptionService; + + @Tool(description = "Analyze an image the user uploaded earlier in this conversation, answering a specific " + + "question about it. Use this whenever the user asks a follow-up about a previously sent image " + + "(e.g. 'what does the error in that screenshot say', 'read the total on the receipt') and the " + + "current model cannot see images natively. By default it analyzes the most recent image; pass " + + "'image' to target a specific one by filename. Returns the vision model's answer as text.") + public String image_analyze( + @ToolParam(description = "The specific question to answer about the image, in the user's own words. " + + "Be concrete — e.g. 'What is the error message?' rather than 'describe it'.") String question, + @ToolParam(description = "Optional filename or path of the target image. Omit to use the most recent " + + "image in the conversation.", required = false) String image, + @Nullable ToolContext ctx + ) { + if (question == null || question.isBlank()) { + return "请提供要针对图片回答的具体问题。"; + } + + String conversationId = ToolExecutionContext.conversationId(ctx); + if (conversationId == null || conversationId.isBlank()) { + return "无法确定当前会话，无法定位已上传的图片。"; + } + + MessageContentPart target = findImagePart(conversationId, image); + if (target == null) { + return image == null || image.isBlank() + ? "本次会话中没有找到可分析的图片，请确认用户已上传图片。" + : "未找到名为「" + image + "」的图片，请检查文件名，或省略该参数以分析最近的图片。"; + } + + ModelConfigEntity visionModel = multimodalRouter.resolveVisionSidecar(); + if (visionModel == null) { + return "尚未配置视觉模型，无法分析图片。请在「设置 → 模型」中将一个具备视觉能力的模型设为默认视觉模型。"; + } + + // Locale null → caption service answers in the question's own language. + MediaCaptionService.CaptionResult result = + mediaCaptionService.caption(visionModel, target, null, question); + if (result.isFailure()) { + log.warn("[image_analyze] caption failed for {} via {}/{}: {}", + target.getFileName(), visionModel.getProvider(), visionModel.getModelName(), + result.failure() == null ? "unknown" : result.failure().getMessage()); + return "视觉模型未能解析该图片（" + safeName(target) + "），请稍后重试或检查视觉模型配置。"; + } + return result.description(); + } + + /** + * Resolve the target image part from the conversation. With a reference, + * matches by filename basename / path / mediaId suffix, scanning newest-first. + * Without one, returns the most recent image part. + */ + private MessageContentPart findImagePart(String conversationId, String reference) { + List messages; + try { + messages = conversationService.listMessages(conversationId); + } catch (Exception e) { + log.warn("[image_analyze] failed to load messages for {}: {}", conversationId, e.getMessage()); + return null; + } + if (messages == null || messages.isEmpty()) { + return null; + } + String ref = reference == null ? null : reference.trim(); + boolean wildcard = ref == null || ref.isBlank() + || ref.equalsIgnoreCase("latest") || ref.equalsIgnoreCase("last"); + for (int i = messages.size() - 1; i >= 0; i--) { + List parts = conversationService.parseMessageParts(messages.get(i)); + for (int j = parts.size() - 1; j >= 0; j--) { + MessageContentPart part = parts.get(j); + if (!isImage(part)) continue; + if (wildcard || matchesReference(part, ref)) { + return part; + } + } + } + return null; + } + + private boolean isImage(MessageContentPart part) { + if (part == null) return false; + String type = part.getType(); + String contentType = part.getContentType(); + boolean image = "image".equals(type) + || ("file".equals(type) && contentType != null && contentType.startsWith("image/")); + return image && !(contentType != null && contentType.contains("svg")); + } + + private boolean matchesReference(MessageContentPart part, String ref) { + if (ref == null) return false; + String basename = ref.contains("/") || ref.contains("\\") + ? ref.substring(Math.max(ref.lastIndexOf('/'), ref.lastIndexOf('\\')) + 1) + : ref; + return endsWithIgnoreCase(part.getFileName(), basename) + || endsWithIgnoreCase(part.getPath(), ref) + || endsWithIgnoreCase(part.getMediaId(), ref); + } + + private boolean endsWithIgnoreCase(String value, String suffix) { + if (value == null || suffix == null || suffix.isBlank()) return false; + return value.toLowerCase().endsWith(suffix.toLowerCase()); + } + + private String safeName(MessageContentPart part) { + String name = part.getFileName(); + return name == null || name.isBlank() ? "image" : name; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java new file mode 100644 index 00000000..937ed041 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java @@ -0,0 +1,174 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.MediaCaptionService; +import vip.mate.llm.routing.MultimodalRouter; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Issue #303: the on-demand {@code image_analyze} tool lets a text-only model + * re-examine a previously uploaded image against a fresh question. These tests + * pin image resolution (most-recent vs by-name) and the guard-rail messages. + */ +class ImageAnalyzeToolTest { + + private ConversationService conv; + private MultimodalRouter router; + private MediaCaptionService caption; + private ImageAnalyzeTool tool; + + private static final String CONV = "conv-1"; + + @BeforeEach + void setUp() { + conv = mock(ConversationService.class); + router = mock(MultimodalRouter.class); + caption = mock(MediaCaptionService.class); + tool = new ImageAnalyzeTool(conv, router, caption); + ToolExecutionContext.set(CONV, "tester"); + } + + @AfterEach + void tearDown() { + ToolExecutionContext.clear(); + } + + @Test + @DisplayName("Blank question → guidance, no vision call") + void blankQuestion() { + String out = tool.image_analyze(" ", null, null); + assertTrue(out.contains("具体问题")); + } + + @Test + @DisplayName("No conversation context → cannot locate image") + void noConversation() { + ToolExecutionContext.clear(); + String out = tool.image_analyze("报错是什么", null, null); + assertTrue(out.contains("无法确定当前会话")); + } + + @Test + @DisplayName("No image in conversation → tells the user none was found") + void noImage() { + MessageEntity m = msg(); + when(conv.listMessages(CONV)).thenReturn(List.of(m)); + when(conv.parseMessageParts(m)).thenReturn(List.of(MessageContentPart.text("你好"))); + + String out = tool.image_analyze("报错是什么", null, null); + assertTrue(out.contains("没有找到")); + } + + @Test + @DisplayName("No default vision model configured → asks user to configure one") + void noVisionModel() { + MessageEntity m = msg(); + when(conv.listMessages(CONV)).thenReturn(List.of(m)); + when(conv.parseMessageParts(m)).thenReturn(List.of(img("a.png", "media-a"))); + when(router.resolveVisionSidecar()).thenReturn(null); + + String out = tool.image_analyze("报错是什么", null, null); + assertTrue(out.contains("尚未配置视觉模型")); + } + + @Test + @DisplayName("Most-recent image is analyzed with the question when no reference is given") + void mostRecentImage_analyzed() { + MessageEntity older = msg(); + MessageEntity newer = msg(); + when(conv.listMessages(CONV)).thenReturn(List.of(older, newer)); + when(conv.parseMessageParts(older)).thenReturn(List.of(img("old.png", "media-old"))); + when(conv.parseMessageParts(newer)).thenReturn(List.of(img("new.png", "media-new"))); + when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class)); + when(caption.caption(any(), any(), any(), any())) + .thenReturn(MediaCaptionService.CaptionResult.success("空指针异常", 5L, false)); + + String out = tool.image_analyze("报错是什么", null, null); + + assertEquals("空指针异常", out); + ArgumentCaptor part = ArgumentCaptor.forClass(MessageContentPart.class); + ArgumentCaptor q = ArgumentCaptor.forClass(String.class); + verify(caption).caption(any(), part.capture(), any(), q.capture()); + assertEquals("new.png", part.getValue().getFileName(), "must pick the most recent image"); + assertEquals("报错是什么", q.getValue(), "the question must reach the vision model"); + } + + @Test + @DisplayName("Explicit filename reference selects the matching image, not the most recent") + void referenceByFilename_selectsMatch() { + MessageEntity m = msg(); + when(conv.listMessages(CONV)).thenReturn(List.of(m)); + when(conv.parseMessageParts(m)).thenReturn(List.of( + img("receipt.png", "media-r"), img("screenshot.png", "media-s"))); + when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class)); + when(caption.caption(any(), any(), any(), any())) + .thenReturn(MediaCaptionService.CaptionResult.success("金额 99 元", 5L, false)); + + tool.image_analyze("总金额是多少", "receipt.png", null); + + ArgumentCaptor part = ArgumentCaptor.forClass(MessageContentPart.class); + verify(caption).caption(any(), part.capture(), any(), eq("总金额是多少")); + assertEquals("receipt.png", part.getValue().getFileName()); + } + + @Test + @DisplayName("Unknown filename reference → reports it was not found") + void referenceNotFound() { + MessageEntity m = msg(); + when(conv.listMessages(CONV)).thenReturn(List.of(m)); + when(conv.parseMessageParts(m)).thenReturn(List.of(img("a.png", "media-a"))); + + String out = tool.image_analyze("看看", "does-not-exist.png", null); + assertTrue(out.contains("does-not-exist.png")); + assertTrue(out.contains("未找到")); + } + + @Test + @DisplayName("Vision call failure → friendly error naming the file") + void captionFailure() { + MessageEntity m = msg(); + when(conv.listMessages(CONV)).thenReturn(List.of(m)); + when(conv.parseMessageParts(m)).thenReturn(List.of(img("broken.png", "media-b"))); + when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class)); + when(caption.caption(any(), any(), any(), any())) + .thenReturn(MediaCaptionService.CaptionResult.failure(5L, new RuntimeException("timeout"))); + + String out = tool.image_analyze("看看", null, null); + assertTrue(out.contains("未能解析")); + assertTrue(out.contains("broken.png")); + } + + // ---------- helpers ---------- + + private static MessageEntity msg() { + MessageEntity m = new MessageEntity(); + m.setRole("user"); + return m; + } + + private static MessageContentPart img(String fileName, String mediaId) { + MessageContentPart p = new MessageContentPart(); + p.setType("image"); + p.setContentType("image/png"); + p.setFileName(fileName); + p.setMediaId(mediaId); + return p; + } +}