diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java
index b4ca2273..be41dda6 100644
--- a/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java
+++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java
@@ -124,6 +124,16 @@ public class MultimodalRouter {
.toList();
}
+ /**
+ * Resolve the configured default vision model, or {@code null} if none is set
+ * or the referenced model is missing/disabled. Exposed so tools (e.g. an
+ * on-demand image-analysis tool) can reuse the exact same model the automatic
+ * sidecar uses, keeping behaviour consistent across the auto and tool paths.
+ */
+ public ModelConfigEntity resolveVisionSidecar() {
+ return resolveSidecar(Modality.VISION);
+ }
+
/**
* Resolve the configured sidecar model for a modality. Returns null only when:
* - the setting is empty / blank;
diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java
new file mode 100644
index 00000000..bedf8f02
--- /dev/null
+++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageAnalyzeTool.java
@@ -0,0 +1,150 @@
+package vip.mate.tool.builtin;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.chat.model.ToolContext;
+import org.springframework.ai.tool.annotation.Tool;
+import org.springframework.ai.tool.annotation.ToolParam;
+import org.springframework.lang.Nullable;
+import org.springframework.stereotype.Component;
+import vip.mate.llm.model.ModelConfigEntity;
+import vip.mate.llm.routing.MediaCaptionService;
+import vip.mate.llm.routing.MultimodalRouter;
+import vip.mate.workspace.conversation.ConversationService;
+import vip.mate.workspace.conversation.model.MessageContentPart;
+import vip.mate.workspace.conversation.model.MessageEntity;
+
+import java.util.List;
+
+/**
+ * On-demand image analysis tool.
+ *
+ *
When the agent's primary model is text-only, images uploaded earlier in the
+ * conversation are only captioned generically once (by the automatic vision
+ * sidecar) and that caption is frozen into history. This tool lets the agent
+ * re-examine a previously uploaded image against the user's actual follow-up
+ * question — passing the question to the configured vision model so the answer is
+ * tailored rather than a stale generic description.
+ *
+ *
It resolves the target image from the current conversation: an explicit
+ * filename/path reference, or the most recent image when none is given. It reuses
+ * the same vision model the automatic sidecar uses, so behaviour stays consistent
+ * across the automatic and on-demand paths.
+ *
+ * @author MateClaw Team
+ */
+@Slf4j
+@Component
+@RequiredArgsConstructor
+public class ImageAnalyzeTool {
+
+ private final ConversationService conversationService;
+ private final MultimodalRouter multimodalRouter;
+ private final MediaCaptionService mediaCaptionService;
+
+ @Tool(description = "Analyze an image the user uploaded earlier in this conversation, answering a specific "
+ + "question about it. Use this whenever the user asks a follow-up about a previously sent image "
+ + "(e.g. 'what does the error in that screenshot say', 'read the total on the receipt') and the "
+ + "current model cannot see images natively. By default it analyzes the most recent image; pass "
+ + "'image' to target a specific one by filename. Returns the vision model's answer as text.")
+ public String image_analyze(
+ @ToolParam(description = "The specific question to answer about the image, in the user's own words. "
+ + "Be concrete — e.g. 'What is the error message?' rather than 'describe it'.") String question,
+ @ToolParam(description = "Optional filename or path of the target image. Omit to use the most recent "
+ + "image in the conversation.", required = false) String image,
+ @Nullable ToolContext ctx
+ ) {
+ if (question == null || question.isBlank()) {
+ return "请提供要针对图片回答的具体问题。";
+ }
+
+ String conversationId = ToolExecutionContext.conversationId(ctx);
+ if (conversationId == null || conversationId.isBlank()) {
+ return "无法确定当前会话,无法定位已上传的图片。";
+ }
+
+ MessageContentPart target = findImagePart(conversationId, image);
+ if (target == null) {
+ return image == null || image.isBlank()
+ ? "本次会话中没有找到可分析的图片,请确认用户已上传图片。"
+ : "未找到名为「" + image + "」的图片,请检查文件名,或省略该参数以分析最近的图片。";
+ }
+
+ ModelConfigEntity visionModel = multimodalRouter.resolveVisionSidecar();
+ if (visionModel == null) {
+ return "尚未配置视觉模型,无法分析图片。请在「设置 → 模型」中将一个具备视觉能力的模型设为默认视觉模型。";
+ }
+
+ // Locale null → caption service answers in the question's own language.
+ MediaCaptionService.CaptionResult result =
+ mediaCaptionService.caption(visionModel, target, null, question);
+ if (result.isFailure()) {
+ log.warn("[image_analyze] caption failed for {} via {}/{}: {}",
+ target.getFileName(), visionModel.getProvider(), visionModel.getModelName(),
+ result.failure() == null ? "unknown" : result.failure().getMessage());
+ return "视觉模型未能解析该图片(" + safeName(target) + "),请稍后重试或检查视觉模型配置。";
+ }
+ return result.description();
+ }
+
+ /**
+ * Resolve the target image part from the conversation. With a reference,
+ * matches by filename basename / path / mediaId suffix, scanning newest-first.
+ * Without one, returns the most recent image part.
+ */
+ private MessageContentPart findImagePart(String conversationId, String reference) {
+ List messages;
+ try {
+ messages = conversationService.listMessages(conversationId);
+ } catch (Exception e) {
+ log.warn("[image_analyze] failed to load messages for {}: {}", conversationId, e.getMessage());
+ return null;
+ }
+ if (messages == null || messages.isEmpty()) {
+ return null;
+ }
+ String ref = reference == null ? null : reference.trim();
+ boolean wildcard = ref == null || ref.isBlank()
+ || ref.equalsIgnoreCase("latest") || ref.equalsIgnoreCase("last");
+ for (int i = messages.size() - 1; i >= 0; i--) {
+ List parts = conversationService.parseMessageParts(messages.get(i));
+ for (int j = parts.size() - 1; j >= 0; j--) {
+ MessageContentPart part = parts.get(j);
+ if (!isImage(part)) continue;
+ if (wildcard || matchesReference(part, ref)) {
+ return part;
+ }
+ }
+ }
+ return null;
+ }
+
+ private boolean isImage(MessageContentPart part) {
+ if (part == null) return false;
+ String type = part.getType();
+ String contentType = part.getContentType();
+ boolean image = "image".equals(type)
+ || ("file".equals(type) && contentType != null && contentType.startsWith("image/"));
+ return image && !(contentType != null && contentType.contains("svg"));
+ }
+
+ private boolean matchesReference(MessageContentPart part, String ref) {
+ if (ref == null) return false;
+ String basename = ref.contains("/") || ref.contains("\\")
+ ? ref.substring(Math.max(ref.lastIndexOf('/'), ref.lastIndexOf('\\')) + 1)
+ : ref;
+ return endsWithIgnoreCase(part.getFileName(), basename)
+ || endsWithIgnoreCase(part.getPath(), ref)
+ || endsWithIgnoreCase(part.getMediaId(), ref);
+ }
+
+ private boolean endsWithIgnoreCase(String value, String suffix) {
+ if (value == null || suffix == null || suffix.isBlank()) return false;
+ return value.toLowerCase().endsWith(suffix.toLowerCase());
+ }
+
+ private String safeName(MessageContentPart part) {
+ String name = part.getFileName();
+ return name == null || name.isBlank() ? "image" : name;
+ }
+}
diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java
new file mode 100644
index 00000000..937ed041
--- /dev/null
+++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ImageAnalyzeToolTest.java
@@ -0,0 +1,174 @@
+package vip.mate.tool.builtin;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import vip.mate.llm.model.ModelConfigEntity;
+import vip.mate.llm.routing.MediaCaptionService;
+import vip.mate.llm.routing.MultimodalRouter;
+import vip.mate.workspace.conversation.ConversationService;
+import vip.mate.workspace.conversation.model.MessageContentPart;
+import vip.mate.workspace.conversation.model.MessageEntity;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Issue #303: the on-demand {@code image_analyze} tool lets a text-only model
+ * re-examine a previously uploaded image against a fresh question. These tests
+ * pin image resolution (most-recent vs by-name) and the guard-rail messages.
+ */
+class ImageAnalyzeToolTest {
+
+ private ConversationService conv;
+ private MultimodalRouter router;
+ private MediaCaptionService caption;
+ private ImageAnalyzeTool tool;
+
+ private static final String CONV = "conv-1";
+
+ @BeforeEach
+ void setUp() {
+ conv = mock(ConversationService.class);
+ router = mock(MultimodalRouter.class);
+ caption = mock(MediaCaptionService.class);
+ tool = new ImageAnalyzeTool(conv, router, caption);
+ ToolExecutionContext.set(CONV, "tester");
+ }
+
+ @AfterEach
+ void tearDown() {
+ ToolExecutionContext.clear();
+ }
+
+ @Test
+ @DisplayName("Blank question → guidance, no vision call")
+ void blankQuestion() {
+ String out = tool.image_analyze(" ", null, null);
+ assertTrue(out.contains("具体问题"));
+ }
+
+ @Test
+ @DisplayName("No conversation context → cannot locate image")
+ void noConversation() {
+ ToolExecutionContext.clear();
+ String out = tool.image_analyze("报错是什么", null, null);
+ assertTrue(out.contains("无法确定当前会话"));
+ }
+
+ @Test
+ @DisplayName("No image in conversation → tells the user none was found")
+ void noImage() {
+ MessageEntity m = msg();
+ when(conv.listMessages(CONV)).thenReturn(List.of(m));
+ when(conv.parseMessageParts(m)).thenReturn(List.of(MessageContentPart.text("你好")));
+
+ String out = tool.image_analyze("报错是什么", null, null);
+ assertTrue(out.contains("没有找到"));
+ }
+
+ @Test
+ @DisplayName("No default vision model configured → asks user to configure one")
+ void noVisionModel() {
+ MessageEntity m = msg();
+ when(conv.listMessages(CONV)).thenReturn(List.of(m));
+ when(conv.parseMessageParts(m)).thenReturn(List.of(img("a.png", "media-a")));
+ when(router.resolveVisionSidecar()).thenReturn(null);
+
+ String out = tool.image_analyze("报错是什么", null, null);
+ assertTrue(out.contains("尚未配置视觉模型"));
+ }
+
+ @Test
+ @DisplayName("Most-recent image is analyzed with the question when no reference is given")
+ void mostRecentImage_analyzed() {
+ MessageEntity older = msg();
+ MessageEntity newer = msg();
+ when(conv.listMessages(CONV)).thenReturn(List.of(older, newer));
+ when(conv.parseMessageParts(older)).thenReturn(List.of(img("old.png", "media-old")));
+ when(conv.parseMessageParts(newer)).thenReturn(List.of(img("new.png", "media-new")));
+ when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class));
+ when(caption.caption(any(), any(), any(), any()))
+ .thenReturn(MediaCaptionService.CaptionResult.success("空指针异常", 5L, false));
+
+ String out = tool.image_analyze("报错是什么", null, null);
+
+ assertEquals("空指针异常", out);
+ ArgumentCaptor part = ArgumentCaptor.forClass(MessageContentPart.class);
+ ArgumentCaptor q = ArgumentCaptor.forClass(String.class);
+ verify(caption).caption(any(), part.capture(), any(), q.capture());
+ assertEquals("new.png", part.getValue().getFileName(), "must pick the most recent image");
+ assertEquals("报错是什么", q.getValue(), "the question must reach the vision model");
+ }
+
+ @Test
+ @DisplayName("Explicit filename reference selects the matching image, not the most recent")
+ void referenceByFilename_selectsMatch() {
+ MessageEntity m = msg();
+ when(conv.listMessages(CONV)).thenReturn(List.of(m));
+ when(conv.parseMessageParts(m)).thenReturn(List.of(
+ img("receipt.png", "media-r"), img("screenshot.png", "media-s")));
+ when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class));
+ when(caption.caption(any(), any(), any(), any()))
+ .thenReturn(MediaCaptionService.CaptionResult.success("金额 99 元", 5L, false));
+
+ tool.image_analyze("总金额是多少", "receipt.png", null);
+
+ ArgumentCaptor part = ArgumentCaptor.forClass(MessageContentPart.class);
+ verify(caption).caption(any(), part.capture(), any(), eq("总金额是多少"));
+ assertEquals("receipt.png", part.getValue().getFileName());
+ }
+
+ @Test
+ @DisplayName("Unknown filename reference → reports it was not found")
+ void referenceNotFound() {
+ MessageEntity m = msg();
+ when(conv.listMessages(CONV)).thenReturn(List.of(m));
+ when(conv.parseMessageParts(m)).thenReturn(List.of(img("a.png", "media-a")));
+
+ String out = tool.image_analyze("看看", "does-not-exist.png", null);
+ assertTrue(out.contains("does-not-exist.png"));
+ assertTrue(out.contains("未找到"));
+ }
+
+ @Test
+ @DisplayName("Vision call failure → friendly error naming the file")
+ void captionFailure() {
+ MessageEntity m = msg();
+ when(conv.listMessages(CONV)).thenReturn(List.of(m));
+ when(conv.parseMessageParts(m)).thenReturn(List.of(img("broken.png", "media-b")));
+ when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class));
+ when(caption.caption(any(), any(), any(), any()))
+ .thenReturn(MediaCaptionService.CaptionResult.failure(5L, new RuntimeException("timeout")));
+
+ String out = tool.image_analyze("看看", null, null);
+ assertTrue(out.contains("未能解析"));
+ assertTrue(out.contains("broken.png"));
+ }
+
+ // ---------- helpers ----------
+
+ private static MessageEntity msg() {
+ MessageEntity m = new MessageEntity();
+ m.setRole("user");
+ return m;
+ }
+
+ private static MessageContentPart img(String fileName, String mediaId) {
+ MessageContentPart p = new MessageContentPart();
+ p.setType("image");
+ p.setContentType("image/png");
+ p.setFileName(fileName);
+ p.setMediaId(mediaId);
+ return p;
+ }
+}