mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(tool): add image_analyze for on-demand image re-analysis (#303)
This commit is contained in:
matevip
parent
aaf06b262c
commit
bad216d686
@ -124,6 +124,16 @@ public class MultimodalRouter {
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the configured default vision model, or {@code null} if none is set
|
||||
* or the referenced model is missing/disabled. Exposed so tools (e.g. an
|
||||
* on-demand image-analysis tool) can reuse the exact same model the automatic
|
||||
* sidecar uses, keeping behaviour consistent across the auto and tool paths.
|
||||
*/
|
||||
public ModelConfigEntity resolveVisionSidecar() {
|
||||
return resolveSidecar(Modality.VISION);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the configured sidecar model for a modality. Returns null only when:
|
||||
* - the setting is empty / blank;
|
||||
|
||||
@ -0,0 +1,150 @@
|
||||
package vip.mate.tool.builtin;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.model.ToolContext;
|
||||
import org.springframework.ai.tool.annotation.Tool;
|
||||
import org.springframework.ai.tool.annotation.ToolParam;
|
||||
import org.springframework.lang.Nullable;
|
||||
import org.springframework.stereotype.Component;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.routing.MediaCaptionService;
|
||||
import vip.mate.llm.routing.MultimodalRouter;
|
||||
import vip.mate.workspace.conversation.ConversationService;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* On-demand image analysis tool.
|
||||
*
|
||||
* <p>When the agent's primary model is text-only, images uploaded earlier in the
|
||||
* conversation are only captioned generically once (by the automatic vision
|
||||
* sidecar) and that caption is frozen into history. This tool lets the agent
|
||||
* re-examine a previously uploaded image against the user's actual follow-up
|
||||
* question — passing the question to the configured vision model so the answer is
|
||||
* tailored rather than a stale generic description.
|
||||
*
|
||||
* <p>It resolves the target image from the current conversation: an explicit
|
||||
* filename/path reference, or the most recent image when none is given. It reuses
|
||||
* the same vision model the automatic sidecar uses, so behaviour stays consistent
|
||||
* across the automatic and on-demand paths.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class ImageAnalyzeTool {
|
||||
|
||||
private final ConversationService conversationService;
|
||||
private final MultimodalRouter multimodalRouter;
|
||||
private final MediaCaptionService mediaCaptionService;
|
||||
|
||||
@Tool(description = "Analyze an image the user uploaded earlier in this conversation, answering a specific "
|
||||
+ "question about it. Use this whenever the user asks a follow-up about a previously sent image "
|
||||
+ "(e.g. 'what does the error in that screenshot say', 'read the total on the receipt') and the "
|
||||
+ "current model cannot see images natively. By default it analyzes the most recent image; pass "
|
||||
+ "'image' to target a specific one by filename. Returns the vision model's answer as text.")
|
||||
public String image_analyze(
|
||||
@ToolParam(description = "The specific question to answer about the image, in the user's own words. "
|
||||
+ "Be concrete — e.g. 'What is the error message?' rather than 'describe it'.") String question,
|
||||
@ToolParam(description = "Optional filename or path of the target image. Omit to use the most recent "
|
||||
+ "image in the conversation.", required = false) String image,
|
||||
@Nullable ToolContext ctx
|
||||
) {
|
||||
if (question == null || question.isBlank()) {
|
||||
return "请提供要针对图片回答的具体问题。";
|
||||
}
|
||||
|
||||
String conversationId = ToolExecutionContext.conversationId(ctx);
|
||||
if (conversationId == null || conversationId.isBlank()) {
|
||||
return "无法确定当前会话，无法定位已上传的图片。";
|
||||
}
|
||||
|
||||
MessageContentPart target = findImagePart(conversationId, image);
|
||||
if (target == null) {
|
||||
return image == null || image.isBlank()
|
||||
? "本次会话中没有找到可分析的图片，请确认用户已上传图片。"
|
||||
: "未找到名为「" + image + "」的图片，请检查文件名，或省略该参数以分析最近的图片。";
|
||||
}
|
||||
|
||||
ModelConfigEntity visionModel = multimodalRouter.resolveVisionSidecar();
|
||||
if (visionModel == null) {
|
||||
return "尚未配置视觉模型，无法分析图片。请在「设置 → 模型」中将一个具备视觉能力的模型设为默认视觉模型。";
|
||||
}
|
||||
|
||||
// Locale null → caption service answers in the question's own language.
|
||||
MediaCaptionService.CaptionResult result =
|
||||
mediaCaptionService.caption(visionModel, target, null, question);
|
||||
if (result.isFailure()) {
|
||||
log.warn("[image_analyze] caption failed for {} via {}/{}: {}",
|
||||
target.getFileName(), visionModel.getProvider(), visionModel.getModelName(),
|
||||
result.failure() == null ? "unknown" : result.failure().getMessage());
|
||||
return "视觉模型未能解析该图片（" + safeName(target) + "），请稍后重试或检查视觉模型配置。";
|
||||
}
|
||||
return result.description();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the target image part from the conversation. With a reference,
|
||||
* matches by filename basename / path / mediaId suffix, scanning newest-first.
|
||||
* Without one, returns the most recent image part.
|
||||
*/
|
||||
private MessageContentPart findImagePart(String conversationId, String reference) {
|
||||
List<MessageEntity> messages;
|
||||
try {
|
||||
messages = conversationService.listMessages(conversationId);
|
||||
} catch (Exception e) {
|
||||
log.warn("[image_analyze] failed to load messages for {}: {}", conversationId, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
if (messages == null || messages.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
String ref = reference == null ? null : reference.trim();
|
||||
boolean wildcard = ref == null || ref.isBlank()
|
||||
|| ref.equalsIgnoreCase("latest") || ref.equalsIgnoreCase("last");
|
||||
for (int i = messages.size() - 1; i >= 0; i--) {
|
||||
List<MessageContentPart> parts = conversationService.parseMessageParts(messages.get(i));
|
||||
for (int j = parts.size() - 1; j >= 0; j--) {
|
||||
MessageContentPart part = parts.get(j);
|
||||
if (!isImage(part)) continue;
|
||||
if (wildcard || matchesReference(part, ref)) {
|
||||
return part;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private boolean isImage(MessageContentPart part) {
|
||||
if (part == null) return false;
|
||||
String type = part.getType();
|
||||
String contentType = part.getContentType();
|
||||
boolean image = "image".equals(type)
|
||||
|| ("file".equals(type) && contentType != null && contentType.startsWith("image/"));
|
||||
return image && !(contentType != null && contentType.contains("svg"));
|
||||
}
|
||||
|
||||
private boolean matchesReference(MessageContentPart part, String ref) {
|
||||
if (ref == null) return false;
|
||||
String basename = ref.contains("/") || ref.contains("\\")
|
||||
? ref.substring(Math.max(ref.lastIndexOf('/'), ref.lastIndexOf('\\')) + 1)
|
||||
: ref;
|
||||
return endsWithIgnoreCase(part.getFileName(), basename)
|
||||
|| endsWithIgnoreCase(part.getPath(), ref)
|
||||
|| endsWithIgnoreCase(part.getMediaId(), ref);
|
||||
}
|
||||
|
||||
private boolean endsWithIgnoreCase(String value, String suffix) {
|
||||
if (value == null || suffix == null || suffix.isBlank()) return false;
|
||||
return value.toLowerCase().endsWith(suffix.toLowerCase());
|
||||
}
|
||||
|
||||
private String safeName(MessageContentPart part) {
|
||||
String name = part.getFileName();
|
||||
return name == null || name.isBlank() ? "image" : name;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,174 @@
|
||||
package vip.mate.tool.builtin;
|
||||
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.routing.MediaCaptionService;
|
||||
import vip.mate.llm.routing.MultimodalRouter;
|
||||
import vip.mate.workspace.conversation.ConversationService;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Issue #303: the on-demand {@code image_analyze} tool lets a text-only model
|
||||
* re-examine a previously uploaded image against a fresh question. These tests
|
||||
* pin image resolution (most-recent vs by-name) and the guard-rail messages.
|
||||
*/
|
||||
class ImageAnalyzeToolTest {
|
||||
|
||||
private ConversationService conv;
|
||||
private MultimodalRouter router;
|
||||
private MediaCaptionService caption;
|
||||
private ImageAnalyzeTool tool;
|
||||
|
||||
private static final String CONV = "conv-1";
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
conv = mock(ConversationService.class);
|
||||
router = mock(MultimodalRouter.class);
|
||||
caption = mock(MediaCaptionService.class);
|
||||
tool = new ImageAnalyzeTool(conv, router, caption);
|
||||
ToolExecutionContext.set(CONV, "tester");
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
ToolExecutionContext.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Blank question → guidance, no vision call")
|
||||
void blankQuestion() {
|
||||
String out = tool.image_analyze(" ", null, null);
|
||||
assertTrue(out.contains("具体问题"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("No conversation context → cannot locate image")
|
||||
void noConversation() {
|
||||
ToolExecutionContext.clear();
|
||||
String out = tool.image_analyze("报错是什么", null, null);
|
||||
assertTrue(out.contains("无法确定当前会话"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("No image in conversation → tells the user none was found")
|
||||
void noImage() {
|
||||
MessageEntity m = msg();
|
||||
when(conv.listMessages(CONV)).thenReturn(List.of(m));
|
||||
when(conv.parseMessageParts(m)).thenReturn(List.of(MessageContentPart.text("你好")));
|
||||
|
||||
String out = tool.image_analyze("报错是什么", null, null);
|
||||
assertTrue(out.contains("没有找到"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("No default vision model configured → asks user to configure one")
|
||||
void noVisionModel() {
|
||||
MessageEntity m = msg();
|
||||
when(conv.listMessages(CONV)).thenReturn(List.of(m));
|
||||
when(conv.parseMessageParts(m)).thenReturn(List.of(img("a.png", "media-a")));
|
||||
when(router.resolveVisionSidecar()).thenReturn(null);
|
||||
|
||||
String out = tool.image_analyze("报错是什么", null, null);
|
||||
assertTrue(out.contains("尚未配置视觉模型"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Most-recent image is analyzed with the question when no reference is given")
|
||||
void mostRecentImage_analyzed() {
|
||||
MessageEntity older = msg();
|
||||
MessageEntity newer = msg();
|
||||
when(conv.listMessages(CONV)).thenReturn(List.of(older, newer));
|
||||
when(conv.parseMessageParts(older)).thenReturn(List.of(img("old.png", "media-old")));
|
||||
when(conv.parseMessageParts(newer)).thenReturn(List.of(img("new.png", "media-new")));
|
||||
when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class));
|
||||
when(caption.caption(any(), any(), any(), any()))
|
||||
.thenReturn(MediaCaptionService.CaptionResult.success("空指针异常", 5L, false));
|
||||
|
||||
String out = tool.image_analyze("报错是什么", null, null);
|
||||
|
||||
assertEquals("空指针异常", out);
|
||||
ArgumentCaptor<MessageContentPart> part = ArgumentCaptor.forClass(MessageContentPart.class);
|
||||
ArgumentCaptor<String> q = ArgumentCaptor.forClass(String.class);
|
||||
verify(caption).caption(any(), part.capture(), any(), q.capture());
|
||||
assertEquals("new.png", part.getValue().getFileName(), "must pick the most recent image");
|
||||
assertEquals("报错是什么", q.getValue(), "the question must reach the vision model");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Explicit filename reference selects the matching image, not the most recent")
|
||||
void referenceByFilename_selectsMatch() {
|
||||
MessageEntity m = msg();
|
||||
when(conv.listMessages(CONV)).thenReturn(List.of(m));
|
||||
when(conv.parseMessageParts(m)).thenReturn(List.of(
|
||||
img("receipt.png", "media-r"), img("screenshot.png", "media-s")));
|
||||
when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class));
|
||||
when(caption.caption(any(), any(), any(), any()))
|
||||
.thenReturn(MediaCaptionService.CaptionResult.success("金额 99 元", 5L, false));
|
||||
|
||||
tool.image_analyze("总金额是多少", "receipt.png", null);
|
||||
|
||||
ArgumentCaptor<MessageContentPart> part = ArgumentCaptor.forClass(MessageContentPart.class);
|
||||
verify(caption).caption(any(), part.capture(), any(), eq("总金额是多少"));
|
||||
assertEquals("receipt.png", part.getValue().getFileName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Unknown filename reference → reports it was not found")
|
||||
void referenceNotFound() {
|
||||
MessageEntity m = msg();
|
||||
when(conv.listMessages(CONV)).thenReturn(List.of(m));
|
||||
when(conv.parseMessageParts(m)).thenReturn(List.of(img("a.png", "media-a")));
|
||||
|
||||
String out = tool.image_analyze("看看", "does-not-exist.png", null);
|
||||
assertTrue(out.contains("does-not-exist.png"));
|
||||
assertTrue(out.contains("未找到"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Vision call failure → friendly error naming the file")
|
||||
void captionFailure() {
|
||||
MessageEntity m = msg();
|
||||
when(conv.listMessages(CONV)).thenReturn(List.of(m));
|
||||
when(conv.parseMessageParts(m)).thenReturn(List.of(img("broken.png", "media-b")));
|
||||
when(router.resolveVisionSidecar()).thenReturn(mock(ModelConfigEntity.class));
|
||||
when(caption.caption(any(), any(), any(), any()))
|
||||
.thenReturn(MediaCaptionService.CaptionResult.failure(5L, new RuntimeException("timeout")));
|
||||
|
||||
String out = tool.image_analyze("看看", null, null);
|
||||
assertTrue(out.contains("未能解析"));
|
||||
assertTrue(out.contains("broken.png"));
|
||||
}
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
private static MessageEntity msg() {
|
||||
MessageEntity m = new MessageEntity();
|
||||
m.setRole("user");
|
||||
return m;
|
||||
}
|
||||
|
||||
private static MessageContentPart img(String fileName, String mediaId) {
|
||||
MessageContentPart p = new MessageContentPart();
|
||||
p.setType("image");
|
||||
p.setContentType("image/png");
|
||||
p.setFileName(fileName);
|
||||
p.setMediaId(mediaId);
|
||||
return p;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user