diff --git a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java index 801de3a0..344b8640 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java @@ -955,6 +955,11 @@ public abstract class BaseAgent { if (decision.strategy() == MultimodalRoutingDecision.Strategy.SIDECAR && mediaCaptionService != null && decision.sidecarModel() != null) { + // The user's actual question (text parts only, excluding media markers) + // so the vision model tailors its description to what was asked rather + // than emitting a generic caption. + String userQuestion = extractUserQuestion(parts); + boolean captionPersisted = false; for (MessageContentPart part : parts) { if (part == null) continue; String contentType = part.getContentType(); @@ -963,13 +968,23 @@ public abstract class BaseAgent { && !contentType.contains("svg"); if (!isImage) continue; MediaCaptionService.CaptionResult result = mediaCaptionService.caption( - decision.sidecarModel(), part, userLocale); + decision.sidecarModel(), part, userLocale, userQuestion); if (result.isFailure()) { log.warn("[{}] Sidecar caption failed for {}: {}", agentName, part.getFileName(), result.failure().getMessage()); - textBuilder.append("\n\n[系统提示] 视觉模型未能解析附件 ") - .append(part.getFileName()) - .append("，请稍后重试或在「设置 → 模型」检查视觉模型配置。"); + if (isRemoteOnlyAttachment(part)) { + // The image was never downloaded locally (only a remote + // channel URL survives) — for WeCom/aibot that URL points + // at short-lived AES-encrypted bytes, so captioning can + // never succeed until media download is enabled. + textBuilder.append("\n\n[系统提示] 图片 ") + .append(part.getFileName()) + .append(" 未下载到本地，无法识别；请在「设置 → 渠道」开启该渠道的媒体下载。"); + } else { + textBuilder.append("\n\n[系统提示] 视觉模型未能解析附件 ") + .append(part.getFileName()) + .append("，请稍后重试或在「设置 → 模型」检查视觉模型配置。"); + } continue; } textBuilder.append("\n\n[图片附件描述: ") @@ -977,9 +992,17 @@ public abstract class BaseAgent { .append("]\n") .append(result.description()) .append("\n[/图片附件描述]"); + // Persist the caption onto the part so later turns retain the image + // content: history user messages replay as text only, and without a + // stored caption every follow-up question loses the attachment. + part.setCaption(result.description()); + captionPersisted = true; String identifier = identifyPart(part); if (identifier != null) sidecarHandledIdentifiers.add(identifier); } + if (captionPersisted && conversationService != null) { + conversationService.updateMessageParts(message, parts); + } } List mediaList = new ArrayList<>(); @@ -1048,7 +1071,7 @@ public abstract class BaseAgent { if (mediaPath == null) { log.warn("[{}] {} file not found for attachment: {}, path: {}, mediaId: {}", agentName, isVideo ? "Video" : "Image", part.getFileName(), part.getPath(), part.getMediaId()); - skippedAttachments.add(part.getFileName() + "(文件未找到)"); + skippedAttachments.add(part.getFileName() + unresolvedAttachmentReason(part)); continue; } try { @@ -1083,6 +1106,52 @@ public abstract class BaseAgent { return new CurrentTurnUserMessage(built, decision); } + /** + * Concatenate the text parts of a message into the user's question, dropping + * image/file/media parts. Returns {@code null} when there is no usable text + * (e.g. an image-only IM message), which makes the caption fall back to the + * generic full-description prompt. + */ + private static String extractUserQuestion(List parts) { + if (parts == null || parts.isEmpty()) return null; + StringBuilder sb = new StringBuilder(); + for (MessageContentPart part : parts) { + if (part == null || !"text".equals(part.getType())) continue; + String text = part.getText(); + if (text == null || text.isBlank()) continue; + if (sb.length() > 0) sb.append('\n'); + sb.append(text.trim()); + } + String question = sb.toString().trim(); + return question.isEmpty() ? null : question; + } + + /** + * True when an attachment has no resolvable local file and its only locator + * is a remote http(s) URL — i.e. the IM channel never downloaded it locally. + * For WeCom/aibot images that URL points at short-lived AES-encrypted bytes, + * so it is unusable as-is. Lets callers turn a generic "file not found" into + * an actionable hint instead of a dead end. + */ + private static boolean isRemoteOnlyAttachment(MessageContentPart part) { + if (part == null) return false; + if (part.getPath() != null && !part.getPath().isBlank()) return false; + String locator = part.getMediaId(); + if (locator == null || locator.isBlank()) locator = part.getFileUrl(); + return locator != null && (locator.startsWith("http://") || locator.startsWith("https://")); + } + + /** + * Reason string appended to a skipped attachment whose local file could not + * be resolved — distinguishes "never downloaded" (channel media download + * off) from a genuine missing-file so the user gets an actionable message. + */ + private static String unresolvedAttachmentReason(MessageContentPart part) { + return isRemoteOnlyAttachment(part) + ? "(图片未下载到本地，无法识别；请在「设置 → 渠道」开启该渠道的媒体下载)" + : "(文件未找到)"; + } + /** * Stable identifier for de-duplicating parts already handled by the sidecar * pass. Falls back across {@code path → mediaId → fileName} since not every @@ -1194,7 +1263,14 @@ public abstract class BaseAgent { if ("user".equals(msg.getRole())) { // 用 DB 中的实际内容（可能包含 contentParts），不用传入的 text String content = conversationService.renderMessageContent(msg); - return buildUserMessageForCurrentTurn(msg, content != null && !content.isBlank() ? content : userMessageText); + CurrentTurnUserMessage built = buildUserMessageForCurrentTurn( + msg, content != null && !content.isBlank() ? content : userMessageText); + // Vision-capable models replay history as text only, so a + // follow-up question about an earlier image would otherwise be + // answered blind. Re-attach the most recent image to this turn + // so the model actually re-sees it. (Text-only models instead + // rely on the persisted sidecar caption — see buildUserMessageInternal.) + return maybeCarryRecentImage(history, i, msg, built); } } } catch (Exception e) { @@ -1204,6 +1280,85 @@ public abstract class BaseAgent { return new CurrentTurnUserMessage(new UserMessage(userMessageText), null); } + /** How far back (in messages) to look for an image to carry into a follow-up turn. */ + private static final int CARRY_IMAGE_LOOKBACK = 8; + + /** + * For a vision-capable model, re-attach the most recent image from a recent + * earlier turn to the current user message when the current turn carries no + * image of its own. History is replayed as text only (see {@link #toSpringMessage}), + * so without this a follow-up like "what's in the top-left of that photo?" is + * answered blind. Bounded to a single image within {@link #CARRY_IMAGE_LOOKBACK} + * messages so a long conversation doesn't re-send pixels on every turn. + * + *

No-op (returns {@code built} unchanged) when: the model can't see images, + * the current turn already has an image, no recent image exists, or the recent + * image has no resolvable local file (e.g. an undownloaded channel URL). + */ + private CurrentTurnUserMessage maybeCarryRecentImage(List history, int currentIdx, + MessageEntity currentMsg, CurrentTurnUserMessage built) { + try { + if (built == null || !modelSupportsVision()) return built; + if (messageHasImagePart(currentMsg)) return built; // current turn already carries an image + + int from = Math.max(0, currentIdx - CARRY_IMAGE_LOOKBACK); + for (int j = currentIdx - 1; j >= from; j--) { + MessageEntity m = history.get(j); + if (m == null || !"user".equals(m.getRole())) continue; + List parts = conversationService.parseMessageParts(m); + for (int k = parts.size() - 1; k >= 0; k--) { + MessageContentPart part = parts.get(k); + if (!isResolvableImagePart(part)) continue; + Path imgPath = resolveImagePath(part.getPath()); + if (imgPath == null && part.getMediaId() != null) imgPath = resolveImagePath(part.getMediaId()); + if (imgPath == null) continue; + String contentType = part.getContentType(); + if (contentType == null || "image/*".equals(contentType)) contentType = "image/jpeg"; + try { + Media carried = new Media(MimeType.valueOf(contentType), new FileSystemResource(imgPath)); + UserMessage orig = built.userMessage(); + String name = part.getFileName() == null ? "image" : part.getFileName(); + String text = (orig.getText() == null ? "" : orig.getText()) + + "\n\n[系统提示] 以下图片是用户本次对话中较早发送的「" + name + + "」，当前问题很可能与它相关。请直接查看该图片作答，不要凭记忆猜测。"; + List media = new ArrayList<>(); + if (orig.getMedia() != null) media.addAll(orig.getMedia()); + media.add(carried); + log.debug("[{}] Carried recent image {} into follow-up turn for vision model", + agentName, name); + return new CurrentTurnUserMessage( + UserMessage.builder().text(text).media(media).build(), + built.routingDecision()); + } catch (Exception e) { + log.debug("[{}] Failed to carry recent image {}: {}", + agentName, part.getFileName(), e.getMessage()); + return built; + } + } + } + } catch (Exception e) { + log.debug("[{}] maybeCarryRecentImage failed: {}", agentName, e.getMessage()); + } + return built; + } + + private boolean messageHasImagePart(MessageEntity message) { + for (MessageContentPart part : conversationService.parseMessageParts(message)) { + if (isResolvableImagePart(part)) return true; + } + return false; + } + + /** An image part (not SVG) — the raster kind a multimodal API can ingest. */ + private static boolean isResolvableImagePart(MessageContentPart part) { + if (part == null) return false; + String type = part.getType(); + String contentType = part.getContentType(); + boolean isImage = ("image".equals(type) || "file".equals(type)) + && contentType != null && contentType.startsWith("image/"); + return isImage && !contentType.contains("svg"); + } + protected Path resolveImagePath(String relativePath) { if (relativePath == null || relativePath.isBlank()) { return null; diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java index 6521009d..d675d163 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java @@ -42,6 +42,18 @@ public class MediaCaptionService { private final RetryTemplate retryTemplate; public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale) { + return caption(visionModel, imagePart, locale, null); + } + + /** + * Caption an image, optionally tailored to a user question. When + * {@code userQuestion} is non-blank the vision model is asked to answer it + * directly (in addition to describing the image), so multi-turn follow-ups + * get an answer rather than a generic description. When blank, falls back to + * the factual full-description prompt. + */ + public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale, + String userQuestion) { if (visionModel == null || imagePart == null) { return CaptionResult.failure(0, new IllegalArgumentException("vision model or image part is null")); } @@ -59,7 +71,7 @@ public class MediaCaptionService { ChatModel chatModel = chatModelFactory.buildFor(visionModel, retryTemplate); ChatClient client = ChatClient.create(chatModel); UserMessage userMessage = UserMessage.builder() - .text(buildPrompt(locale, imagePart.getFileName())) + .text(buildPrompt(locale, imagePart.getFileName(), userQuestion)) .media(List.of(new Media(MimeType.valueOf(contentType), new FileSystemResource(mediaPath)))) .build(); String description = client.prompt() @@ -87,9 +99,27 @@ public class MediaCaptionService { * (matches the primary user base) but switches to English so vision-model output * matches the chat language and avoids polluting English-only contexts. */ - private String buildPrompt(Locale locale, String fileName) { + private String buildPrompt(Locale locale, String fileName, String userQuestion) { boolean english = locale != null && Locale.ENGLISH.getLanguage().equalsIgnoreCase(locale.getLanguage()); String fileHint = (fileName == null || fileName.isBlank()) ? "" : " (" + fileName + ")"; + boolean hasQuestion = userQuestion != null && !userQuestion.isBlank(); + + if (hasQuestion) { + String question = userQuestion.trim(); + // Question-aware: extract everything relevant to the user's ask, then + // answer it. Answering in the question's own language keeps the caption + // consistent with the chat regardless of the configured locale. + if (english) { + return "Look at this image" + fileHint + " and answer the user's question. " + + "First note any details relevant to the question (objects, scene, visible text/OCR, " + + "numbers, layout), then answer directly. Reply in the same language as the question. " + + "Question: " + question; + } + return "请仔细查看这张图片" + fileHint + "，并回答用户的问题。" + + "先指出与问题相关的细节（物体、场景、画面文字/OCR、数字、排版等），再直接作答。" + + "用与问题相同的语言回复。问题：" + question; + } + if (english) { return "Describe this image" + fileHint + " concisely: list the main objects, scene, any visible text (OCR), " diff --git a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java index 1c71606e..984e31dc 100644 --- a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java +++ b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java @@ -1033,10 +1033,38 @@ public class ConversationService { name = "未命名"; } String path = safe(part.getPath()); - if (path.isBlank()) { - return label + " " + name; + StringBuilder rendered = new StringBuilder(label).append(' ').append(name); + if (!path.isBlank()) { + rendered.append("（路径: ").append(path).append("）"); + } + // A persisted caption (vision sidecar output) carries the image content + // into later turns: history user messages replay as text only, so without + // this the model would lose all knowledge of the attachment after turn 1. + String caption = safe(part.getCaption()); + if (!caption.isBlank()) { + rendered.append("\n[图片内容] ").append(caption.trim()); + } + return rendered.toString(); + } + + /** + * Overwrite a message's {@code content_parts} with an updated list — used by + * the vision sidecar to persist generated captions back onto image parts so + * later turns retain the image description (history replay is text-only). + * Best-effort: a serialization or DB failure is logged, not propagated, so + * the in-flight chat turn is never broken by a caption write. + */ + public void updateMessageParts(MessageEntity message, List parts) { + if (message == null || message.getId() == null || parts == null || parts.isEmpty()) { + return; + } + try { + message.setContentParts(serializeParts(parts)); + messageMapper.updateById(message); + } catch (Exception e) { + log.warn("Failed to persist updated content_parts for message {}: {}", + message.getId(), e.getMessage()); } - return label + " " + name + "（路径: " + path + "）"; } private void appendSegment(StringBuilder builder, String text) { diff --git a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/model/MessageContentPart.java b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/model/MessageContentPart.java index 181da4b4..53958928 100644 --- a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/model/MessageContentPart.java +++ b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/model/MessageContentPart.java @@ -49,6 +49,15 @@ public class MessageContentPart { */ private String mediaId; + /** + * Vision-model description of an image/video part, produced by the sidecar + * captioning path when the primary model is text-only. Persisted so the + * description survives into later turns: history replay sends user messages + * as text, and without a stored caption the image content would be lost on + * every follow-up question. Null for non-media parts or when no captioning ran. + */ + private String caption; + // ==================== 工厂方法 ==================== public static MessageContentPart text(String text) { diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentCarryRecentImageTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentCarryRecentImageTest.java new file mode 100644 index 00000000..6f6936ba --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentCarryRecentImageTest.java @@ -0,0 +1,176 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.UserMessage; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.EnumSet; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Issue #303 follow-up: a vision-capable model replays history as text only, so a + * follow-up question about an earlier image was answered blind. The current turn + * must re-attach the most recent image so the model actually re-sees it. + */ +class BaseAgentCarryRecentImageTest { + + @Test + @DisplayName("Vision model + follow-up with no image → most recent image is carried into the turn") + void followUp_carriesRecentImage() throws Exception { + Path img = Files.createTempFile("carry-test", ".jpg"); + Files.write(img, new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF, 0x00}); + try { + TestAgent agent = visionAgent(); + + MessageEntity imgTurn = userMsg("看看这张图"); + MessageContentPart imagePart = imagePart(img.toAbsolutePath().toString()); + MessageEntity asst = assistantMsg("图里是手写的字"); + MessageEntity followUp = userMsg("左上角有没有小字"); + + List history = List.of(imgTurn, asst, followUp); + when(agent.conversationService.listMessages("c1")).thenReturn(history); + when(agent.conversationService.renderMessageContent(followUp)).thenReturn("左上角有没有小字"); + when(agent.conversationService.parseMessageParts(imgTurn)).thenReturn(List.of(imagePart)); + when(agent.conversationService.parseMessageParts(followUp)).thenReturn(List.of()); + when(agent.conversationService.parseMessageParts(asst)).thenReturn(List.of()); + + UserMessage result = agent.callBuildCurrent("c1", "左上角有没有小字"); + + assertTrue(result.getMedia() != null && result.getMedia().size() == 1, + "the recent image must be re-attached to the follow-up turn"); + assertTrue(result.getText().contains("较早发送的"), + "a note must explain the carried image to the model"); + } finally { + Files.deleteIfExists(img); + } + } + + @Test + @DisplayName("Text-only model → no image carried (relies on persisted caption instead)") + void textOnlyModel_doesNotCarry() throws Exception { + Path img = Files.createTempFile("carry-test", ".jpg"); + Files.write(img, new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF, 0x00}); + try { + TestAgent agent = newAgent(EnumSet.of(ModelCapabilityService.Modality.TEXT)); + MessageEntity imgTurn = userMsg("看看这张图"); + MessageEntity followUp = userMsg("左上角有没有小字"); + List history = List.of(imgTurn, followUp); + when(agent.conversationService.listMessages("c1")).thenReturn(history); + when(agent.conversationService.renderMessageContent(followUp)).thenReturn("左上角有没有小字"); + when(agent.conversationService.parseMessageParts(any())).thenReturn(List.of()); + + UserMessage result = agent.callBuildCurrent("c1", "左上角有没有小字"); + + assertTrue(result.getMedia() == null || result.getMedia().isEmpty(), + "text-only model must not get raw image bytes carried over"); + } finally { + Files.deleteIfExists(img); + } + } + + @Test + @DisplayName("Current turn already has an image → nothing extra carried") + void currentTurnHasImage_noCarry() throws Exception { + Path older = Files.createTempFile("carry-old", ".jpg"); + Path now = Files.createTempFile("carry-now", ".jpg"); + Files.write(older, new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF, 0x00}); + Files.write(now, new byte[]{(byte) 0xFF, (byte) 0xD8, (byte) 0xFF, 0x00}); + try { + TestAgent agent = visionAgent(); + MessageEntity oldTurn = userMsg("第一张"); + MessageEntity curTurn = userMsg("第二张"); + List history = List.of(oldTurn, curTurn); + when(agent.conversationService.listMessages("c1")).thenReturn(history); + when(agent.conversationService.renderMessageContent(curTurn)).thenReturn("第二张"); + when(agent.conversationService.parseMessageParts(oldTurn)) + .thenReturn(List.of(imagePart(older.toAbsolutePath().toString()))); + when(agent.conversationService.parseMessageParts(curTurn)) + .thenReturn(List.of(imagePart(now.toAbsolutePath().toString()))); + + UserMessage result = agent.callBuildCurrent("c1", "第二张"); + + assertTrue(result.getMedia() != null && result.getMedia().size() == 1, + "only the current turn's own image should be present — no extra carry"); + assertFalse(result.getText().contains("较早发送的"), + "no carry note when the current turn already has an image"); + } finally { + Files.deleteIfExists(older); + Files.deleteIfExists(now); + } + } + + // ---------- scaffold ---------- + + private static MessageContentPart imagePart(String path) { + MessageContentPart p = new MessageContentPart(); + p.setType("image"); + p.setContentType("image/jpeg"); + p.setFileName("image.jpg"); + p.setPath(path); + return p; + } + + private static MessageEntity userMsg(String content) { + MessageEntity m = new MessageEntity(); + m.setRole("user"); + m.setContent(content); + return m; + } + + private static MessageEntity assistantMsg(String content) { + MessageEntity m = new MessageEntity(); + m.setRole("assistant"); + m.setContent(content); + return m; + } + + private static TestAgent visionAgent() { + return newAgent(EnumSet.of(ModelCapabilityService.Modality.VISION, ModelCapabilityService.Modality.TEXT)); + } + + private static TestAgent newAgent(EnumSet caps) { + ConversationService conv = mock(ConversationService.class); + TestAgent agent = new TestAgent(conv); + agent.modelCapabilities = caps; + agent.modelName = "test-model"; + agent.agentName = "test-agent"; + return agent; + } + + static class TestAgent extends BaseAgent { + TestAgent(ConversationService conv) { + super(null, conv); + } + + UserMessage callBuildCurrent(String conversationId, String text) { + return buildCurrentUserMessageWithRouting(conversationId, text).userMessage(); + } + + @Override + public String chat(String userMessage, String conversationId) { + throw new UnsupportedOperationException(); + } + + @Override + public reactor.core.publisher.Flux chatStream(String userMessage, String conversationId) { + throw new UnsupportedOperationException(); + } + + @Override + public String execute(String goal, String conversationId) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentImageCaptionPersistTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentImageCaptionPersistTest.java new file mode 100644 index 00000000..50a92988 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentImageCaptionPersistTest.java @@ -0,0 +1,163 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.ai.chat.messages.UserMessage; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.MediaCaptionService; +import vip.mate.llm.routing.MultimodalRouter; +import vip.mate.llm.routing.model.MultimodalRoutingDecision; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.EnumSet; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Issue #303: when a text-only primary model captions an uploaded image via the + * vision sidecar, the caption must (1) be tailored to the user's actual question + * and (2) be persisted back onto the message part, so later turns — which replay + * user messages as text only — retain the image content instead of losing it. + */ +class BaseAgentImageCaptionPersistTest { + + private static final String DESCRIPTION = "图中是一段 NullPointerException 堆栈，发生在 UserService.login 第 42 行。"; + + @Test + @DisplayName("Sidecar caption is persisted onto the image part and folded into the prompt") + void sidecarCaption_persistedAndInjected() { + TestHarness h = newHarness(); + MessageContentPart image = imagePart(); + MessageEntity msg = userMessage(); + when(h.agent.conversationService.parseMessageParts(msg)) + .thenReturn(List.of(MessageContentPart.text("图里的报错是什么"), image)); + + UserMessage result = h.agent.callBuildCurrentTurn(msg, "图里的报错是什么"); + + // (1) caption stored on the part → survives into later turns + assertEquals(DESCRIPTION, image.getCaption(), "caption must be written onto the image part"); + verify(h.agent.conversationService).updateMessageParts(eq(msg), any()); + // (2) caption folded into the current-turn prompt text + assertTrue(result.getText().contains(DESCRIPTION), "caption must be injected into the prompt"); + assertTrue(result.getText().contains("[图片附件描述"), "caption must be wrapped in the attachment marker"); + } + + @Test + @DisplayName("The user's text question is passed to the caption service") + void userQuestion_passedToCaption() { + TestHarness h = newHarness(); + MessageEntity msg = userMessage(); + when(h.agent.conversationService.parseMessageParts(msg)) + .thenReturn(List.of(MessageContentPart.text("报错的行号是多少"), imagePart())); + + h.agent.callBuildCurrentTurn(msg, "报错的行号是多少"); + + ArgumentCaptor question = ArgumentCaptor.forClass(String.class); + verify(h.caption).caption(any(), any(), any(), question.capture()); + assertEquals("报错的行号是多少", question.getValue(), + "the user's question (text part) must drive a context-aware caption"); + } + + @Test + @DisplayName("Image-only message (no text part) → caption called with null question") + void imageOnly_nullQuestion() { + TestHarness h = newHarness(); + MessageEntity msg = userMessage(); + // WeChat Work image upload: only an image part, content placeholder "[图片]". + when(h.agent.conversationService.parseMessageParts(msg)) + .thenReturn(List.of(imagePart())); + + h.agent.callBuildCurrentTurn(msg, "[图片]"); + + ArgumentCaptor question = ArgumentCaptor.forClass(String.class); + verify(h.caption).caption(any(), any(), any(), question.capture()); + assertEquals(null, question.getValue(), + "no text part → null question → caption falls back to generic description"); + } + + // ---------- scaffold ---------- + + private static MessageContentPart imagePart() { + MessageContentPart p = new MessageContentPart(); + p.setType("image"); + p.setContentType("image/png"); + p.setFileName("err.png"); + p.setMediaId("media-1"); + return p; + } + + private static MessageEntity userMessage() { + MessageEntity m = new MessageEntity(); + m.setId(1001L); + m.setRole("user"); + m.setContent("[图片]"); + return m; + } + + private TestHarness newHarness() { + ConversationService conv = mock(ConversationService.class); + MultimodalRouter router = mock(MultimodalRouter.class); + MediaCaptionService caption = mock(MediaCaptionService.class); + ModelConfigEntity sidecar = mock(ModelConfigEntity.class); + + when(router.route(any(), any())).thenReturn( + MultimodalRoutingDecision.sidecar(sidecar, + EnumSet.of(ModelCapabilityService.Modality.VISION), + EnumSet.of(ModelCapabilityService.Modality.VISION))); + when(caption.caption(any(), any(), any(), any())) + .thenReturn(MediaCaptionService.CaptionResult.success(DESCRIPTION, 12L, false)); + + TestAgent agent = new TestAgent(conv); + agent.multimodalRouter = router; + agent.mediaCaptionService = caption; + agent.modelCapabilities = EnumSet.noneOf(ModelCapabilityService.Modality.class); + agent.modelName = "text-only-model"; + agent.agentName = "test-agent"; + + TestHarness h = new TestHarness(); + h.agent = agent; + h.caption = caption; + return h; + } + + static class TestHarness { + TestAgent agent; + MediaCaptionService caption; + } + + static class TestAgent extends BaseAgent { + TestAgent(ConversationService conv) { + super(null, conv); + } + + UserMessage callBuildCurrentTurn(MessageEntity msg, String renderedContent) { + return buildUserMessageForCurrentTurn(msg, renderedContent).userMessage(); + } + + @Override + public String chat(String userMessage, String conversationId) { + throw new UnsupportedOperationException(); + } + + @Override + public reactor.core.publisher.Flux chatStream(String userMessage, String conversationId) { + throw new UnsupportedOperationException(); + } + + @Override + public String execute(String goal, String conversationId) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java index e5a94d8b..59d70d08 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java @@ -86,6 +86,35 @@ class BaseAgentMultimodalSkipNoticeTest { "image must NOT be injected when model has no VISION capability"); } + @Test + @DisplayName("Vision-capable model + image with only a remote URL (never downloaded) → actionable 'enable media download' hint") + void remoteOnlyImage_emitsDownloadHint() { + // Issue #303 follow-up: WeCom/aibot delivers images as short-lived, + // AES-encrypted COS URLs. With channel media download off, the part is + // stored URL-only (path=null, mediaId=https://...). The model supports + // vision, so we reach the file-resolution branch — which must surface an + // actionable hint instead of a dead-end "文件未找到". + TestAgent agent = newAgentWithCaps( + EnumSet.of(ModelCapabilityService.Modality.VISION, ModelCapabilityService.Modality.TEXT)); + MessageEntity msg = userMessage("看看这张图"); + MessageContentPart remote = new MessageContentPart(); + remote.setType("image"); + remote.setContentType("image/jpeg"); + remote.setFileName("image.jpg"); + remote.setMediaId("https://ww-aibot-img.cos.example.com/x?sign=abc"); + when(agent.conversationService.parseMessageParts(msg)).thenReturn(List.of(remote)); + + UserMessage result = agent.callBuildUserMessage(msg, "看看这张图"); + + String text = result.getText(); + assertTrue(text.contains("未下载到本地"), + "remote-only attachment must hint that it was never downloaded locally"); + assertTrue(text.contains("开启") && text.contains("媒体下载"), + "hint must tell the user to enable channel media download"); + assertTrue(result.getMedia() == null || result.getMedia().isEmpty(), + "a remote URL must not be injected as Media"); + } + @Test @DisplayName("No attachments → no system notice, prompt text unchanged") void noAttachments_noNoticeAdded() { diff --git a/mateclaw-server/src/test/java/vip/mate/llm/routing/MediaCaptionServiceBuildPromptTest.java b/mateclaw-server/src/test/java/vip/mate/llm/routing/MediaCaptionServiceBuildPromptTest.java new file mode 100644 index 00000000..759da212 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/routing/MediaCaptionServiceBuildPromptTest.java @@ -0,0 +1,70 @@ +package vip.mate.llm.routing; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pins the two-stage prompt contract of {@link MediaCaptionService}: when a user + * question is supplied the vision model must be asked to answer it (so multi-turn + * follow-ups get a tailored answer instead of a generic caption), and when no + * question is supplied it must fall back to the factual full-description prompt. + * + *

{@code buildPrompt} is private — it is the smallest unit that captures the + * branching, so it is exercised via reflection rather than driving a real + * (networked) vision call. + */ +class MediaCaptionServiceBuildPromptTest { + + private static String buildPrompt(Locale locale, String fileName, String userQuestion) throws Exception { + // Dependencies are unused by buildPrompt; null is fine for this unit. + MediaCaptionService service = new MediaCaptionService(null, null); + Method m = MediaCaptionService.class.getDeclaredMethod( + "buildPrompt", Locale.class, String.class, String.class); + m.setAccessible(true); + return (String) m.invoke(service, locale, fileName, userQuestion); + } + + @Test + @DisplayName("Question + Chinese locale → question-aware prompt embedding the question") + void chineseWithQuestion_isQuestionAware() throws Exception { + String prompt = buildPrompt(Locale.SIMPLIFIED_CHINESE, "err.png", "图里的报错是什么"); + + assertTrue(prompt.contains("图里的报错是什么"), "the user's question must be embedded verbatim"); + assertTrue(prompt.contains("回答用户的问题"), "must instruct the model to answer, not just describe"); + assertFalse(prompt.contains("不超过 300 字"), + "question-aware prompt must not reuse the generic description template"); + } + + @Test + @DisplayName("Question + English locale → English question-aware prompt") + void englishWithQuestion_isQuestionAware() throws Exception { + String prompt = buildPrompt(Locale.ENGLISH, "err.png", "What is the error message?"); + + assertTrue(prompt.contains("What is the error message?")); + assertTrue(prompt.contains("answer the user's question")); + } + + @Test + @DisplayName("Blank question → generic Chinese description prompt") + void chineseNoQuestion_isGeneric() throws Exception { + String prompt = buildPrompt(Locale.SIMPLIFIED_CHINESE, "photo.jpg", " "); + + assertTrue(prompt.contains("请用一段简洁的中文描述这张图片")); + assertFalse(prompt.contains("回答用户的问题")); + } + + @Test + @DisplayName("Null question + English → generic English description prompt") + void englishNoQuestion_isGeneric() throws Exception { + String prompt = buildPrompt(Locale.ENGLISH, null, null); + + assertTrue(prompt.contains("Describe this image")); + assertFalse(prompt.contains("answer the user's question")); + } +}