diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java index a92af82e..319b96a9 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java @@ -1006,6 +1006,47 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } } + // Apply any quoted-message context. The "quote" field appears at + // body level alongside the new message regardless of outer + // msgtype — without parsing it, the agent only sees the user's + // current text and silently loses the conversational reference + // ("user quoted the bot's previous image and asked '什么意思'" + // arrives as bare "什么意思", agent goes off-topic). + QuoteContext quote = extractQuoteContext(body, msgId, senderId, chatId, chatType); + if (quote != null && !quote.isEmpty()) { + String prefixedText = quote.prefix() + + (textContent != null && !textContent.isBlank() ? textContent : ""); + textContent = prefixedText; + + // Find an existing text part (text/voice cases produce one) + // and overwrite it with the prefixed text. Also pull it to + // the front so agent reading order starts with quote prefix. + boolean updated = false; + for (int i = 0; i < contentParts.size(); i++) { + if ("text".equals(contentParts.get(i).getType())) { + contentParts.set(i, MessageContentPart.text(prefixedText)); + if (i != 0) { + MessageContentPart promoted = contentParts.remove(i); + contentParts.add(0, promoted); + } + updated = true; + break; + } + } + if (!updated) { + // image/file/mixed cases without a text part — insert one. + contentParts.add(0, MessageContentPart.text(prefixedText)); + } + // Quoted media (image/file the user referenced) goes right + // after the text prefix so the agent reads: + // prefix → quoted media → user's own media (if any) + if (!quote.attachedParts().isEmpty()) { + contentParts.addAll(1, quote.attachedParts()); + } + log.debug("[wecom] Applied quote context: prefixLen={}, attachedParts={}", + quote.prefix().length(), quote.attachedParts().size()); + } + if (contentParts.isEmpty()) { if (textContent != null && !textContent.isBlank()) { contentParts.add(MessageContentPart.text(textContent)); @@ -2255,6 +2296,135 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { return isGroup ? "wecom:group:" + chatId : "wecom:" + senderId; } + // ==================== 引用消息（quote）解析 ==================== + + /** + * Quoted-message extraction result: a human-readable prefix string the + * agent prompt prepends, plus any media (image / file) that was quoted + * and needs to be available as a content part. Either field may be + * empty; {@link #isEmpty()} returns true only when both are. + */ + private record QuoteContext(String prefix, List attachedParts) { + boolean isEmpty() { + return (prefix == null || prefix.isBlank()) && attachedParts.isEmpty(); + } + } + + /** + * Parse the {@code body.quote} field of an inbound WeCom AI Bot frame. + * Quote payloads sit alongside the new message at body level — + * independent of the outer {@code msgtype} — and may themselves carry + * any of text / voice / image / file / mixed. We flatten everything + * into: + * + * Returns null when {@code body.quote} is absent / empty / malformed — + * the caller treats that the same as "no quote context". + */ + private QuoteContext extractQuoteContext(Map body, String msgId, + String senderId, String chatId, String chatType) { + Object raw = body.get("quote"); + if (!(raw instanceof Map map)) return null; + @SuppressWarnings("unchecked") + Map quote = (Map) map; + String quoteType = (String) quote.getOrDefault("msgtype", ""); + if (quoteType == null || quoteType.isBlank()) return null; + + // Flatten: a "mixed" quote nests its own msg_item array; single-type + // quotes act as a one-element list of themselves. + List> items; + if ("mixed".equals(quoteType)) { + @SuppressWarnings("unchecked") + Map mixed = (Map) quote.getOrDefault("mixed", Map.of()); + @SuppressWarnings("unchecked") + List> mi = (List>) mixed.getOrDefault("msg_item", List.of()); + items = mi; + } else { + items = List.of(quote); + } + + String inboundConvId = inboundConversationId(senderId, chatId, chatType); + StringBuilder summary = new StringBuilder(); + List attached = new ArrayList<>(); + + for (Map item : items) { + String itemType = (String) item.getOrDefault("msgtype", ""); + switch (itemType == null ? "" : itemType) { + case "text" -> { + @SuppressWarnings("unchecked") + Map t = (Map) item.getOrDefault("text", Map.of()); + String content = ((String) t.getOrDefault("content", "")).trim(); + if (!content.isBlank()) { + appendQuoteSummary(summary, content); + } + } + case "voice" -> { + @SuppressWarnings("unchecked") + Map v = (Map) item.getOrDefault("voice", Map.of()); + String asr = ((String) v.getOrDefault("content", "")).trim(); + if (!asr.isBlank()) { + appendQuoteSummary(summary, "[语音] " + asr); + } else { + appendQuoteSummary(summary, "[语音消息]"); + } + } + case "image" -> { + @SuppressWarnings("unchecked") + Map img = (Map) item.getOrDefault("image", Map.of()); + String url = (String) img.getOrDefault("url", ""); + String aesKey = (String) img.getOrDefault("aeskey", ""); + if (!url.isBlank()) { + attached.add(buildInboundImagePart(url, aesKey, msgId, + "quoted_image.jpg", inboundConvId)); + } + appendQuoteSummary(summary, "[图片]"); + } + case "file" -> { + @SuppressWarnings("unchecked") + Map f = (Map) item.getOrDefault("file", Map.of()); + String url = (String) f.getOrDefault("url", ""); + String aesKey = (String) f.getOrDefault("aeskey", ""); + String filename = (String) f.getOrDefault("filename", + f.getOrDefault("file_name", f.getOrDefault("name", "file.bin"))); + if (!url.isBlank()) { + MessageContentPart part = buildInboundFilePart(url, aesKey, msgId, + filename, inboundConvId); + attached.add(part); + if (part.getFileName() != null && !part.getFileName().isBlank()) { + filename = part.getFileName(); + } + } + appendQuoteSummary(summary, "[文件: " + filename + "]"); + } + default -> { + // Unknown quote sub-type — surface the type tag so the + // agent knows something was quoted even if we can't + // unpack it. + if (itemType != null && !itemType.isBlank()) { + appendQuoteSummary(summary, "[" + itemType + "]"); + } + } + } + } + + if (summary.length() == 0 && attached.isEmpty()) { + return null; + } + String prefix = "[引用消息: " + summary + "]\n"; + return new QuoteContext(prefix, attached); + } + + private static void appendQuoteSummary(StringBuilder summary, String fragment) { + if (summary.length() > 0) summary.append(' '); + summary.append(fragment); + } + /** * Build a fully-populated image content part for inbound WeCom media. *

diff --git a/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java b/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java index 9f055aea..741f5b8d 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java +++ b/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java @@ -41,8 +41,39 @@ public class SystemSettingController { return R.ok(systemSettingService.saveLanguage(request.getLanguage())); } + /** + * Dedicated endpoint for the multimodal sidecar configuration. + *

+ * Separated from the bulk {@code PUT /settings} because the bulk endpoint + * now guards sidecar keys with null checks (so unrelated settings pages + * can't clobber them via partial payloads). This endpoint always writes + * both fields, so passing {@code null} for either explicitly clears that + * sidecar — preserving the "clear via UI" UX without leaking the + * write-on-null semantics into every other settings save. + */ + @Operation(summary = "更新多模态 sidecar 配置") + @PutMapping("/sidecar") + public R saveSidecar(@RequestBody SidecarRequest request) { + return R.ok(systemSettingService.updateSidecarSettings( + request.getDefaultVisionModelId(), + request.getDefaultVideoModelId())); + } + @Data public static class LanguageRequest { private String language; } + + /** + * Body for {@code PUT /settings/sidecar}. Both fields are nullable; + * {@code null} means "explicit clear". Field absence in the JSON + * payload also deserializes to null, which is the same outcome — the + * sidecar UI is the only caller of this endpoint and always sends both + * fields, so the absent-vs-null distinction doesn't matter here. + */ + @Data + public static class SidecarRequest { + private Long defaultVisionModelId; + private Long defaultVideoModelId; + } } diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index fa1232d5..1a22ca6a 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -351,13 +351,47 @@ public class SystemSettingService { saveValue(MODEL3D_FALLBACK_ENABLED_KEY, String.valueOf(dto.getModel3dFallbackEnabled()), "3D Provider 级 Fallback"); } - // Multimodal sidecar routing — write empty string to clear (parse-back returns null) - // Always written so users can revert to "not configured" via the UI. + // Multimodal sidecar routing — guarded with null check, matching the + // pattern used for music / 3D / image / video / tts / stt blocks + // above. The bulk PUT /settings is used by every settings page (System, + // Music, Video, Image, Stt, Tts, Model3D), each sending a partial + // payload that omits sidecar fields. Without this guard, saving any + // unrelated setting would silently write "" into the sidecar keys + // (Long? defaultVisionModelId deserializes to null when absent), which + // wiped users' configured vision/video models the moment they touched + // an unrelated settings page. Explicit clearing via the sidecar UI now + // routes through {@link #updateSidecarSettings} instead. + if (dto.getDefaultVisionModelId() != null) { + saveValue(DEFAULT_VISION_MODEL_KEY, + String.valueOf(dto.getDefaultVisionModelId()), + "Default vision-capable model id (mate_model_config.id) for sidecar routing"); + } + if (dto.getDefaultVideoModelId() != null) { + saveValue(DEFAULT_VIDEO_MODEL_KEY, + String.valueOf(dto.getDefaultVideoModelId()), + "Default video-capable model id (mate_model_config.id) for sidecar routing"); + } + return getSettings(); + } + + /** + * Dedicated update path for the multimodal sidecar configuration. + *

+ * This endpoint is the ONLY place vision/video model ids can be written + * unconditionally — null is treated as an explicit "clear" and writes + * an empty string (parse-back returns null). The bulk + * {@link #saveSettings} now guards both keys with non-null checks so + * unrelated settings pages can't accidentally clobber sidecar config. + *

+ * Both fields are always written so a single API call can independently + * assign / clear either modality. + */ + public SystemSettingsDTO updateSidecarSettings(Long visionModelId, Long videoModelId) { saveValue(DEFAULT_VISION_MODEL_KEY, - dto.getDefaultVisionModelId() == null ? "" : String.valueOf(dto.getDefaultVisionModelId()), + visionModelId == null ? "" : String.valueOf(visionModelId), "Default vision-capable model id (mate_model_config.id) for sidecar routing"); saveValue(DEFAULT_VIDEO_MODEL_KEY, - dto.getDefaultVideoModelId() == null ? "" : String.valueOf(dto.getDefaultVideoModelId()), + videoModelId == null ? "" : String.valueOf(videoModelId), "Default video-capable model id (mate_model_config.id) for sidecar routing"); return getSettings(); } diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 6a33abab..6ff54a20 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -518,6 +518,13 @@ export const settingsApi = { update: (data: any) => http.put('/settings', data), getLanguage: () => http.get('/settings/language'), updateLanguage: (language: string) => http.put('/settings/language', { language }), + // Dedicated endpoint for the multimodal sidecar configuration. The bulk + // /settings PUT now guards vision/video model ids with non-null checks so + // unrelated settings pages can't clobber them via partial payloads. This + // endpoint is the only path that writes those fields unconditionally — + // pass {defaultVisionModelId: null} here to explicitly clear a sidecar. + updateSidecar: (data: { defaultVisionModelId: number | null; defaultVideoModelId: number | null }) => + http.put('/settings/sidecar', data), } // ==================== Workspace ==================== diff --git a/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue b/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue index 7f9c7ed9..842afc45 100644 --- a/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue +++ b/mateclaw-ui/src/views/Settings/Models/MultimodalSidecarSection.vue @@ -144,7 +144,12 @@ async function loadAll() { } async function persistSettings(payload: { defaultVisionModelId: number | null; defaultVideoModelId: number | null }) { - await settingsApi.update(payload) + // Use the dedicated sidecar endpoint so the bulk /settings PUT can keep + // guarding vision/video keys with non-null checks (preventing unrelated + // settings pages from clobbering this configuration via partial payloads). + // This endpoint always writes both keys, so passing null here means + // "explicit clear" which is the original UX of this card. + await settingsApi.updateSidecar(payload) } async function onSaveVision() {