mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-14 11:37:31 +08:00
feat(channel + settings): wecom quote message parsing + sidecar config preservation
WeCom quoted-message context: - Parse the body.quote field that arrives alongside any inbound message (text / image / voice / file / mixed sub-types). When a user long- presses a previous bot bubble and types a follow-up like "解释一下", the agent now sees both the user's new text and the referenced content as proper context — replies stay on topic instead of guessing what was being explained. - Quoted images / files are downloaded through the same pipeline as inbound new media (magic-byte sniff, ZIP container peek for DOCX / XLSX / PPTX recovery, chat-uploads layout) so the vision sidecar and document tools can actually analyse what was quoted. - Reading order in the assembled prompt: "[引用消息: ...]\n<user text>" first, then quoted media parts, then the user's own current-message media. Mixed quotes flatten into a space-joined summary. Multimodal sidecar settings preservation: - The bulk settings PUT used to unconditionally overwrite the vision / video sidecar model ids — null in a partial payload became "" in the DB, silently wiping the configured sidecar every time a user saved an unrelated settings page (System / Music / Image / etc.). Symptom: "I picked a vision model, saved a different settings tab, now the bot can't see images anymore." - Bulk save now guards both keys with non-null checks, matching the pattern used for music / 3D / image / video / tts / stt blocks. - A dedicated /settings/sidecar endpoint always writes both keys, so the sidecar UI can still explicitly clear via null without leaking the write-on-null semantics into every other settings save. - Frontend sidecar card switches to the dedicated endpoint; other settings pages keep their existing partial-payload behaviour.
This commit is contained in:
parent
67b3b548e6
commit
9075abfef2
@ -1006,6 +1006,47 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Apply any quoted-message context. The "quote" field appears at
|
||||||
|
// body level alongside the new message regardless of outer
|
||||||
|
// msgtype — without parsing it, the agent only sees the user's
|
||||||
|
// current text and silently loses the conversational reference
|
||||||
|
// ("user quoted the bot's previous image and asked '什么意思'"
|
||||||
|
// arrives as bare "什么意思", agent goes off-topic).
|
||||||
|
QuoteContext quote = extractQuoteContext(body, msgId, senderId, chatId, chatType);
|
||||||
|
if (quote != null && !quote.isEmpty()) {
|
||||||
|
String prefixedText = quote.prefix()
|
||||||
|
+ (textContent != null && !textContent.isBlank() ? textContent : "");
|
||||||
|
textContent = prefixedText;
|
||||||
|
|
||||||
|
// Find an existing text part (text/voice cases produce one)
|
||||||
|
// and overwrite it with the prefixed text. Also pull it to
|
||||||
|
// the front so agent reading order starts with quote prefix.
|
||||||
|
boolean updated = false;
|
||||||
|
for (int i = 0; i < contentParts.size(); i++) {
|
||||||
|
if ("text".equals(contentParts.get(i).getType())) {
|
||||||
|
contentParts.set(i, MessageContentPart.text(prefixedText));
|
||||||
|
if (i != 0) {
|
||||||
|
MessageContentPart promoted = contentParts.remove(i);
|
||||||
|
contentParts.add(0, promoted);
|
||||||
|
}
|
||||||
|
updated = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!updated) {
|
||||||
|
// image/file/mixed cases without a text part — insert one.
|
||||||
|
contentParts.add(0, MessageContentPart.text(prefixedText));
|
||||||
|
}
|
||||||
|
// Quoted media (image/file the user referenced) goes right
|
||||||
|
// after the text prefix so the agent reads:
|
||||||
|
// prefix → quoted media → user's own media (if any)
|
||||||
|
if (!quote.attachedParts().isEmpty()) {
|
||||||
|
contentParts.addAll(1, quote.attachedParts());
|
||||||
|
}
|
||||||
|
log.debug("[wecom] Applied quote context: prefixLen={}, attachedParts={}",
|
||||||
|
quote.prefix().length(), quote.attachedParts().size());
|
||||||
|
}
|
||||||
|
|
||||||
if (contentParts.isEmpty()) {
|
if (contentParts.isEmpty()) {
|
||||||
if (textContent != null && !textContent.isBlank()) {
|
if (textContent != null && !textContent.isBlank()) {
|
||||||
contentParts.add(MessageContentPart.text(textContent));
|
contentParts.add(MessageContentPart.text(textContent));
|
||||||
@ -2255,6 +2296,135 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
|||||||
return isGroup ? "wecom:group:" + chatId : "wecom:" + senderId;
|
return isGroup ? "wecom:group:" + chatId : "wecom:" + senderId;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== 引用消息(quote)解析 ====================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Quoted-message extraction result: a human-readable prefix string the
|
||||||
|
* agent prompt prepends, plus any media (image / file) that was quoted
|
||||||
|
* and needs to be available as a content part. Either field may be
|
||||||
|
* empty; {@link #isEmpty()} returns true only when both are.
|
||||||
|
*/
|
||||||
|
private record QuoteContext(String prefix, List<MessageContentPart> attachedParts) {
|
||||||
|
boolean isEmpty() {
|
||||||
|
return (prefix == null || prefix.isBlank()) && attachedParts.isEmpty();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the {@code body.quote} field of an inbound WeCom AI Bot frame.
|
||||||
|
* Quote payloads sit alongside the new message at body level —
|
||||||
|
* independent of the outer {@code msgtype} — and may themselves carry
|
||||||
|
* any of text / voice / image / file / mixed. We flatten everything
|
||||||
|
* into:
|
||||||
|
* <ul>
|
||||||
|
* <li>a single {@code prefix} string of the form
|
||||||
|
* {@code "[引用消息: <flattened summary>]\n"} that prepends to the
|
||||||
|
* agent's user prompt</li>
|
||||||
|
* <li>a list of {@link MessageContentPart}s for any quoted media so
|
||||||
|
* the vision / document tools can analyse the actually-quoted
|
||||||
|
* image or PDF, not just see "[图片]" in the prefix</li>
|
||||||
|
* </ul>
|
||||||
|
* Returns null when {@code body.quote} is absent / empty / malformed —
|
||||||
|
* the caller treats that the same as "no quote context".
|
||||||
|
*/
|
||||||
|
private QuoteContext extractQuoteContext(Map<String, Object> body, String msgId,
|
||||||
|
String senderId, String chatId, String chatType) {
|
||||||
|
Object raw = body.get("quote");
|
||||||
|
if (!(raw instanceof Map<?, ?> map)) return null;
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
Map<String, Object> quote = (Map<String, Object>) map;
|
||||||
|
String quoteType = (String) quote.getOrDefault("msgtype", "");
|
||||||
|
if (quoteType == null || quoteType.isBlank()) return null;
|
||||||
|
|
||||||
|
// Flatten: a "mixed" quote nests its own msg_item array; single-type
|
||||||
|
// quotes act as a one-element list of themselves.
|
||||||
|
List<Map<String, Object>> items;
|
||||||
|
if ("mixed".equals(quoteType)) {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
Map<String, Object> mixed = (Map<String, Object>) quote.getOrDefault("mixed", Map.of());
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
List<Map<String, Object>> mi = (List<Map<String, Object>>) mixed.getOrDefault("msg_item", List.of());
|
||||||
|
items = mi;
|
||||||
|
} else {
|
||||||
|
items = List.of(quote);
|
||||||
|
}
|
||||||
|
|
||||||
|
String inboundConvId = inboundConversationId(senderId, chatId, chatType);
|
||||||
|
StringBuilder summary = new StringBuilder();
|
||||||
|
List<MessageContentPart> attached = new ArrayList<>();
|
||||||
|
|
||||||
|
for (Map<String, Object> item : items) {
|
||||||
|
String itemType = (String) item.getOrDefault("msgtype", "");
|
||||||
|
switch (itemType == null ? "" : itemType) {
|
||||||
|
case "text" -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
Map<String, Object> t = (Map<String, Object>) item.getOrDefault("text", Map.of());
|
||||||
|
String content = ((String) t.getOrDefault("content", "")).trim();
|
||||||
|
if (!content.isBlank()) {
|
||||||
|
appendQuoteSummary(summary, content);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "voice" -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
Map<String, Object> v = (Map<String, Object>) item.getOrDefault("voice", Map.of());
|
||||||
|
String asr = ((String) v.getOrDefault("content", "")).trim();
|
||||||
|
if (!asr.isBlank()) {
|
||||||
|
appendQuoteSummary(summary, "[语音] " + asr);
|
||||||
|
} else {
|
||||||
|
appendQuoteSummary(summary, "[语音消息]");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "image" -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
Map<String, Object> img = (Map<String, Object>) item.getOrDefault("image", Map.of());
|
||||||
|
String url = (String) img.getOrDefault("url", "");
|
||||||
|
String aesKey = (String) img.getOrDefault("aeskey", "");
|
||||||
|
if (!url.isBlank()) {
|
||||||
|
attached.add(buildInboundImagePart(url, aesKey, msgId,
|
||||||
|
"quoted_image.jpg", inboundConvId));
|
||||||
|
}
|
||||||
|
appendQuoteSummary(summary, "[图片]");
|
||||||
|
}
|
||||||
|
case "file" -> {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
Map<String, Object> f = (Map<String, Object>) item.getOrDefault("file", Map.of());
|
||||||
|
String url = (String) f.getOrDefault("url", "");
|
||||||
|
String aesKey = (String) f.getOrDefault("aeskey", "");
|
||||||
|
String filename = (String) f.getOrDefault("filename",
|
||||||
|
f.getOrDefault("file_name", f.getOrDefault("name", "file.bin")));
|
||||||
|
if (!url.isBlank()) {
|
||||||
|
MessageContentPart part = buildInboundFilePart(url, aesKey, msgId,
|
||||||
|
filename, inboundConvId);
|
||||||
|
attached.add(part);
|
||||||
|
if (part.getFileName() != null && !part.getFileName().isBlank()) {
|
||||||
|
filename = part.getFileName();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
appendQuoteSummary(summary, "[文件: " + filename + "]");
|
||||||
|
}
|
||||||
|
default -> {
|
||||||
|
// Unknown quote sub-type — surface the type tag so the
|
||||||
|
// agent knows something was quoted even if we can't
|
||||||
|
// unpack it.
|
||||||
|
if (itemType != null && !itemType.isBlank()) {
|
||||||
|
appendQuoteSummary(summary, "[" + itemType + "]");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (summary.length() == 0 && attached.isEmpty()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
String prefix = "[引用消息: " + summary + "]\n";
|
||||||
|
return new QuoteContext(prefix, attached);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void appendQuoteSummary(StringBuilder summary, String fragment) {
|
||||||
|
if (summary.length() > 0) summary.append(' ');
|
||||||
|
summary.append(fragment);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build a fully-populated image content part for inbound WeCom media.
|
* Build a fully-populated image content part for inbound WeCom media.
|
||||||
* <p>
|
* <p>
|
||||||
|
|||||||
@ -41,8 +41,39 @@ public class SystemSettingController {
|
|||||||
return R.ok(systemSettingService.saveLanguage(request.getLanguage()));
|
return R.ok(systemSettingService.saveLanguage(request.getLanguage()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dedicated endpoint for the multimodal sidecar configuration.
|
||||||
|
* <p>
|
||||||
|
* Separated from the bulk {@code PUT /settings} because the bulk endpoint
|
||||||
|
* now guards sidecar keys with null checks (so unrelated settings pages
|
||||||
|
* can't clobber them via partial payloads). This endpoint always writes
|
||||||
|
* both fields, so passing {@code null} for either explicitly clears that
|
||||||
|
* sidecar — preserving the "clear via UI" UX without leaking the
|
||||||
|
* write-on-null semantics into every other settings save.
|
||||||
|
*/
|
||||||
|
@Operation(summary = "更新多模态 sidecar 配置")
|
||||||
|
@PutMapping("/sidecar")
|
||||||
|
public R<SystemSettingsDTO> saveSidecar(@RequestBody SidecarRequest request) {
|
||||||
|
return R.ok(systemSettingService.updateSidecarSettings(
|
||||||
|
request.getDefaultVisionModelId(),
|
||||||
|
request.getDefaultVideoModelId()));
|
||||||
|
}
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
public static class LanguageRequest {
|
public static class LanguageRequest {
|
||||||
private String language;
|
private String language;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Body for {@code PUT /settings/sidecar}. Both fields are nullable;
|
||||||
|
* {@code null} means "explicit clear". Field absence in the JSON
|
||||||
|
* payload also deserializes to null, which is the same outcome — the
|
||||||
|
* sidecar UI is the only caller of this endpoint and always sends both
|
||||||
|
* fields, so the absent-vs-null distinction doesn't matter here.
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
public static class SidecarRequest {
|
||||||
|
private Long defaultVisionModelId;
|
||||||
|
private Long defaultVideoModelId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -351,13 +351,47 @@ public class SystemSettingService {
|
|||||||
saveValue(MODEL3D_FALLBACK_ENABLED_KEY, String.valueOf(dto.getModel3dFallbackEnabled()), "3D Provider 级 Fallback");
|
saveValue(MODEL3D_FALLBACK_ENABLED_KEY, String.valueOf(dto.getModel3dFallbackEnabled()), "3D Provider 级 Fallback");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Multimodal sidecar routing — write empty string to clear (parse-back returns null)
|
// Multimodal sidecar routing — guarded with null check, matching the
|
||||||
// Always written so users can revert to "not configured" via the UI.
|
// pattern used for music / 3D / image / video / tts / stt blocks
|
||||||
|
// above. The bulk PUT /settings is used by every settings page (System,
|
||||||
|
// Music, Video, Image, Stt, Tts, Model3D), each sending a partial
|
||||||
|
// payload that omits sidecar fields. Without this guard, saving any
|
||||||
|
// unrelated setting would silently write "" into the sidecar keys
|
||||||
|
// (Long? defaultVisionModelId deserializes to null when absent), which
|
||||||
|
// wiped users' configured vision/video models the moment they touched
|
||||||
|
// an unrelated settings page. Explicit clearing via the sidecar UI now
|
||||||
|
// routes through {@link #updateSidecarSettings} instead.
|
||||||
|
if (dto.getDefaultVisionModelId() != null) {
|
||||||
|
saveValue(DEFAULT_VISION_MODEL_KEY,
|
||||||
|
String.valueOf(dto.getDefaultVisionModelId()),
|
||||||
|
"Default vision-capable model id (mate_model_config.id) for sidecar routing");
|
||||||
|
}
|
||||||
|
if (dto.getDefaultVideoModelId() != null) {
|
||||||
|
saveValue(DEFAULT_VIDEO_MODEL_KEY,
|
||||||
|
String.valueOf(dto.getDefaultVideoModelId()),
|
||||||
|
"Default video-capable model id (mate_model_config.id) for sidecar routing");
|
||||||
|
}
|
||||||
|
return getSettings();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dedicated update path for the multimodal sidecar configuration.
|
||||||
|
* <p>
|
||||||
|
* This endpoint is the ONLY place vision/video model ids can be written
|
||||||
|
* unconditionally — null is treated as an explicit "clear" and writes
|
||||||
|
* an empty string (parse-back returns null). The bulk
|
||||||
|
* {@link #saveSettings} now guards both keys with non-null checks so
|
||||||
|
* unrelated settings pages can't accidentally clobber sidecar config.
|
||||||
|
* <p>
|
||||||
|
* Both fields are always written so a single API call can independently
|
||||||
|
* assign / clear either modality.
|
||||||
|
*/
|
||||||
|
public SystemSettingsDTO updateSidecarSettings(Long visionModelId, Long videoModelId) {
|
||||||
saveValue(DEFAULT_VISION_MODEL_KEY,
|
saveValue(DEFAULT_VISION_MODEL_KEY,
|
||||||
dto.getDefaultVisionModelId() == null ? "" : String.valueOf(dto.getDefaultVisionModelId()),
|
visionModelId == null ? "" : String.valueOf(visionModelId),
|
||||||
"Default vision-capable model id (mate_model_config.id) for sidecar routing");
|
"Default vision-capable model id (mate_model_config.id) for sidecar routing");
|
||||||
saveValue(DEFAULT_VIDEO_MODEL_KEY,
|
saveValue(DEFAULT_VIDEO_MODEL_KEY,
|
||||||
dto.getDefaultVideoModelId() == null ? "" : String.valueOf(dto.getDefaultVideoModelId()),
|
videoModelId == null ? "" : String.valueOf(videoModelId),
|
||||||
"Default video-capable model id (mate_model_config.id) for sidecar routing");
|
"Default video-capable model id (mate_model_config.id) for sidecar routing");
|
||||||
return getSettings();
|
return getSettings();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -518,6 +518,13 @@ export const settingsApi = {
|
|||||||
update: (data: any) => http.put('/settings', data),
|
update: (data: any) => http.put('/settings', data),
|
||||||
getLanguage: () => http.get('/settings/language'),
|
getLanguage: () => http.get('/settings/language'),
|
||||||
updateLanguage: (language: string) => http.put('/settings/language', { language }),
|
updateLanguage: (language: string) => http.put('/settings/language', { language }),
|
||||||
|
// Dedicated endpoint for the multimodal sidecar configuration. The bulk
|
||||||
|
// /settings PUT now guards vision/video model ids with non-null checks so
|
||||||
|
// unrelated settings pages can't clobber them via partial payloads. This
|
||||||
|
// endpoint is the only path that writes those fields unconditionally —
|
||||||
|
// pass {defaultVisionModelId: null} here to explicitly clear a sidecar.
|
||||||
|
updateSidecar: (data: { defaultVisionModelId: number | null; defaultVideoModelId: number | null }) =>
|
||||||
|
http.put('/settings/sidecar', data),
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== Workspace ====================
|
// ==================== Workspace ====================
|
||||||
|
|||||||
@ -144,7 +144,12 @@ async function loadAll() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function persistSettings(payload: { defaultVisionModelId: number | null; defaultVideoModelId: number | null }) {
|
async function persistSettings(payload: { defaultVisionModelId: number | null; defaultVideoModelId: number | null }) {
|
||||||
await settingsApi.update(payload)
|
// Use the dedicated sidecar endpoint so the bulk /settings PUT can keep
|
||||||
|
// guarding vision/video keys with non-null checks (preventing unrelated
|
||||||
|
// settings pages from clobbering this configuration via partial payloads).
|
||||||
|
// This endpoint always writes both keys, so passing null here means
|
||||||
|
// "explicit clear" which is the original UX of this card.
|
||||||
|
await settingsApi.updateSidecar(payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function onSaveVision() {
|
async function onSaveVision() {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user