From 881badd15f4d33e711b533bf9e95996098a75ff2 Mon Sep 17 00:00:00 2001 From: matevip Date: Mon, 13 Apr 2026 16:38:09 +0800 Subject: [PATCH] feat(channel): weixin voice ASR fallback, wecom mixed message handling --- .../channel/wecom/WeComChannelAdapter.java | 158 ++++++++++++++- .../channel/weixin/WeixinChannelAdapter.java | 180 +++++++++++++++++- 2 files changed, 327 insertions(+), 11 deletions(-) diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java index 68916e96..b9122492 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java @@ -508,6 +508,7 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { List contentParts = new ArrayList<>(); String textContent = null; + boolean hasVoice = false; switch (msgType) { case "text" -> { @@ -534,6 +535,7 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { textContent = "[图片]"; } case "voice" -> { + hasVoice = true; Map voiceBody = (Map) body.getOrDefault("voice", Map.of()); String asrText = ((String) voiceBody.getOrDefault("content", "")).trim(); if (!asrText.isBlank()) { @@ -569,11 +571,27 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { textBuilder.append(txt).append('\n'); } } else if ("image".equals(itemType)) { + // 与独立 image 消息对齐：下载 + AES 解密（对齐 CoPaw） Map img = (Map) item.getOrDefault("image", Map.of()); String url = (String) img.getOrDefault("url", ""); - if (!url.isBlank()) { + String aesKey = (String) img.getOrDefault("aeskey", ""); + if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { + String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "mixed_image.jpg"); + if (localPath != null) { + contentParts.add(MessageContentPart.image(localPath, url)); + } else { + contentParts.add(MessageContentPart.image(url, url)); + } + } else if (!url.isBlank()) { contentParts.add(MessageContentPart.image(url, url)); } + } else if ("voice".equals(itemType)) { + Map v = (Map) item.getOrDefault("voice", Map.of()); + String asrText = ((String) v.getOrDefault("content", "")).trim(); + if (!asrText.isBlank()) { + hasVoice = true; + textBuilder.append(asrText).append('\n'); + } } } textContent = textBuilder.toString().trim(); @@ -621,6 +639,7 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { .content(textContent != null ? textContent.trim() : "") .contentType(msgType) .contentParts(contentParts) + .inputMode(hasVoice ? "voice" : "text") .timestamp(LocalDateTime.now()) .replyToken(isGroup ? chatId : senderId) .rawPayload(Map.of( @@ -734,7 +753,9 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { content, filterThinking, filterToolMessages, format, maxLen); boolean first = true; - for (String segment : segments) { + for (String rawSegment : segments) { + // WeCom 专用：格式化 Markdown 表格（对齐 CoPaw format_markdown_tables） + String segment = formatMarkdownTables(rawSegment); // 第一条分段用 processingStreamId 覆盖"思考中..." if (first && ctx != null && ctx.processingStreamId() != null && !ctx.processingStreamId().isBlank()) { @@ -769,6 +790,14 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { sentText = true; } } + case "refusal" -> { + // 模型拒绝回复（如内容策略限制），以文本形式发送 + String refusalText = part.getText(); + if (refusalText != null && !refusalText.isBlank()) { + sendMessage(targetId, "⚠️ " + refusalText); + sentText = true; + } + } case "image" -> sendImagePart(targetId, part, ctx); case "audio" -> sendAudioPart(targetId, part, ctx); case "file" -> sendFilePart(targetId, part, ctx); @@ -1135,6 +1164,131 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } } + // ==================== Markdown 表格格式化（对齐 CoPaw format_markdown_tables）==================== + + /** + * 格式化 GFM Markdown 表格，使其在企业微信中对齐显示。 + *

+ * 企业微信要求表格列宽一致才能正确渲染。此方法解析表格， + * 计算每列最大宽度，统一填充空格对齐。 + * 代码块内的表格不做处理。 + *

+ * 移植自 CoPaw wecom/utils.py format_markdown_tables() + */ + static String formatMarkdownTables(String text) { + if (text == null || !text.contains("|")) return text; + + String[] lines = text.split("\n", -1); + List result = new ArrayList<>(); + int i = 0; + boolean inCodeFence = false; + + while (i < lines.length) { + String line = lines[i]; + String stripped = line.strip(); + + // 跟踪代码块（``` 内的内容不处理） + if (stripped.startsWith("```")) { + inCodeFence = !inCodeFence; + result.add(line); + i++; + continue; + } + if (inCodeFence) { + result.add(line); + i++; + continue; + } + + // 检测表格开始（含 | 的行） + if (line.contains("|")) { + List tableLines = new ArrayList<>(); + while (i < lines.length && lines[i].contains("|") + && !lines[i].strip().startsWith("```")) { + tableLines.add(lines[i]); + i++; + } + if (!tableLines.isEmpty()) { + result.addAll(formatTable(tableLines)); + } + continue; + } + + result.add(line); + i++; + } + return String.join("\n", result); + } + + /** + * 格式化单个 Markdown 表格 + */ + private static List formatTable(List lines) { + if (lines.isEmpty()) return lines; + + // 检测第二行是否为分隔行（只含 -, :, |, 空格） + boolean hasSeparator = lines.size() >= 2 + && lines.get(1).strip().matches("[\\s\\-:|]+"); + + // 解析单元格（跳过分隔行，后面会重建） + List> rows = new ArrayList<>(); + for (int idx = 0; idx < lines.size(); idx++) { + if (hasSeparator && idx == 1) continue; + String[] cells = lines.get(idx).split("\\|", -1); + List trimmed = new ArrayList<>(); + for (String cell : cells) { + trimmed.add(cell.strip()); + } + // 去掉首尾空元素（由前导/尾随 | 产生） + if (!trimmed.isEmpty() && trimmed.getFirst().isEmpty()) trimmed.removeFirst(); + if (!trimmed.isEmpty() && trimmed.getLast().isEmpty()) trimmed.removeLast(); + if (!trimmed.isEmpty()) rows.add(trimmed); + } + + if (rows.isEmpty()) return lines; + + // 计算每列最大宽度 + int colCount = rows.stream().mapToInt(List::size).max().orElse(0); + int[] widths = new int[colCount]; + for (List row : rows) { + for (int j = 0; j < colCount; j++) { + String cell = j < row.size() ? row.get(j) : ""; + widths[j] = Math.max(widths[j], cell.length()); + } + } + + // 构建格式化结果 + List formatted = new ArrayList<>(); + for (int idx = 0; idx < rows.size(); idx++) { + List row = rows.get(idx); + StringBuilder sb = new StringBuilder("| "); + for (int j = 0; j < colCount; j++) { + String cell = j < row.size() ? row.get(j) : ""; + sb.append(padRight(cell, widths[j])); + if (j < colCount - 1) sb.append(" | "); + } + sb.append(" |"); + formatted.add(sb.toString()); + + // 头部行后插入分隔行 + if (idx == 0) { + StringBuilder sep = new StringBuilder("| "); + for (int j = 0; j < colCount; j++) { + sep.append("-".repeat(Math.max(3, widths[j]))); + if (j < colCount - 1) sep.append(" | "); + } + sep.append(" |"); + formatted.add(sep.toString()); + } + } + return formatted; + } + + private static String padRight(String s, int width) { + if (s.length() >= width) return s; + return s + " ".repeat(width - s.length()); + } + // ==================== 帧发送基础设施 ==================== /** diff --git a/mateclaw-server/src/main/java/vip/mate/channel/weixin/WeixinChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/weixin/WeixinChannelAdapter.java index 4f895ad8..a881deff 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/weixin/WeixinChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/weixin/WeixinChannelAdapter.java @@ -83,6 +83,19 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { /** 用户最新 context_token 缓存（用于主动推送） */ private final ConcurrentHashMap userContextTokens = new ConcurrentHashMap<>(); + /** context_token 持久化文件路径 */ + private Path contextTokensFile; + + /** bot_token 持久化文件路径 */ + private Path botTokenFile; + + /** 文件名扩展名列表（用于过滤纯文件名文本，避免误触发 Agent） */ + private static final Set FILENAME_EXTENSIONS = Set.of( + ".txt", ".doc", ".docx", ".pdf", ".jpg", ".jpeg", ".png", ".gif", + ".mp4", ".avi", ".mov", ".mp3", ".wav", ".zip", ".rar", + ".xlsx", ".xls", ".ppt", ".pptx", ".csv", ".json", ".xml" + ); + // ==================== 输入中提示 ==================== /** 输入提示 ticket 缓存：userId -> (ticket, expireTime) */ @@ -124,7 +137,17 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { @Override protected void doStart() { + // 初始化持久化路径 + String dataDir = getConfigString("data_dir", "data/weixin"); + Path dataDirPath = Path.of(dataDir, String.valueOf(channelEntity.getId())); + botTokenFile = dataDirPath.resolve("bot_token.txt"); + contextTokensFile = dataDirPath.resolve("context_tokens.json"); + + // bot_token 优先级：config > 持久化文件 String botToken = getConfigString("bot_token", ""); + if (botToken.isBlank()) { + botToken = loadBotTokenFromFile(); + } String baseUrl = getConfigString("base_url", ILinkClient.DEFAULT_BASE_URL); if (botToken.isBlank()) { @@ -139,6 +162,12 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { return t; }); + // 加载持久化的 context_tokens（用于重启后主动推送） + loadContextTokens(); + + // 持久化 bot_token（QR 登录后或首次启动时保存） + saveBotTokenToFile(botToken); + // 启动长轮询线程 stopSignal.set(false); cursor = ""; @@ -146,14 +175,19 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { pollThread.setDaemon(true); pollThread.start(); - log.info("[weixin] Channel started: {} (token={}...)", channelEntity.getName(), - botToken.substring(0, Math.min(12, botToken.length()))); + log.info("[weixin] Channel started: {} (token={}..., cached_contexts={})", + channelEntity.getName(), + botToken.substring(0, Math.min(12, botToken.length())), + userContextTokens.size()); } @Override protected void doStop() { stopSignal.set(true); + // 持久化 context_tokens（重启后可恢复主动推送能力） + saveContextTokens(); + // 停止所有输入提示任务 typingTasks.values().forEach(f -> f.cancel(false)); typingTasks.clear(); @@ -275,10 +309,10 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { switch (itemType) { case 1 -> { - // Text + // Text — 过滤纯文件名文本（借鉴 CoPaw: 避免文件名误触发 Agent） Map textItem = (Map) item.getOrDefault("text_item", Map.of()); String text = getStr(textItem, "text").strip(); - if (!text.isEmpty()) { + if (!text.isEmpty() && !isFilenameOnly(text)) { textParts.add(text); } } @@ -313,14 +347,53 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { } case 3 -> { // Voice — 使用 ASR 语音识别文本 + // iLink API 的 ASR 文本可能在两个位置（参考 CoPaw 实现）： + // 路径1: voice_item.text_item.text（嵌套结构） + // 路径2: voice_item.text（直接结构） hasVoice = true; Map voiceItem = (Map) item.getOrDefault("voice_item", Map.of()); - Map voiceTextItem = (Map) voiceItem.getOrDefault("text_item", Map.of()); - String asrText = getStr(voiceTextItem, "text").strip(); + String asrText = ""; + + // 路径1: voice_item → text_item → text + Object textItemObj = voiceItem.get("text_item"); + if (textItemObj instanceof Map textItemMap) { + asrText = getStr((Map) textItemMap, "text").strip(); + } + + // 路径2: voice_item → text（直接字段，CoPaw fallback） + if (asrText.isEmpty()) { + asrText = getStr(voiceItem, "text").strip(); + } + + // 路径3: voice_item → content（与 WeCom 一致的字段名） + if (asrText.isEmpty()) { + asrText = getStr(voiceItem, "content").strip(); + } + + log.debug("[weixin] Voice item payload: {}", voiceItem); + if (!asrText.isEmpty()) { textParts.add(asrText); + log.info("[weixin] Voice ASR text: {}", asrText.length() > 50 + ? asrText.substring(0, 50) + "..." : asrText); } else { - textParts.add("[语音: 无转写结果]"); + // ASR 为空：可能是语音过短、噪音、或 iLink API 字段变更 + // 尝试下载语音文件保存到本地（供后续调试 / 自有 STT 使用） + if (mediaDownloadEnabled) { + String voicePath = downloadMediaItem(item, "voice_item", "voice.amr", mediaDir); + if (voicePath != null) { + // 保存为 audio content part，即使无 ASR 文本 + MessageContentPart audioPart = new MessageContentPart(); + audioPart.setType("audio"); + audioPart.setPath(voicePath); + audioPart.setFileName("voice.amr"); + contentParts.add(audioPart); + log.info("[weixin] Voice audio downloaded (no ASR): {}", voicePath); + } + } + textParts.add("[语音消息]"); + log.warn("[weixin] Voice message with no ASR result. voice_item keys: {}, full: {}", + voiceItem.keySet(), voiceItem); } } case 4 -> { @@ -383,9 +456,13 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { return; } - // 缓存 context_token（用于主动推送） + // 缓存 context_token（用于主动推送）并定期持久化 if (!fromUserId.isBlank() && !contextToken.isBlank()) { - userContextTokens.put(fromUserId, contextToken); + String prev = userContextTokens.put(fromUserId, contextToken); + // token 变更时才持久化（减少 I/O） + if (!contextToken.equals(prev)) { + saveContextTokens(); + } } // 构建统一消息 @@ -842,4 +919,89 @@ public class WeixinChannelAdapter extends AbstractChannelAdapter { } return data; } + + // ==================== Token 持久化（对齐 CoPaw）==================== + + /** + * 从文件加载 bot_token（启动时如果 config 中无 token，尝试从文件恢复） + */ + private String loadBotTokenFromFile() { + if (botTokenFile == null) return ""; + try { + if (Files.exists(botTokenFile)) { + String token = Files.readString(botTokenFile).strip(); + if (!token.isBlank()) { + log.info("[weixin] Loaded bot_token from {}", botTokenFile); + return token; + } + } + } catch (Exception e) { + log.debug("[weixin] Failed to read bot_token file: {}", e.getMessage()); + } + return ""; + } + + /** + * 持久化 bot_token 到文件（QR 登录后或首次启动时保存） + */ + private void saveBotTokenToFile(String token) { + if (botTokenFile == null || token == null || token.isBlank()) return; + try { + Files.createDirectories(botTokenFile.getParent()); + Files.writeString(botTokenFile, token); + log.info("[weixin] Bot token saved to {}", botTokenFile); + } catch (Exception e) { + log.warn("[weixin] Failed to save bot_token file: {}", e.getMessage()); + } + } + + /** + * 从文件加载 context_tokens（启动时恢复主动推送能力） + */ + @SuppressWarnings("unchecked") + private void loadContextTokens() { + if (contextTokensFile == null) return; + try { + if (Files.exists(contextTokensFile)) { + String json = Files.readString(contextTokensFile); + Map data = objectMapper.readValue(json, + objectMapper.getTypeFactory().constructMapType(HashMap.class, String.class, String.class)); + if (data != null && !data.isEmpty()) { + userContextTokens.putAll(data); + log.info("[weixin] Loaded {} context_tokens from {}", data.size(), contextTokensFile); + } + } + } catch (Exception e) { + log.debug("[weixin] Failed to load context_tokens: {}", e.getMessage()); + } + } + + /** + * 持久化 context_tokens 到文件（停止时保存 + token 变更时保存） + */ + private void saveContextTokens() { + if (contextTokensFile == null || userContextTokens.isEmpty()) return; + try { + Files.createDirectories(contextTokensFile.getParent()); + Files.writeString(contextTokensFile, + objectMapper.writeValueAsString(new HashMap<>(userContextTokens))); + } catch (Exception e) { + log.debug("[weixin] Failed to save context_tokens: {}", e.getMessage()); + } + } + + // ==================== 文件名过滤（对齐 CoPaw）==================== + + /** + * 判断文本是否仅为文件名（如 "photo.jpg"、"report.pdf"）。 + * 微信发送文件时会同时发一条文本消息包含文件名，这不应触发 Agent 回复。 + * 参考 CoPaw channel.py:538-566 + */ + private static boolean isFilenameOnly(String text) { + if (text == null || text.isBlank()) return false; + // 文件名不应包含换行（多行文本不是纯文件名） + if (text.contains("\n")) return false; + String lower = text.strip().toLowerCase(); + return FILENAME_EXTENSIONS.stream().anyMatch(lower::endsWith); + } }