From 134fa1a975612cca109f2d54a462ec9ea7dcdc0c Mon Sep 17 00:00:00 2001 From: matevip Date: Sun, 10 May 2026 08:28:14 +0800 Subject: [PATCH] fix(channel): wecom file pipeline + IM approval hint visibility --- .../java/vip/mate/channel/ChannelManager.java | 3 +- .../mate/channel/ChannelMessageRouter.java | 77 ++- .../channel/wecom/WeComChannelAdapter.java | 589 +++++++++++++++++- 3 files changed, 634 insertions(+), 35 deletions(-) diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java index a4fea314..5130c159 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java @@ -481,7 +481,8 @@ public class ChannelManager { case "telegram" -> new TelegramChannelAdapter(channel, messageRouter, objectMapper); case "discord" -> new DiscordChannelAdapter(channel, messageRouter, objectMapper); case "wecom" -> new WeComChannelAdapter(channel, messageRouter, objectMapper, - approvalNotificationService, weComCardDispatcher, weComKeepaliveScheduler); + approvalNotificationService, weComCardDispatcher, weComKeepaliveScheduler, + generatedFileCache); case "qq" -> new QQChannelAdapter(channel, messageRouter, objectMapper); case "weixin" -> new WeixinChannelAdapter(channel, messageRouter, objectMapper); case "slack" -> new vip.mate.channel.slack.SlackChannelAdapter(channel, messageRouter, objectMapper); diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java index 802db239..ecff71fd 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java @@ -469,7 +469,10 @@ public class ChannelMessageRouter { ResolveOutcome denyOutcome = approvalService.resolve( pending.getPendingId(), message.getSenderId(), "denied"); conversationService.removeApprovalPlaceholders(conversationId); - adapter.sendMessage(replyTarget, "⛔ 已拒绝执行工具: " + pending.getToolName()); + String denyHint = "⛔ 已拒绝执行工具: " + pending.getToolName(); + persistAndBroadcastApprovalHint(conversationId, denyHint, + "denied", pending.getPendingId(), pending.getToolName()); + adapter.sendMessage(replyTarget, denyHint); log.info("[{}] Approval DENIED via IM command: pendingId={}, tool={}, msgRewritten={}", adapter.getChannelType(), pending.getPendingId(), pending.getToolName(), denyOutcome.messagesRewritten()); @@ -479,7 +482,10 @@ public class ChannelMessageRouter { // Non-approval message while a pending exists → treat as implicit deny. approvalService.resolve(pending.getPendingId(), message.getSenderId(), "denied"); conversationService.removeApprovalPlaceholders(conversationId); - adapter.sendMessage(replyTarget, "⛔ 审批已取消。将继续处理您的新消息。"); + String cancelHint = "⛔ 审批已取消。将继续处理您的新消息。"; + persistAndBroadcastApprovalHint(conversationId, cancelHint, + "cancelled", pending.getPendingId(), pending.getToolName()); + adapter.sendMessage(replyTarget, cancelHint); log.info("[{}] Approval auto-cancelled (non-approval message): pendingId={}", adapter.getChannelType(), pending.getPendingId()); // Fall through to process the new message normally. @@ -763,8 +769,14 @@ public class ChannelMessageRouter { String replyTarget = resolveReplyTarget(triggerMessage); Long agentId = channelEntity.getAgentId(); - // 通知用户审批已通过 - adapter.sendMessage(replyTarget, "✅ 已批准执行工具: " + consumed.getToolName()); + // Notify the user that the approval went through. Persist + broadcast so a + // Web mirror of the same conversationId sees the resolution; otherwise this + // hint would only land in the IM channel and the Web admin console would + // show the replay reply with no preceding "approved" marker. + String approveHint = "✅ 已批准执行工具: " + consumed.getToolName(); + persistAndBroadcastApprovalHint(conversationId, approveHint, + "approved", consumed.getPendingId(), consumed.getToolName()); + adapter.sendMessage(replyTarget, approveHint); // 清理 DB 中残留的审批占位消息 conversationService.removeApprovalPlaceholders(conversationId); @@ -800,7 +812,62 @@ public class ChannelMessageRouter { adapter.getChannelType(), consumed.getToolName(), reply.length()); } catch (Exception e) { log.error("[approval-replay] Replay failed: {}", e.getMessage(), e); - adapter.sendMessage(replyTarget, "❌ 工具执行失败: " + e.getMessage()); + String errHint = "❌ 工具执行失败: " + e.getMessage(); + persistAndBroadcastApprovalHint(conversationId, errHint, null, null, null); + adapter.sendMessage(replyTarget, errHint); + } + } + + /** + * Persist an approval-related hint as an assistant message and best-effort + * broadcast it to any live SSE viewer of the conversation. + *

+ * Without this, IM-driven approve/deny only reaches the originating IM + * channel via {@code adapter.sendMessage(...)} — a Web mirror of the same + * conversationId has no record of the resolution because nothing lands in + * {@code mate_message} and no SSE event is emitted. The hint then "vanishes" + * from the Web admin console even though it shows up on the user's phone. + *

+ * Persistence is the load-bearing fix (Web reload picks it up). Broadcast + * is best-effort: if no SSE stream is currently registered for the + * conversation, the broadcast no-ops silently — that's the common case + * since IM-driven clicks rarely race with an active web subscriber. + * + * @param conversationId conversation owning the hint + * @param hint text to render as an assistant bubble + * @param decision "approved" / "denied" / "cancelled" / null (skips the + * structured resolved event when null, e.g. on replay error) + * @param pendingId pending approval id; null when not applicable + * @param toolName tool name for the structured event; null when not applicable + */ + private void persistAndBroadcastApprovalHint(String conversationId, String hint, + String decision, String pendingId, + String toolName) { + try { + conversationService.saveMessage(conversationId, "assistant", hint, null, "completed"); + } catch (Exception e) { + log.warn("[approval-hint] saveMessage failed for conv={}: {}", + conversationId, e.getMessage()); + } + try { + if (decision != null) { + streamTracker.broadcastObject(conversationId, "tool_approval_resolved", Map.of( + "pendingId", pendingId == null ? "" : pendingId, + "decision", decision, + "toolName", toolName == null ? "" : toolName, + "timestamp", System.currentTimeMillis() + )); + } + streamTracker.broadcastObject(conversationId, "message_start", + Map.of("role", "assistant")); + streamTracker.broadcastObject(conversationId, "content_delta", + Map.of("delta", hint)); + streamTracker.broadcastObject(conversationId, "message_complete", + Map.of("status", "completed")); + } catch (Exception e) { + // Broadcast is best-effort; a missing run state is the common case. + log.debug("[approval-hint] broadcast skipped/failed for conv={}: {}", + conversationId, e.getMessage()); } } diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java index 96338fdc..a92af82e 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java @@ -12,6 +12,7 @@ import vip.mate.workspace.conversation.model.MessageContentPart; import javax.crypto.Cipher; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; @@ -26,6 +27,8 @@ import java.time.Duration; import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -240,16 +243,40 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { */ private final WeComKeepaliveScheduler keepaliveScheduler; + /** + * In-memory cache of bytes generated by tools like + * {@code DocxRenderTool} / {@code PptxRenderTool}. The agent emits a + * {@code /api/v1/files/generated/{id}} URL referencing this cache; the + * channel layer resolves that URL back to bytes and uploads them as a + * native WeCom file message so the user actually receives a tappable + * document instead of an unopenable link. Null-tolerant: if missing + * (older constructor / test DI gap), URL stays inline as plain markdown + * which renders as a non-interactive link in the bubble. + */ + private final vip.mate.tool.document.GeneratedFileCache generatedFileCache; + public WeComChannelAdapter(ChannelEntity channelEntity, ChannelMessageRouter messageRouter, ObjectMapper objectMapper, vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService, vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher, WeComKeepaliveScheduler keepaliveScheduler) { + this(channelEntity, messageRouter, objectMapper, approvalNotificationService, + cardDispatcher, keepaliveScheduler, null); + } + + public WeComChannelAdapter(ChannelEntity channelEntity, + ChannelMessageRouter messageRouter, + ObjectMapper objectMapper, + vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService, + vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher, + WeComKeepaliveScheduler keepaliveScheduler, + vip.mate.tool.document.GeneratedFileCache generatedFileCache) { super(channelEntity, messageRouter, objectMapper); this.approvalNotificationService = approvalNotificationService; this.cardDispatcher = cardDispatcher; this.keepaliveScheduler = keepaliveScheduler; + this.generatedFileCache = generatedFileCache; // Default to 8 bounded attempts (~4 minutes total at 2s..30s exponential) // so the UI eventually settles in ERROR instead of getting stuck in // RECONNECTING forever. User config still overrides (-1 = infinite). @@ -895,15 +922,9 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map imgBody = (Map) body.getOrDefault("image", Map.of()); String url = (String) imgBody.getOrDefault("url", ""); String aesKey = (String) imgBody.getOrDefault("aeskey", ""); - if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { - String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "image.jpg"); - if (localPath != null) { - contentParts.add(MessageContentPart.image(localPath, url)); - } else { - contentParts.add(MessageContentPart.image(url, url)); - } - } else if (!url.isBlank()) { - contentParts.add(MessageContentPart.image(url, url)); + String inboundConvId = inboundConversationId(senderId, chatId, chatType); + if (!url.isBlank()) { + contentParts.add(buildInboundImagePart(url, aesKey, msgId, "image.jpg", inboundConvId)); } textContent = "[图片]"; } @@ -922,11 +943,23 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map fileBody = (Map) body.getOrDefault("file", Map.of()); String url = (String) fileBody.getOrDefault("url", ""); String aesKey = (String) fileBody.getOrDefault("aeskey", ""); - String filename = (String) fileBody.getOrDefault("filename", "file.bin"); - if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { - String localPath = downloadAndDecryptMedia(url, aesKey, msgId, filename); - if (localPath != null) { - contentParts.add(MessageContentPart.file(localPath, filename, null)); + // WeCom sometimes omits filename for forwarded files. Try a + // few fallback keys before giving up to "file.bin"; the + // magic-byte sniffer in downloadInboundMedia will fix the + // extension either way, but having something user-readable + // here keeps the bubble title meaningful. + String filename = (String) fileBody.getOrDefault("filename", + fileBody.getOrDefault("file_name", + fileBody.getOrDefault("name", "file.bin"))); + String fileConvId = inboundConversationId(senderId, chatId, chatType); + if (!url.isBlank()) { + MessageContentPart filePart = buildInboundFilePart( + url, aesKey, msgId, filename, fileConvId); + contentParts.add(filePart); + // Surface the corrected filename (with proper extension) + // back into the [文件: X] text marker the agent sees. + if (filePart.getFileName() != null && !filePart.getFileName().isBlank()) { + filename = filePart.getFileName(); } } textContent = "[文件: " + filename + "]"; @@ -948,15 +981,10 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map img = (Map) item.getOrDefault("image", Map.of()); String url = (String) img.getOrDefault("url", ""); String aesKey = (String) img.getOrDefault("aeskey", ""); - if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { - String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "mixed_image.jpg"); - if (localPath != null) { - contentParts.add(MessageContentPart.image(localPath, url)); - } else { - contentParts.add(MessageContentPart.image(url, url)); - } - } else if (!url.isBlank()) { - contentParts.add(MessageContentPart.image(url, url)); + String mixedConvId = inboundConversationId(senderId, chatId, chatType); + if (!url.isBlank()) { + contentParts.add(buildInboundImagePart( + url, aesKey, msgId, "mixed_image.jpg", mixedConvId)); } } else if ("voice".equals(itemType)) { Map v = (Map) item.getOrDefault("voice", Map.of()); @@ -1243,6 +1271,15 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { keepaliveScheduler.stop(ctx.processingStreamId()); } + // Sniff `/api/v1/files/generated/{id}` URLs out of the agent's text + // BEFORE rendering. Each hit gets upgraded to a native WeCom file + // message via the chunked upload API; the URL in the text is replaced + // with a "📎 filename" marker so the bubble doesn't repeat itself. + // Without this, a generated docx/pptx would arrive as a markdown link + // the user can't open inside WeCom (no public access + JWT required). + List uploadJobs = new ArrayList<>(); + String rewrittenContent = sniffGeneratedFiles(content, uploadJobs); + // 先进行正常的内容渲染（过滤 thinking、分割长文本） boolean filterThinking = getConfigBoolean("filter_thinking", true); boolean filterToolMessages = getConfigBoolean("filter_tool_messages", true); @@ -1250,7 +1287,7 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { int maxLen = vip.mate.channel.ChannelMessageRenderer.PLATFORM_LIMITS.getOrDefault(getChannelType(), 2048); List segments = vip.mate.channel.ChannelMessageRenderer.renderForChannel( - content, filterThinking, filterToolMessages, format, maxLen); + rewrittenContent, filterThinking, filterToolMessages, format, maxLen); boolean first = true; for (String rawSegment : segments) { @@ -1265,6 +1302,89 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { sendMessage(targetId, segment); } } + + // Upload + dispatch any generated files we sniffed out. Done after the + // text bubble so the order in the IM client mirrors the markdown: + // explanatory text first, then the actual file card the user can tap. + // Use the original frameReqId once for the first attachment (so it + // rides the reply path) and active-push for the rest. + if (!uploadJobs.isEmpty()) { + String frameReqId = ctx != null ? ctx.frameReqId() : null; + for (int i = 0; i < uploadJobs.size(); i++) { + UploadJob job = uploadJobs.get(i); + String mediaId = uploadMedia(job.bytes(), job.fileName(), job.mediaType()); + if (mediaId == null) { + log.warn("[wecom] Generated-file upload failed: {} ({} bytes)", + job.fileName(), job.bytes().length); + continue; + } + // Only the first attachment can use the inbound frameReqId + // reply slot; subsequent attachments must go via active-push. + String replyReqId = (i == 0) ? frameReqId : null; + sendMediaMessage(targetId, mediaId, job.mediaType(), replyReqId); + } + } + } + + /** Carries one to-be-uploaded generated file from {@link #sniffGeneratedFiles}. */ + private record UploadJob(byte[] bytes, String fileName, String mediaType) {} + + /** + * URL pattern for the in-memory generated-file cache served by + * {@code GeneratedFileController}. Lives in the channel layer because + * each adapter rewrites the URL to a channel-native attachment. + */ + private static final java.util.regex.Pattern GENERATED_URL_PATTERN = + java.util.regex.Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)"); + + /** + * Scan the agent's text for {@code /api/v1/files/generated/{id}} URLs; + * for each hit, look up the cached bytes and queue an {@link UploadJob}. + * Replaces the URL in the returned text with a "📎 filename" marker so + * the bubble shows the file name without dangling an unopenable link. + * Cache misses (entry expired or never existed) leave the URL untouched + * — the user can still try clicking from the Web mirror's history view. + */ + private String sniffGeneratedFiles(String text, List jobs) { + if (text == null || text.isEmpty() || generatedFileCache == null) return text; + java.util.regex.Matcher m = GENERATED_URL_PATTERN.matcher(text); + StringBuilder out = new StringBuilder(); + while (m.find()) { + String id = m.group(1); + var entry = generatedFileCache.get(id).orElse(null); + if (entry != null) { + String mediaType = isImageMime(entry.mimeType()) ? "image" : "file"; + jobs.add(new UploadJob(entry.bytes(), entry.filename(), mediaType)); + m.appendReplacement(out, + java.util.regex.Matcher.quoteReplacement("📎 " + entry.filename())); + } else { + // Cache miss has two real-world causes, both surfaced with + // the same retry hint so the user just resubmits: + // 1) LLM hallucinated a UUID-shaped string instead of + // calling a render tool — IDs like + // "a1b2c3d4-e5f6-7890-abcd-ef1234567890" with sequential + // hex are textbook fakes. {@code GeneratedFileCache} + // logs every real {@code put}, so its absence here is + // proof the file was never generated this turn. + // 2) Cache entry expired (10-min TTL) before the IM + // client got around to clicking, or was wiped on + // JVM restart. + // Without this replacement, users tap a markdown link that + // returns 404 and the IM client saves the error body as a + // ".docx" — they then "open" what is actually an HTML 404 + // page and report "file is corrupted". + log.warn("[wecom] Generated-file cache miss for id={} — likely LLM skipped the render tool and wrote a fake URL (toolCallCount=0 in this turn). Bubble will show retry hint.", + id); + m.appendReplacement(out, java.util.regex.Matcher.quoteReplacement( + "⚠️ 文件未真正生成（模型未调用文档生成工具），请重新发送请求")); + } + } + m.appendTail(out); + return out.toString(); + } + + private static boolean isImageMime(String mimeType) { + return mimeType != null && mimeType.toLowerCase().startsWith("image/"); } @Override @@ -1276,19 +1396,28 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { boolean sentText = false; boolean firstText = true; + // Mirror renderAndSend's sniff so contentParts-mode replies also + // upgrade /api/v1/files/generated/{id} URLs into native WeCom file + // attachments. Text parts get the URL replaced with "📎 filename"; + // the actual bytes are queued for native upload after all parts are + // dispatched (preserves the "text first, attachments after" ordering). + List uploadJobs = new ArrayList<>(); + for (MessageContentPart part : parts) { if (part == null) continue; try { switch (part.getType()) { case "text" -> { - if (part.getText() != null && !part.getText().isBlank()) { + String txt = part.getText(); + if (txt != null && !txt.isBlank()) { + String rewritten = sniffGeneratedFiles(txt, uploadJobs); // 第一条文本用 processingStreamId 覆盖"思考中..." if (firstText && ctx != null && ctx.processingStreamId() != null && !ctx.processingStreamId().isBlank()) { - replyStream(ctx.frameReqId(), ctx.processingStreamId(), part.getText(), true); + replyStream(ctx.frameReqId(), ctx.processingStreamId(), rewritten, true); firstText = false; } else { - sendMessage(targetId, part.getText()); + sendMessage(targetId, rewritten); } sentText = true; } @@ -1326,6 +1455,26 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { log.debug("[wecom] Failed to clear processing indicator: {}", e.getMessage()); } } + + // After all parts are dispatched, upload any generated files we + // sniffed out of text parts. First attachment rides the inbound + // frameReqId reply slot; subsequent ones go via active-push. Mirror + // the ordering used in renderAndSend so the user sees text bubble + // first, then the actual file card. + if (!uploadJobs.isEmpty()) { + String frameReqId = ctx != null ? ctx.frameReqId() : null; + for (int i = 0; i < uploadJobs.size(); i++) { + UploadJob job = uploadJobs.get(i); + String mediaId = uploadMedia(job.bytes(), job.fileName(), job.mediaType()); + if (mediaId == null) { + log.warn("[wecom] Generated-file upload failed: {} ({} bytes)", + job.fileName(), job.bytes().length); + continue; + } + String replyReqId = (i == 0) ? frameReqId : null; + sendMediaMessage(targetId, mediaId, job.mediaType(), replyReqId); + } + } } /** @@ -1747,7 +1896,13 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map chunkBody = new LinkedHashMap<>(); chunkBody.put("upload_id", uploadId); chunkBody.put("chunk_index", i); - chunkBody.put("data", base64Data); + // Field name MUST be "base64_data" — the WeCom AI bot upload + // server reads the chunk bytes from this exact key. A previous + // version sent "data" which the server silently dropped, so + // metadata (filename/size) committed but the bytes never made + // it to storage. Receivers then saw the file with the right + // name/size but couldn't open it ("文件已损坏"). + chunkBody.put("base64_data", base64Data); Map chunkFrame = Map.of( "cmd", CMD_UPLOAD_CHUNK, @@ -2089,7 +2244,383 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { // ==================== 媒体文件下载与 AES 解密 ==================== /** - * 下载并解密企业微信媒体文件 + * Conversation id for inbound media: matches the format + * {@code ChannelMessageRouter.buildConversationId} produces from the same + * (channelType, chatId, senderId) tuple. Pre-computing it here lets the + * media-download helper write into the right per-conversation directory + * before the {@link ChannelMessage} is built. + */ + private static String inboundConversationId(String senderId, String chatId, String chatType) { + boolean isGroup = "group".equals(chatType); + return isGroup ? "wecom:group:" + chatId : "wecom:" + senderId; + } + + /** + * Build a fully-populated image content part for inbound WeCom media. + *

+ * When download is enabled and succeeds, the part carries + * {@code path}, {@code fileUrl} (browser-servable), {@code fileName}, + * {@code storedName}, {@code fileSize}, and a precise {@code contentType} + * — every field the chat bubble and the multimodal sidecar inspect. When + * download is disabled or fails, the part falls back to URL-only fields + * but still sets {@code fileName} so the bubble doesn't render + * "未命名 / unknown". + */ + private MessageContentPart buildInboundImagePart(String url, String aesKey, String msgId, + String fileNameHint, String conversationId) { + if (getConfigBoolean("media_download_enabled", true)) { + InboundMediaResult r = downloadInboundMedia(url, aesKey, msgId, fileNameHint, conversationId); + if (r != null) { + MessageContentPart part = new MessageContentPart(); + part.setType("image"); + part.setFileName(r.fileName()); + part.setStoredName(r.storedName()); + part.setPath(r.localPath()); + part.setFileUrl(r.fileUrl()); + part.setFileSize(r.fileSize()); + // Prefer the sniffed contentType (could be image/png) over a + // hardcoded image/jpeg. Falls back to image/jpeg only when the + // sniff was inconclusive. + String ct = r.contentType(); + part.setContentType((ct != null && ct.startsWith("image/")) ? ct : "image/jpeg"); + // mediaId mirrors path so callers that prefer it still resolve + // to the same on-disk file (matches Web upload's behaviour). + part.setMediaId(r.localPath()); + return part; + } + } + // Fallback: download disabled or failed. Browser preview will be broken + // because the WeCom CDN URL carries a short-lived signature, but at + // least the bubble shows "image.jpg" instead of "未命名 / unknown". + MessageContentPart part = new MessageContentPart(); + part.setType("image"); + part.setFileName(fileNameHint); + part.setFileUrl(url); + part.setMediaId(url); + part.setContentType("image/jpeg"); + return part; + } + + /** + * Build a fully-populated file content part for inbound WeCom media. + *

+ * Mirrors {@link #buildInboundImagePart} but for non-image attachments + * (PDF, DOCX, ZIP, etc.). The magic-byte sniffer inside + * {@link #downloadInboundMedia} fixes generic {@code file.bin} hints to + * the real extension so downstream tools (PDF text extractor, magika, + * etc.) key off the correct mime. + */ + private MessageContentPart buildInboundFilePart(String url, String aesKey, String msgId, + String fileNameHint, String conversationId) { + if (getConfigBoolean("media_download_enabled", true)) { + InboundMediaResult r = downloadInboundMedia(url, aesKey, msgId, fileNameHint, conversationId); + if (r != null) { + MessageContentPart part = new MessageContentPart(); + part.setType("file"); + part.setFileName(r.fileName()); + part.setStoredName(r.storedName()); + part.setPath(r.localPath()); + part.setFileUrl(r.fileUrl()); + part.setFileSize(r.fileSize()); + part.setContentType(r.contentType()); + part.setMediaId(r.localPath()); + return part; + } + } + // Fallback when download is disabled or fails — at least keep the + // original hint so the bubble doesn't say "file.bin" for a PDF. + MessageContentPart part = new MessageContentPart(); + part.setType("file"); + part.setFileName(fileNameHint); + part.setFileUrl(url); + part.setMediaId(url); + return part; + } + + /** + * Inbound-media download result. Carries every field the bubble renderer + * and the multimodal sidecar need so callers don't have to re-derive + * storedName / fileUrl from scratch. + * + * @param localPath absolute filesystem path of the saved file + * @param storedName the on-disk filename (matches the last segment of localPath) + * @param fileUrl browser-servable URL: {@code /api/v1/chat/files/{convId}/{storedName}} + * @param fileSize byte length after decryption + * @param fileName human-readable display name (extension corrected by magic-byte sniff) + * @param contentType MIME type derived from magic bytes (or {@code application/octet-stream}) + */ + record InboundMediaResult(String localPath, String storedName, + String fileUrl, long fileSize, String fileName, + String contentType) {} + + /** Magic-byte sniff result. */ + private record MagicSniff(String extension, String contentType) { + static final MagicSniff UNKNOWN = new MagicSniff(".bin", "application/octet-stream"); + } + + /** + * Best-effort MIME sniff from the first 12 bytes of a file. Covers the + * formats users routinely forward to bots (PDF, Office, archives, common + * image / audio / video). When nothing matches, returns + * {@link MagicSniff#UNKNOWN} so the caller falls back to {@code .bin}. + *

+ * This exists because WeCom's {@code aibot_msg_callback} {@code file} + * body sometimes omits {@code filename} entirely (forwarded files in + * particular), and shipping the agent a part labelled {@code file.bin} + * makes downstream tools mis-route the content. Sniffing recovers a + * useful extension so PDF tools fire on PDFs. + */ + private static MagicSniff sniffMagic(byte[] head) { + if (head == null || head.length < 4) return MagicSniff.UNKNOWN; + // PDF: %PDF + if (head[0] == 0x25 && head[1] == 0x50 && head[2] == 0x44 && head[3] == 0x46) { + return new MagicSniff(".pdf", "application/pdf"); + } + // PNG: 89 50 4E 47 + if (head[0] == (byte) 0x89 && head[1] == 0x50 && head[2] == 0x4E && head[3] == 0x47) { + return new MagicSniff(".png", "image/png"); + } + // JPEG: FF D8 FF + if (head[0] == (byte) 0xFF && head[1] == (byte) 0xD8 && head[2] == (byte) 0xFF) { + return new MagicSniff(".jpg", "image/jpeg"); + } + // GIF: "GIF8" + if (head[0] == 0x47 && head[1] == 0x49 && head[2] == 0x46 && head[3] == 0x38) { + return new MagicSniff(".gif", "image/gif"); + } + // ZIP-based container: PK\x03\x04. Could be a plain ZIP, a JAR, + // an OOXML document (DOCX/XLSX/PPTX), an ODF document (ODT/ODS/ODP), + // or an EPUB. Magic-byte alone can't tell them apart — caller is + // expected to follow up with refineZipKind(fullBytes) to pick a + // specific type. + if (head[0] == 0x50 && head[1] == 0x4B && head[2] == 0x03 && head[3] == 0x04) { + return new MagicSniff(".zip", "application/zip"); + } + // Legacy Office (DOC/XLS/PPT): D0 CF 11 E0 A1 B1 1A E1 + if (head.length >= 8 + && head[0] == (byte) 0xD0 && head[1] == (byte) 0xCF + && head[2] == 0x11 && head[3] == (byte) 0xE0 + && head[4] == (byte) 0xA1 && head[5] == (byte) 0xB1 + && head[6] == 0x1A && head[7] == (byte) 0xE1) { + return new MagicSniff(".doc", "application/msword"); + } + // RTF: "{\rtf" + if (head.length >= 5 + && head[0] == 0x7B && head[1] == 0x5C + && head[2] == 0x72 && head[3] == 0x74 && head[4] == 0x66) { + return new MagicSniff(".rtf", "application/rtf"); + } + // 7z: 37 7A BC AF 27 1C + if (head.length >= 6 + && head[0] == 0x37 && head[1] == 0x7A && head[2] == (byte) 0xBC + && head[3] == (byte) 0xAF && head[4] == 0x27 && head[5] == 0x1C) { + return new MagicSniff(".7z", "application/x-7z-compressed"); + } + // RAR: "Rar!\x1A\x07" + if (head.length >= 6 + && head[0] == 0x52 && head[1] == 0x61 && head[2] == 0x72 + && head[3] == 0x21 && head[4] == 0x1A && head[5] == 0x07) { + return new MagicSniff(".rar", "application/x-rar-compressed"); + } + // MP3: ID3v2 ("ID3") or MPEG sync 0xFFFB / 0xFFF3 / 0xFFF2 + if (head[0] == 0x49 && head[1] == 0x44 && head[2] == 0x33) { + return new MagicSniff(".mp3", "audio/mpeg"); + } + // MP4: "....ftyp" — bytes 4..7 == "ftyp" + if (head.length >= 8 + && head[4] == 0x66 && head[5] == 0x74 && head[6] == 0x79 && head[7] == 0x70) { + return new MagicSniff(".mp4", "video/mp4"); + } + // OGG: "OggS" + if (head[0] == 0x4F && head[1] == 0x67 && head[2] == 0x67 && head[3] == 0x53) { + return new MagicSniff(".ogg", "audio/ogg"); + } + return MagicSniff.UNKNOWN; + } + + /** + * Peek inside a ZIP container to distinguish OOXML (DOCX/XLSX/PPTX), + * ODF (ODT/ODS/ODP), JAR, and EPUB from a plain ZIP. Reads the local + * file headers in order via {@link ZipInputStream}; the discriminator + * entry is almost always within the first few entries (OOXML places + * {@code [Content_Types].xml} first, ODF places {@code mimetype} first), + * so we cap iteration at 16 entries to bound CPU. + *

+ * Returns the original {@code zipDefault} sniff (plain + * {@code application/zip}) when no specific kind is detected — that's + * the right answer for actual ZIPs and unknown archive formats. + */ + private static MagicSniff refineZipKind(byte[] fileData, MagicSniff zipDefault) { + if (fileData == null || fileData.length < 30) return zipDefault; + String mimetypeContent = null; + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(fileData))) { + ZipEntry entry; + int seen = 0; + while ((entry = zis.getNextEntry()) != null && seen < 16) { + String name = entry.getName(); + // OOXML — Office Open XML (Word/Excel/PowerPoint). Each format + // has a distinct top-level directory; we match on prefix + // because the entry order isn't guaranteed. + if (name.startsWith("word/")) { + return new MagicSniff(".docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + } + if (name.startsWith("xl/")) { + return new MagicSniff(".xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + } + if (name.startsWith("ppt/")) { + return new MagicSniff(".pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation"); + } + // Visio (rare but worth catching) + if (name.startsWith("visio/")) { + return new MagicSniff(".vsdx", + "application/vnd.ms-visio.drawing"); + } + // ODF marker: a {@code mimetype} entry that contains the full + // application/vnd.oasis.opendocument.* string — read its body + // and decide once we have it. + if ("mimetype".equals(name)) { + byte[] buf = zis.readAllBytes(); + mimetypeContent = new String(buf, java.nio.charset.StandardCharsets.UTF_8).trim(); + } + // JAR + if ("META-INF/MANIFEST.MF".equals(name)) { + return new MagicSniff(".jar", "application/java-archive"); + } + // EPUB always has META-INF/container.xml + if ("META-INF/container.xml".equals(name)) { + return new MagicSniff(".epub", "application/epub+zip"); + } + seen++; + } + } catch (Exception e) { + log.debug("[wecom] refineZipKind failed (treating as plain zip): {}", e.getMessage()); + return zipDefault; + } + if (mimetypeContent != null) { + if (mimetypeContent.contains("opendocument.text")) { + return new MagicSniff(".odt", "application/vnd.oasis.opendocument.text"); + } + if (mimetypeContent.contains("opendocument.spreadsheet")) { + return new MagicSniff(".ods", "application/vnd.oasis.opendocument.spreadsheet"); + } + if (mimetypeContent.contains("opendocument.presentation")) { + return new MagicSniff(".odp", "application/vnd.oasis.opendocument.presentation"); + } + if (mimetypeContent.contains("epub")) { + return new MagicSniff(".epub", "application/epub+zip"); + } + } + return zipDefault; + } + + /** + * Strip a trailing extension from a filename. {@code "image.jpg" → "image"}; + * {@code "no_ext" → "no_ext"}; {@code "" → ""}. + */ + private static String stripExtension(String name) { + if (name == null || name.isBlank()) return ""; + int dot = name.lastIndexOf('.'); + if (dot <= 0) return name; + return name.substring(0, dot); + } + + /** + * Download + decrypt an inbound media attachment and stash it under + * {@code data/chat-uploads/{conversationId}/} so the existing + * {@code /api/v1/chat/files/...} endpoint can serve it back to the chat + * bubble. Returns a fully-populated {@link InboundMediaResult} on success + * or null on download/decrypt failure (callers fall back to URL-only). + *

+ * Storing under chat-uploads rather than {@code data/media} means + * {@link MessageContentPart#getPath()} resolves to a real file for the + * vision sidecar AND {@code fileUrl} renders as a thumbnail in the Web + * mirror — instead of the WeCom-signed CDN URL whose 5-minute query-string + * signature expires before the browser can fetch it. + */ + private InboundMediaResult downloadInboundMedia(String url, String aesKey, String msgId, + String fileNameHint, String conversationId) { + try { + // Mirror ChatController.uploadRoot ("data/chat-uploads") so the + // serve endpoint at /api/v1/chat/files/{convId}/{storedName} works + // without any extra wiring. The conversationId may contain ':' + // (e.g. "wecom:XuZhanFu" or "wecom:group:abc"); Path resolution + // tolerates this on macOS/Linux but Windows would reject the + // colon — for now we keep parity with the existing chat-uploads + // layout and revisit if Windows support comes up. + Path uploadDir = Path.of("data", "chat-uploads", conversationId); + Files.createDirectories(uploadDir); + + // 1. HTTP GET 下载文件 + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + byte[] encryptedData = response.body().readAllBytes(); + + byte[] fileData; + // 2. AES 解密（如果提供了 aesKey） + if (aesKey != null && !aesKey.isBlank()) { + fileData = decryptAes256Cbc(encryptedData, aesKey); + } else { + fileData = encryptedData; + } + + // 3. Magic-byte sniff to recover a real extension when WeCom + // didn't include filename in the body (forwarded files often + // arrive nameless — saving them as "file.bin" misroutes the + // agent because every PDF tool keys off the .pdf extension). + byte[] head = new byte[Math.min(12, fileData.length)]; + System.arraycopy(fileData, 0, head, 0, head.length); + MagicSniff sniff = sniffMagic(head); + // ZIP container needs a deeper look — DOCX/XLSX/PPTX/ODF/EPUB/JAR + // all share the PK\x03\x04 magic. Peek inside the first few + // entries to pick the specific kind. + if (".zip".equals(sniff.extension())) { + sniff = refineZipKind(fileData, sniff); + } + + // 4. Compose a URL-safe storedName. If the hint is generic + // (e.g. "file.bin"), prefer the sniffed extension. + String urlHash = md5Hex(url).substring(0, 8); + String hintRaw = (fileNameHint == null ? "media" : fileNameHint).trim(); + String safeName = hintRaw.replaceAll("[^a-zA-Z0-9._-]", "_"); + if (safeName.isBlank()) safeName = "media"; + // "file.bin" is the WeCom-no-filename sentinel; if magic gave us + // something better, replace the extension. Same when hint had no + // extension at all. + boolean hintIsGeneric = safeName.equals("file.bin") || safeName.equals("media") + || !safeName.contains("."); + if (hintIsGeneric && !".bin".equals(sniff.extension())) { + safeName = stripExtension(safeName) + sniff.extension(); + } + String storedName = "wecom_" + urlHash + "_" + safeName; + Path filePath = uploadDir.resolve(storedName); + Files.write(filePath, fileData); + + String fileUrl = "/api/v1/chat/files/" + conversationId + "/" + storedName; + log.info("[wecom] Inbound media saved: {} ({} bytes, sniffed={}), serve URL={}", + filePath, fileData.length, sniff.contentType(), fileUrl); + return new InboundMediaResult( + filePath.toAbsolutePath().toString(), + storedName, + fileUrl, + fileData.length, + safeName, + sniff.contentType()); + } catch (Exception e) { + log.error("[wecom] Failed to download inbound media: {}", e.getMessage(), e); + return null; + } + } + + /** + * 下载并解密企业微信媒体文件（旧版本，保留给 outbound / 其他场景使用） *

* AES-256-CBC 解密：base64 decode aesKey → IV = 前 16 字节 → PKCS#7 去填充 *