From 62b94b522f89fc83ea16dea1eaaa2c3ef827d39c Mon Sep 17 00:00:00 2001 From: matevip Date: Tue, 28 Apr 2026 14:59:59 +0800 Subject: [PATCH] fix(dingtalk): make inbound images visible to vision model and chat UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three knots untangled so an image sent from DingTalk lands in both the LLM's multimodal prompt and the chat history bubble: - Prefer MessageContent.downloadCode (universal, used by the new api.dingtalk.com messageFiles/download) over pictureDownloadCode (legacy oapi field). Sending the legacy code to the new API got HTTP 500 unknownError, which was the original 'image not recognized'. - After fetching bytes, persist to ~/.mateclaw/media/dingtalk/ so vision can read via FileSystemResource, AND stuff the same bytes into GeneratedFileCache so the UI gets an /api/v1/files/generated/{id} URL to render. Without the URL the message bubble showed an empty card. - Carry filename / contentType / size on the MessageContentPart so the chat history doesn't fall back to the 'unknown' caption. Same treatment applied to the richText branch (inline images from the PC client) and threaded through the Stream SDK path. Bundles in the prerequisite ChannelManager wiring of GeneratedFileCache into DingTalkChannelAdapter and the new DingTalkMediaUploader used by the outbound attachment flow that this work depends on. Known limit: GeneratedFileCache TTL is 10 min — fresh refreshes work, but viewing the image after a JVM restart needs a stable on-disk serving endpoint, which is intentionally out of scope here. --- .../java/vip/mate/channel/ChannelManager.java | 3 +- .../dingtalk/DingTalkChannelAdapter.java | 505 ++++++++++++++++-- .../dingtalk/DingTalkMediaUploader.java | 146 +++++ 3 files changed, 621 insertions(+), 33 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkMediaUploader.java diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java index e8369eb5..15d06837 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java @@ -44,6 +44,7 @@ public class ChannelManager { private final ChannelMessageRouter messageRouter; private final ChannelSessionStore channelSessionStore; private final ObjectMapper objectMapper; + private final vip.mate.tool.document.GeneratedFileCache generatedFileCache; /** 运行中的渠道适配器：channelId -> adapter */ private final Map activeAdapters = new HashMap<>(); @@ -440,7 +441,7 @@ public class ChannelManager { String type = channel.getChannelType(); return switch (type) { case "web" -> new WebChannelAdapter(channel, messageRouter, objectMapper); - case "dingtalk" -> new DingTalkChannelAdapter(channel, messageRouter, objectMapper); + case "dingtalk" -> new DingTalkChannelAdapter(channel, messageRouter, objectMapper, generatedFileCache); case "feishu" -> new FeishuChannelAdapter(channel, messageRouter, objectMapper); case "telegram" -> new TelegramChannelAdapter(channel, messageRouter, objectMapper); case "discord" -> new DiscordChannelAdapter(channel, messageRouter, objectMapper); diff --git a/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkChannelAdapter.java index 49508735..26cda9c5 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkChannelAdapter.java @@ -62,10 +62,18 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St /** AI Card 管理器（message_type=card 时初始化） */ private DingTalkAICardManager aiCardManager; + /** 钉钉媒体上传器（doStart 时初始化） */ + private DingTalkMediaUploader mediaUploader; + + /** 工具产生的可下载字节缓存（DocxRenderTool 等用，注入避免回调到自己的 HTTP API） */ + private final vip.mate.tool.document.GeneratedFileCache generatedFileCache; + public DingTalkChannelAdapter(ChannelEntity channelEntity, ChannelMessageRouter messageRouter, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + vip.mate.tool.document.GeneratedFileCache generatedFileCache) { super(channelEntity, messageRouter, objectMapper); + this.generatedFileCache = generatedFileCache; // 钉钉 Stream 重连：2s→4s→8s→16s→30s，无限重试 this.backoff = new ExponentialBackoff(2000, 30000, 2.0, -1); } @@ -97,6 +105,9 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St .connectTimeout(Duration.ofSeconds(10)) .build(); + // 媒体上传器：sampleFile / sampleImageMsg 都靠它先把字节传到钉钉拿 mediaId + this.mediaUploader = new DingTalkMediaUploader(httpClient, objectMapper); + // 初始化 AI Card 管理器（message_type=card 且配置了模板 ID） String cardTemplateId = getConfigString("card_template_id"); String messageType = getConfigString("message_type", "markdown"); @@ -168,13 +179,25 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St // 消息内容 // 钉钉服务端已经把语音转写好放在 MessageContent.recognition 里（跟 // 企业微信 voice.content 一个模式），不需要 STT。优先读 recognition； - // 否则读 text.content。其他复杂类型（picture / richText）暂由 - // handleWebhook 内部处理 —— 但 stream 模式下我们目前没把那些类型 - // 的字段塞进 payload，是个遗留待修项（picture / richText 同样会掉消息）。 - String recognition = msg.getContent() != null ? msg.getContent().getRecognition() : null; + // picture msgtype 把 downloadCode / pictureDownloadCode 透传给 webhook 处理； + // 否则读 text.content。 + // richText 还没接上 —— stream 模式收到 richText 会重新落到 webhook 默认 else + // 分支静默丢消息，单独修。 + com.dingtalk.open.app.api.models.bot.MessageContent body = msg.getContent(); + String recognition = body != null ? body.getRecognition() : null; + String pictureDownloadCode = body != null ? body.getPictureDownloadCode() : null; + String downloadCode = body != null ? body.getDownloadCode() : null; + if (recognition != null && !recognition.isBlank()) { payload.put("msgtype", "audio"); payload.put("audio", Map.of("recognition", recognition)); + } else if ("picture".equals(msg.getMsgtype()) + && (pictureDownloadCode != null || downloadCode != null)) { + payload.put("msgtype", "picture"); + Map picture = new java.util.HashMap<>(); + if (pictureDownloadCode != null) picture.put("pictureDownloadCode", pictureDownloadCode); + if (downloadCode != null) picture.put("downloadCode", downloadCode); + payload.put("picture", picture); } else if (msg.getText() != null) { payload.put("msgtype", "text"); payload.put("text", Map.of("content", msg.getText().getContent() != null ? msg.getText().getContent() : "")); @@ -234,10 +257,16 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St } /** - * 获取机器人编码 + * 获取机器人编码。 + *

+ * 钉钉自建应用机器人的 robotCode 大多数情况等于 AppKey（即 client_id）， + * 所以 robot_code 显式没配时直接 fallback 到 client_id —— 99% 的用户零配置就能用。 + * 第三方应用 / 单独申请的机器人才需要手动填 robot_code。 */ public String getRobotCode() { - return getConfigString("robot_code"); + String configured = getConfigString("robot_code"); + if (configured != null && !configured.isBlank()) return configured; + return getConfigString("client_id"); } // ==================== StreamingChannelAdapter ==================== @@ -387,7 +416,21 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St String downloadCode = (String) item.get("downloadCode"); String pictureUrl = (String) item.get("pictureUrl"); if (downloadCode != null || pictureUrl != null) { - contentParts.add(MessageContentPart.image(downloadCode, pictureUrl)); + MessageContentPart imgPart = MessageContentPart.image(downloadCode, pictureUrl); + // Same as the standalone picture branch: vision needs bytes, not just an opaque + // downloadCode. Best-effort fetch; falls back to image-with-id if download fails. + if (downloadCode != null && !downloadCode.isBlank()) { + DownloadedMedia media = downloadDingTalkMedia(downloadCode); + if (media != null) { + if (media.path() != null) imgPart.setPath(media.path()); + // Only override richText pictureUrl if we have a local-served URL. + if (media.url() != null) imgPart.setFileUrl(media.url()); + if (media.fileName() != null) imgPart.setFileName(media.fileName()); + if (media.contentType() != null) imgPart.setContentType(media.contentType()); + if (media.size() > 0) imgPart.setFileSize(media.size()); + } + } + contentParts.add(imgPart); } } textContent = textBuilder.toString().trim(); @@ -402,6 +445,35 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St textContent = recognition.trim(); contentParts.add(MessageContentPart.text(textContent)); } + } else if ("picture".equals(msgtype)) { + // 单图消息。钉钉只给 downloadCode（不透明 ID），vision 模型直接读不了。 + // 调 /v1.0/robot/messageFiles/download 拿临时 URL，下载字节存本地，把 + // path 塞进 MessageContentPart 让多模态 LLM 能从磁盘读图。下载失败仍保留 + // image part（带 downloadCode）作为占位 — 至少消息不丢。 + Map pictureBody = (Map) payload.get("picture"); + String picDownloadCode = pictureBody != null ? (String) pictureBody.get("pictureDownloadCode") : null; + String dlCode = pictureBody != null ? (String) pictureBody.get("downloadCode") : null; + // Prefer the universal `downloadCode` — that's what the new + // api.dingtalk.com/v1.0/robot/messageFiles/download API expects. + // `pictureDownloadCode` is the legacy field for the old + // oapi.dingtalk.com endpoint; passing it to the new API gets + // HTTP 500 "unknownError". + String code = dlCode != null ? dlCode : picDownloadCode; + if (code != null && !code.isBlank()) { + DownloadedMedia media = downloadDingTalkMedia(code); + // mediaId stays as downloadCode for traceability; fileUrl is the + // browser-renderable URL (or null on download failure). + MessageContentPart imgPart = MessageContentPart.image( + code, media != null ? media.url() : null); + if (media != null) { + if (media.path() != null) imgPart.setPath(media.path()); + if (media.fileName() != null) imgPart.setFileName(media.fileName()); + if (media.contentType() != null) imgPart.setContentType(media.contentType()); + if (media.size() > 0) imgPart.setFileSize(media.size()); + } + contentParts.add(imgPart); + textContent = "[图片]"; + } } else { // 默认 text 消息 Map msgBody = (Map) payload.get("text"); @@ -446,7 +518,12 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St .rawPayload(payload) .build(); - message.setReplyToken(sessionWebhook); + // Reply token 编码上下文：sessionWebhook 走 Markdown 文本（无附件）， + // userId / conversationId 走 Robot API（能传 sampleFile / sampleImageMsg 真附件）。 + // 出站时 sendContentParts 解析这个 token 决定路径。格式是不透明字符串，sendMessage + // 也能处理 —— 如果只是发 markdown 就直接走 webhook，不需要 access_token。 + String dtReplyToken = encodeReplyToken(sessionWebhook, senderId, conversationId, conversationType); + message.setReplyToken(dtReplyToken); onMessage(message); } catch (Exception e) { @@ -460,19 +537,22 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St log.warn("[dingtalk] Channel not started, cannot send message"); return; } + // Delegate to sendContentParts so the URL sniff for /api/v1/files/generated/ runs on + // streaming-response markdown too — processStreamAsText's renderAndSend → sendMessage + // path used to bypass it. content==null/blank short-circuits to no-op. + if (content == null || content.isEmpty()) return; + sendContentParts(targetId, List.of(MessageContentPart.text(content))); + } + /** Existing sessionWebhook reply path, factored out so sendMessage and sendContentParts share it. */ + private void sendMarkdownViaWebhook(String webhookUrl, String content) { String messageType = getConfigString("message_type", "markdown"); - try { String jsonBody; - // card 模式的文本回退也使用 markdown 格式 if ("markdown".equals(messageType) || "card".equals(messageType)) { jsonBody = objectMapper.writeValueAsString(Map.of( "msgtype", "markdown", - "markdown", Map.of( - "title", "MateClaw", - "text", content - ) + "markdown", Map.of("title", "MateClaw", "text", content) )); } else { jsonBody = objectMapper.writeValueAsString(Map.of( @@ -480,20 +560,17 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St "text", Map.of("content", content) )); } - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(targetId)) + .uri(URI.create(webhookUrl)) .header("Content-Type", "application/json") .POST(HttpRequest.BodyPublishers.ofString(jsonBody)) .build(); - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); if (response.statusCode() != 200) { log.warn("[dingtalk] Send message failed: status={}, body={}", response.statusCode(), response.body()); } else { log.debug("[dingtalk] Message sent successfully via sessionWebhook"); } - } catch (Exception e) { log.error("[dingtalk] Failed to send message: {}", e.getMessage(), e); } @@ -506,26 +583,388 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St return; } - // 钉钉 sessionWebhook 只支持 text/markdown/link/actionCard 等类型。 - // 图片需要通过上传 media 后发送，这里暂时将媒体内容以 Markdown 图片语法发出。 + ReplyContext ctx = decodeReplyToken(targetId); + + // Phase 1 — walk parts. Text concatenates into the markdown reply; image/file + // parts (with resolvable bytes) become upload jobs. Text segments are also + // scanned for /api/v1/files/generated/{id} URLs (DocxRenderTool output etc.): + // hits are pulled out of the cache and queued as file uploads, with the URL + // replaced inline by a "📎 filename" badge so the user doesn't see a stale link. + List uploadJobs = new ArrayList<>(); StringBuilder markdown = new StringBuilder(); for (MessageContentPart part : parts) { if (part == null) continue; switch (part.getType()) { - case "text" -> { if (part.getText() != null) markdown.append(part.getText()); } + case "text" -> { + String text = part.getText(); + if (text == null || text.isEmpty()) break; + markdown.append(sniffGeneratedFiles(text, uploadJobs)); + } case "image" -> { - if (part.getFileUrl() != null) { + byte[] bytes = resolveBytes(part); + String fileName = part.getFileName() != null ? part.getFileName() : "image.png"; + if (bytes != null) { + uploadJobs.add(new UploadJob(bytes, fileName, "image")); + } else if (part.getFileUrl() != null) { markdown.append("\n![图片](").append(part.getFileUrl()).append(")\n"); } else { markdown.append("\n[图片]\n"); } } - case "file" -> markdown.append("\n[文件: ").append(part.getFileName() != null ? part.getFileName() : "").append("]\n"); + case "file" -> { + byte[] bytes = resolveBytes(part); + String fileName = part.getFileName() != null ? part.getFileName() : "file.bin"; + if (bytes != null) { + uploadJobs.add(new UploadJob(bytes, fileName, "file")); + } else { + markdown.append("\n[文件: ").append(fileName).append("]\n"); + } + } default -> { if (part.getText() != null) markdown.append(part.getText()); } } } - sendMessage(targetId, markdown.toString().trim()); + // Phase 2 — send the markdown body (if any) via sessionWebhook. + String md = markdown.toString().trim(); + if (!md.isEmpty()) { + if (ctx.webhook != null && ctx.webhook.startsWith("http")) { + sendMarkdownViaWebhook(ctx.webhook, md); + } else { + // No sessionWebhook — fall back to Robot API text send. proactiveSend() + // re-decodes the same token and routes by userId/conv. Don't call back into + // sendMessage() here — sendMessage() now delegates to sendContentParts(), + // which would recurse infinitely. + proactiveSend(ctx.userId != null ? ctx.userId : targetId, md); + } + } + + // Phase 3 — upload + send media jobs via Robot API. Skips silently and falls back + // to a markdown placeholder if Robot API isn't usable (no robot_code, or + // no userId / conversationId in the reply context). + if (!uploadJobs.isEmpty()) { + sendUploadJobs(ctx, uploadJobs); + } + } + + /** Scan a text fragment for `/api/v1/files/generated/{id}` URLs; return the rewritten text + * with each hit replaced by "📎 filename" and the corresponding bytes pushed onto jobs. */ + private String sniffGeneratedFiles(String text, List jobs) { + if (generatedFileCache == null) return text; + java.util.regex.Matcher m = GENERATED_URL_PATTERN.matcher(text); + StringBuilder out = new StringBuilder(); + while (m.find()) { + String id = m.group(1); + var entry = generatedFileCache.get(id).orElse(null); + if (entry != null) { + String type = isImageMime(entry.mimeType()) ? "image" : "file"; + jobs.add(new UploadJob(entry.bytes(), entry.filename(), type)); + m.appendReplacement(out, java.util.regex.Matcher.quoteReplacement("📎 " + entry.filename())); + } else { + // Cache miss / expired — leave the URL alone so user can still try clicking. + m.appendReplacement(out, java.util.regex.Matcher.quoteReplacement(m.group(0))); + } + } + m.appendTail(out); + return out.toString(); + } + + private static final java.util.regex.Pattern GENERATED_URL_PATTERN = + java.util.regex.Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)"); + + private static boolean isImageMime(String mimeType) { + return mimeType != null && mimeType.toLowerCase().startsWith("image/"); + } + + /** Pull bytes from disk (part.path) or from the in-memory generated-file cache (part.fileUrl). */ + private byte[] resolveBytes(MessageContentPart part) { + if (part == null) return null; + // Disk path: trust the agent's filesystem write, but guard against blowup on big files. + String path = part.getPath(); + if (path != null && !path.isBlank()) { + try { + java.nio.file.Path p = java.nio.file.Paths.get(path); + if (java.nio.file.Files.exists(p) && java.nio.file.Files.size(p) <= DingTalkMediaUploader.MAX_FILE_BYTES) { + return java.nio.file.Files.readAllBytes(p); + } + } catch (Exception e) { + log.debug("[dingtalk] resolveBytes from path failed: {}", e.getMessage()); + } + } + // GeneratedFileCache URL: /api/v1/files/generated/{id} + String url = part.getFileUrl(); + if (url != null && generatedFileCache != null) { + java.util.regex.Matcher m = GENERATED_URL_PATTERN.matcher(url); + if (m.find()) { + var entry = generatedFileCache.get(m.group(1)).orElse(null); + if (entry != null) return entry.bytes(); + } + } + return null; + } + + /** Records to keep sendContentParts readable. */ + private record UploadJob(byte[] bytes, String fileName, String type) {} + + /** Decoded reply context, populated from {@link #encodeReplyToken}. */ + private static class ReplyContext { + String webhook; + String userId; + String convId; + /** "1" = one-to-one, "2" = group; empty/null = unknown. */ + String chatType; + } + + /** + * Reply token is JSON now: {wh, user, conv, ct}. Old bare-URL tokens (e.g. from + * an in-flight conversation that started before this change) are still accepted — + * the leading-{ check tells JSON apart from a raw HTTP URL. + */ + private String encodeReplyToken(String webhook, String userId, String conversationId, String chatType) { + java.util.LinkedHashMap ctx = new java.util.LinkedHashMap<>(); + if (webhook != null) ctx.put("wh", webhook); + if (userId != null) ctx.put("user", userId); + if (conversationId != null) ctx.put("conv", conversationId); + if (chatType != null) ctx.put("ct", chatType); + try { + return objectMapper.writeValueAsString(ctx); + } catch (Exception e) { + return webhook != null ? webhook : ""; + } + } + + @SuppressWarnings("unchecked") + private ReplyContext decodeReplyToken(String token) { + ReplyContext c = new ReplyContext(); + if (token == null || token.isBlank()) return c; + String trimmed = token.trim(); + if (!trimmed.startsWith("{")) { + // Legacy bare sessionWebhook URL or userId — treat http-prefixed as webhook, + // otherwise as userId for proactive send. + if (trimmed.startsWith("http")) c.webhook = trimmed; + else c.userId = trimmed; + return c; + } + try { + Map m = objectMapper.readValue(trimmed, Map.class); + c.webhook = (String) m.get("wh"); + c.userId = (String) m.get("user"); + c.convId = (String) m.get("conv"); + c.chatType = (String) m.get("ct"); + } catch (Exception e) { + c.webhook = trimmed; + } + return c; + } + + /** + * Upload each job to DingTalk's media endpoint, then send a sampleFile / sampleImageMsg + * via the Robot API. Falls back gracefully (one log line per skipped job) if the channel + * isn't configured for proactive send (no robot_code, or no usable target id). + */ + private void sendUploadJobs(ReplyContext ctx, List jobs) { + String robotCode = getRobotCode(); + if (robotCode == null || robotCode.isBlank()) { + log.warn("[dingtalk] {} attachment(s) skipped: robot_code not configured (Robot API needed for sampleFile / sampleImageMsg)", jobs.size()); + return; + } + boolean isGroup = "2".equals(ctx.chatType) && ctx.convId != null && !ctx.convId.isBlank(); + boolean isOneToOne = ctx.userId != null && !ctx.userId.isBlank(); + if (!isGroup && !isOneToOne) { + log.warn("[dingtalk] {} attachment(s) skipped: no usable userId / conversationId in reply context", jobs.size()); + return; + } + + String accessToken = getDingTalkAccessToken(); + if (accessToken == null) { + log.warn("[dingtalk] {} attachment(s) skipped: failed to get access_token", jobs.size()); + return; + } + + for (UploadJob job : jobs) { + try { + String mediaId = "image".equals(job.type) + ? mediaUploader.uploadImage(job.bytes, job.fileName, accessToken) + : mediaUploader.uploadFile(job.bytes, job.fileName, accessToken); + if (mediaId == null) { + log.warn("[dingtalk] upload failed for {}, attachment skipped", job.fileName); + continue; + } + + String msgKey; + String msgParam; + if ("image".equals(job.type)) { + msgKey = "sampleImageMsg"; + msgParam = objectMapper.writeValueAsString(Map.of("photoURL", mediaId)); + } else { + msgKey = "sampleFile"; + msgParam = objectMapper.writeValueAsString(Map.of( + "mediaId", mediaId, + "fileName", job.fileName, + "fileType", inferFileType(job.fileName) + )); + } + + Map body = new java.util.LinkedHashMap<>(); + body.put("robotCode", robotCode); + body.put("msgKey", msgKey); + body.put("msgParam", msgParam); + String endpoint; + if (isGroup) { + body.put("openConversationId", ctx.convId); + endpoint = "https://api.dingtalk.com/v1.0/robot/groupMessages/send"; + } else { + body.put("userIds", List.of(ctx.userId)); + endpoint = "https://api.dingtalk.com/v1.0/robot/oToMessages/batchSend"; + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(endpoint)) + .header("Content-Type", "application/json") + .header("x-acs-dingtalk-access-token", accessToken) + .POST(HttpRequest.BodyPublishers.ofString(objectMapper.writeValueAsString(body))) + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + log.warn("[dingtalk] sampleFile send HTTP {}: {}", response.statusCode(), response.body()); + } else { + log.info("[dingtalk] attachment sent: {} ({})", job.fileName, job.type); + } + } catch (Exception e) { + log.warn("[dingtalk] sendUploadJobs error for {}: {}", job.fileName, e.getMessage()); + } + } + } + + /** sampleFile expects fileType as a short extension string (docx / pdf / xlsx / ...). */ + private static String inferFileType(String fileName) { + if (fileName == null) return ""; + int dot = fileName.lastIndexOf('.'); + if (dot < 0 || dot == fileName.length() - 1) return ""; + return fileName.substring(dot + 1).toLowerCase(); + } + + // ==================== 入站媒体下载 ==================== + + /** + * Result of a media download: local path (for vision pipeline) + HTTP URL + * (for browser rendering) + metadata to populate MessageContentPart so the + * UI doesn't fall back to "unknown" filenames. + */ + private record DownloadedMedia(String path, String url, String fileName, + String contentType, long size) { + } + + /** + * 把入站消息里的 downloadCode 解析成本地文件路径 + 浏览器可访问的 URL。 + *

+ * 钉钉的 inbound 图片 / 文件不直接给字节，只给 downloadCode（不透明的内部 ID）。 + * 要让 vision 模型读图、或落盘存档给用户回看，必须先调 + * {@code /v1.0/robot/messageFiles/download} 拿一个短期 downloadUrl，再 GET 拉字节。 + *

+ * 字节落两份：磁盘 (~/.mateclaw/media/dingtalk/) 给 vision 读 path， + * GeneratedFileCache 给 UI 通过 /api/v1/files/generated/{id} 渲染（10 min TTL）。 + *

+ * 失败 / 没配 robot_code / token 拿不到 → 返回 null，调用方继续把 downloadCode 当占位用。 + */ + private DownloadedMedia downloadDingTalkMedia(String downloadCode) { + if (downloadCode == null || downloadCode.isBlank()) return null; + String robotCode = getRobotCode(); + if (robotCode == null || robotCode.isBlank()) { + log.debug("[dingtalk] media download skipped: no robot_code"); + return null; + } + String accessToken = getDingTalkAccessToken(); + if (accessToken == null) { + log.debug("[dingtalk] media download skipped: no access_token"); + return null; + } + + try { + // Step 1: ask DingTalk for a short-TTL download URL + String body = objectMapper.writeValueAsString(Map.of( + "downloadCode", downloadCode, + "robotCode", robotCode + )); + HttpRequest infoReq = HttpRequest.newBuilder() + .uri(URI.create("https://api.dingtalk.com/v1.0/robot/messageFiles/download")) + .header("Content-Type", "application/json") + .header("x-acs-dingtalk-access-token", accessToken) + .timeout(Duration.ofSeconds(10)) + .POST(HttpRequest.BodyPublishers.ofString(body)) + .build(); + HttpResponse infoResp = httpClient.send(infoReq, HttpResponse.BodyHandlers.ofString()); + if (infoResp.statusCode() != 200) { + // Mask the codes so the log doesn't carry full credentials but is still + // useful for diagnosing whether we sent the right shape. + String codeTail = downloadCode.length() > 8 + ? "..." + downloadCode.substring(downloadCode.length() - 8) : downloadCode; + String robotTail = robotCode.length() > 6 + ? "..." + robotCode.substring(robotCode.length() - 6) : robotCode; + log.warn("[dingtalk] media download info HTTP {} (downloadCode={}, robotCode={}): {}", + infoResp.statusCode(), codeTail, robotTail, infoResp.body()); + return null; + } + Map infoData = objectMapper.readValue(infoResp.body(), Map.class); + String downloadUrl = (String) infoData.get("downloadUrl"); + if (downloadUrl == null || downloadUrl.isBlank()) { + log.warn("[dingtalk] media download info missing downloadUrl: {}", infoResp.body()); + return null; + } + + // Step 2: pull bytes from the signed URL + HttpRequest dlReq = HttpRequest.newBuilder() + .uri(URI.create(downloadUrl)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build(); + HttpResponse dlResp = httpClient.send(dlReq, HttpResponse.BodyHandlers.ofByteArray()); + if (dlResp.statusCode() != 200) { + log.warn("[dingtalk] media download fetch HTTP {}", dlResp.statusCode()); + return null; + } + byte[] bytes = dlResp.body(); + + // Step 3: persist to ~/.mateclaw/media/dingtalk/. Filename derived from + // downloadCode (sanitised) + extension inferred from Content-Type so vision + // pipelines that key on extension still work. + String contentType = dlResp.headers().firstValue("Content-Type").orElse("image/jpeg"); + String ext = "jpg"; + if (contentType.contains("png")) ext = "png"; + else if (contentType.contains("gif")) ext = "gif"; + else if (contentType.contains("webp")) ext = "webp"; + else if (contentType.contains("pdf")) ext = "pdf"; + + java.nio.file.Path mediaDir = java.nio.file.Paths.get( + System.getProperty("user.home"), ".mateclaw", "media", "dingtalk"); + java.nio.file.Files.createDirectories(mediaDir); + String safeCode = downloadCode.replaceAll("[^a-zA-Z0-9_-]", "_"); + if (safeCode.length() > 32) safeCode = safeCode.substring(safeCode.length() - 32); + java.nio.file.Path filePath = mediaDir.resolve(safeCode + "." + ext); + java.nio.file.Files.write(filePath, bytes); + + // Step 4: also stuff into GeneratedFileCache so the UI can render the image + // via /api/v1/files/generated/{id}. Best-effort — failure here doesn't kill + // the vision pipeline (path is still useful). + String url = null; + if (generatedFileCache != null) { + try { + String fileName = safeCode + "." + ext; + String id = generatedFileCache.put(bytes, fileName, contentType); + url = "/api/v1/files/generated/" + id; + } catch (Exception cacheEx) { + log.debug("[dingtalk] cache put failed: {}", cacheEx.getMessage()); + } + } + + String fileName = safeCode + "." + ext; + log.info("[dingtalk] media downloaded: {} ({} bytes, {}) url={}", + fileName, bytes.length, contentType, url); + return new DownloadedMedia( + filePath.toAbsolutePath().toString(), url, fileName, contentType, bytes.length); + } catch (Exception e) { + log.warn("[dingtalk] media download failed: {}", e.getMessage()); + return null; + } } // ==================== 主动推送 ==================== @@ -549,19 +988,21 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St return; } - if (targetId.startsWith("http")) { - // sessionWebhook 直接发送 - sendMessage(targetId, content); + // targetId may be: sessionWebhook URL, raw userId, or our JSON-encoded reply token + // (after the replyToken refactor). Decode unifies all three. + ReplyContext ctx = decodeReplyToken(targetId); + if (ctx.webhook != null && ctx.webhook.startsWith("http")) { + sendMarkdownViaWebhook(ctx.webhook, content); return; } // 通过 Robot API 发送：获取 access_token 后调用 /v1.0/robot/oToMessages/batchSend - String robotCode = getConfigString("robot_code"); + String robotCode = getRobotCode(); if (robotCode == null || robotCode.isBlank()) { - log.warn("[dingtalk] robot_code not configured, falling back to sendMessage"); - sendMessage(targetId, content); + log.warn("[dingtalk] robot_code not configured, no usable webhook either — proactive send skipped"); return; } + String resolvedUserId = ctx.userId != null ? ctx.userId : targetId; try { String accessToken = getDingTalkAccessToken(); @@ -583,7 +1024,7 @@ public class DingTalkChannelAdapter extends AbstractChannelAdapter implements St String jsonBody = objectMapper.writeValueAsString(Map.of( "robotCode", robotCode, - "userIds", List.of(targetId), + "userIds", List.of(resolvedUserId), "msgKey", msgKey, "msgParam", objectMapper.writeValueAsString(msgParam) )); diff --git a/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkMediaUploader.java b/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkMediaUploader.java new file mode 100644 index 00000000..14aa9fdc --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/dingtalk/DingTalkMediaUploader.java @@ -0,0 +1,146 @@ +package vip.mate.channel.dingtalk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * 钉钉媒体文件上传 helper + *

+ * 钉钉 SampleFile / SampleImageMsg 消息要求 mediaId，必须先把字节传到钉钉的 media + * 存储拿到 mediaId（3 天有效）。 + * + * 

+ *   POST https://oapi.dingtalk.com/media/upload?access_token=XXX&type=file|image|voice|video
+ *   Content-Type: multipart/form-data
+ *
+ *   form field: media (file content)
+ *
+ *   ← { "errcode": 0, "media_id": "@xxx", "type": "file", "created_at": ... }
+ *
+ *

+ * 单文件上限 20 MB（钉钉服务端限制；客户端上传时 timeout 30s 已经够大文件用）。 + * + * @author MateClaw Team + */ +@Slf4j +@RequiredArgsConstructor +public class DingTalkMediaUploader { + + private static final String UPLOAD_URL = "https://oapi.dingtalk.com/media/upload"; + + /** 钉钉单文件大小上限 20 MB —— 不是我们的政策，是钉钉服务端拒绝上传更大的，提前拦下省一次失败往返 */ + public static final int MAX_FILE_BYTES = 20 * 1024 * 1024; + + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + + public String uploadFile(byte[] bytes, String fileName, String accessToken) { + return upload(bytes, fileName, "file", accessToken); + } + + public String uploadImage(byte[] bytes, String fileName, String accessToken) { + return upload(bytes, fileName, "image", accessToken); + } + + /** + * @param type "file" / "image" / "voice" / "video" + * @return mediaId（带 `@` 前缀）；任何失败返回 null，调用方自己决定怎么回退 + */ + private String upload(byte[] bytes, String fileName, String type, String accessToken) { + if (bytes == null || bytes.length == 0) { + log.warn("[dingtalk-upload] empty bytes for {} ({})", type, fileName); + return null; + } + if (bytes.length > MAX_FILE_BYTES) { + log.warn("[dingtalk-upload] file too large: {} bytes (limit {} bytes)", + bytes.length, MAX_FILE_BYTES); + return null; + } + if (accessToken == null || accessToken.isBlank()) { + log.warn("[dingtalk-upload] missing access_token"); + return null; + } + + try { + String boundary = "----DingTalkBoundary" + UUID.randomUUID().toString().replace("-", ""); + byte[] body = buildMultipartBody(boundary, fileName, bytes); + + String url = UPLOAD_URL + "?access_token=" + accessToken + "&type=" + type; + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .timeout(Duration.ofSeconds(30)) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + log.warn("[dingtalk-upload] HTTP {}: {}", response.statusCode(), response.body()); + return null; + } + + Map result = objectMapper.readValue(response.body(), Map.class); + Integer errcode = result.get("errcode") instanceof Number n ? n.intValue() : null; + if (errcode != null && errcode != 0) { + log.warn("[dingtalk-upload] errcode={}, errmsg={}", errcode, result.get("errmsg")); + return null; + } + String mediaId = (String) result.get("media_id"); + if (mediaId == null || mediaId.isBlank()) { + log.warn("[dingtalk-upload] empty media_id in response: {}", response.body()); + return null; + } + log.info("[dingtalk-upload] uploaded {} ({} bytes) → mediaId suffix=...{}", + type, bytes.length, + mediaId.length() > 8 ? mediaId.substring(mediaId.length() - 8) : mediaId); + return mediaId; + } catch (Exception e) { + log.warn("[dingtalk-upload] failed: {}", e.getMessage()); + return null; + } + } + + /** + * 手写最小 multipart 体，避免引入第三方 multipart 库。 + * 字段名 "media"，content-type application/octet-stream（钉钉只看字节，不嫌弃 mime）。 + */ + private byte[] buildMultipartBody(String boundary, String fileName, byte[] fileBytes) { + String safeFileName = fileName != null && !fileName.isBlank() ? fileName : "upload.bin"; + // Use a simple ASCII-only fallback if the filename has non-ASCII chars to avoid header + // encoding issues; the actual bytes are unaffected. + String headerSafe = safeFileName.replaceAll("[\\r\\n\"]", "_"); + + String prefix = "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"media\"; filename=\"" + headerSafe + "\"\r\n" + + "Content-Type: application/octet-stream\r\n\r\n"; + String suffix = "\r\n--" + boundary + "--\r\n"; + + byte[] prefixBytes = prefix.getBytes(StandardCharsets.UTF_8); + byte[] suffixBytes = suffix.getBytes(StandardCharsets.UTF_8); + List chunks = new ArrayList<>(3); + chunks.add(prefixBytes); + chunks.add(fileBytes); + chunks.add(suffixBytes); + + int total = prefixBytes.length + fileBytes.length + suffixBytes.length; + byte[] body = new byte[total]; + int pos = 0; + for (byte[] c : chunks) { + System.arraycopy(c, 0, body, pos, c.length); + pos += c.length; + } + return body; + } +}