mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(channel): wecom file pipeline + IM approval hint visibility
This commit is contained in:
matevip
parent
225d14026b
commit
134fa1a975
@ -481,7 +481,8 @@ public class ChannelManager {
|
||||
case "telegram" -> new TelegramChannelAdapter(channel, messageRouter, objectMapper);
|
||||
case "discord" -> new DiscordChannelAdapter(channel, messageRouter, objectMapper);
|
||||
case "wecom" -> new WeComChannelAdapter(channel, messageRouter, objectMapper,
|
||||
approvalNotificationService, weComCardDispatcher, weComKeepaliveScheduler);
|
||||
approvalNotificationService, weComCardDispatcher, weComKeepaliveScheduler,
|
||||
generatedFileCache);
|
||||
case "qq" -> new QQChannelAdapter(channel, messageRouter, objectMapper);
|
||||
case "weixin" -> new WeixinChannelAdapter(channel, messageRouter, objectMapper);
|
||||
case "slack" -> new vip.mate.channel.slack.SlackChannelAdapter(channel, messageRouter, objectMapper);
|
||||
|
||||
@ -469,7 +469,10 @@ public class ChannelMessageRouter {
|
||||
ResolveOutcome denyOutcome = approvalService.resolve(
|
||||
pending.getPendingId(), message.getSenderId(), "denied");
|
||||
conversationService.removeApprovalPlaceholders(conversationId);
|
||||
adapter.sendMessage(replyTarget, "⛔ 已拒绝执行工具: " + pending.getToolName());
|
||||
String denyHint = "⛔ 已拒绝执行工具: " + pending.getToolName();
|
||||
persistAndBroadcastApprovalHint(conversationId, denyHint,
|
||||
"denied", pending.getPendingId(), pending.getToolName());
|
||||
adapter.sendMessage(replyTarget, denyHint);
|
||||
log.info("[{}] Approval DENIED via IM command: pendingId={}, tool={}, msgRewritten={}",
|
||||
adapter.getChannelType(), pending.getPendingId(), pending.getToolName(),
|
||||
denyOutcome.messagesRewritten());
|
||||
@ -479,7 +482,10 @@ public class ChannelMessageRouter {
|
||||
// Non-approval message while a pending exists → treat as implicit deny.
|
||||
approvalService.resolve(pending.getPendingId(), message.getSenderId(), "denied");
|
||||
conversationService.removeApprovalPlaceholders(conversationId);
|
||||
adapter.sendMessage(replyTarget, "⛔ 审批已取消。将继续处理您的新消息。");
|
||||
String cancelHint = "⛔ 审批已取消。将继续处理您的新消息。";
|
||||
persistAndBroadcastApprovalHint(conversationId, cancelHint,
|
||||
"cancelled", pending.getPendingId(), pending.getToolName());
|
||||
adapter.sendMessage(replyTarget, cancelHint);
|
||||
log.info("[{}] Approval auto-cancelled (non-approval message): pendingId={}",
|
||||
adapter.getChannelType(), pending.getPendingId());
|
||||
// Fall through to process the new message normally.
|
||||
@ -763,8 +769,14 @@ public class ChannelMessageRouter {
|
||||
String replyTarget = resolveReplyTarget(triggerMessage);
|
||||
Long agentId = channelEntity.getAgentId();
|
||||
|
||||
// 通知用户审批已通过
|
||||
adapter.sendMessage(replyTarget, "✅ 已批准执行工具: " + consumed.getToolName());
|
||||
// Notify the user that the approval went through. Persist + broadcast so a
|
||||
// Web mirror of the same conversationId sees the resolution; otherwise this
|
||||
// hint would only land in the IM channel and the Web admin console would
|
||||
// show the replay reply with no preceding "approved" marker.
|
||||
String approveHint = "✅ 已批准执行工具: " + consumed.getToolName();
|
||||
persistAndBroadcastApprovalHint(conversationId, approveHint,
|
||||
"approved", consumed.getPendingId(), consumed.getToolName());
|
||||
adapter.sendMessage(replyTarget, approveHint);
|
||||
|
||||
// 清理 DB 中残留的审批占位消息
|
||||
conversationService.removeApprovalPlaceholders(conversationId);
|
||||
@ -800,7 +812,62 @@ public class ChannelMessageRouter {
|
||||
adapter.getChannelType(), consumed.getToolName(), reply.length());
|
||||
} catch (Exception e) {
|
||||
log.error("[approval-replay] Replay failed: {}", e.getMessage(), e);
|
||||
adapter.sendMessage(replyTarget, "❌ 工具执行失败: " + e.getMessage());
|
||||
String errHint = "❌ 工具执行失败: " + e.getMessage();
|
||||
persistAndBroadcastApprovalHint(conversationId, errHint, null, null, null);
|
||||
adapter.sendMessage(replyTarget, errHint);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist an approval-related hint as an assistant message and best-effort
|
||||
* broadcast it to any live SSE viewer of the conversation.
|
||||
* <p>
|
||||
* Without this, IM-driven approve/deny only reaches the originating IM
|
||||
* channel via {@code adapter.sendMessage(...)} — a Web mirror of the same
|
||||
* conversationId has no record of the resolution because nothing lands in
|
||||
* {@code mate_message} and no SSE event is emitted. The hint then "vanishes"
|
||||
* from the Web admin console even though it shows up on the user's phone.
|
||||
* <p>
|
||||
* Persistence is the load-bearing fix (Web reload picks it up). Broadcast
|
||||
* is best-effort: if no SSE stream is currently registered for the
|
||||
* conversation, the broadcast no-ops silently — that's the common case
|
||||
* since IM-driven clicks rarely race with an active web subscriber.
|
||||
*
|
||||
* @param conversationId conversation owning the hint
|
||||
* @param hint text to render as an assistant bubble
|
||||
* @param decision "approved" / "denied" / "cancelled" / null (skips the
|
||||
* structured resolved event when null, e.g. on replay error)
|
||||
* @param pendingId pending approval id; null when not applicable
|
||||
* @param toolName tool name for the structured event; null when not applicable
|
||||
*/
|
||||
private void persistAndBroadcastApprovalHint(String conversationId, String hint,
|
||||
String decision, String pendingId,
|
||||
String toolName) {
|
||||
try {
|
||||
conversationService.saveMessage(conversationId, "assistant", hint, null, "completed");
|
||||
} catch (Exception e) {
|
||||
log.warn("[approval-hint] saveMessage failed for conv={}: {}",
|
||||
conversationId, e.getMessage());
|
||||
}
|
||||
try {
|
||||
if (decision != null) {
|
||||
streamTracker.broadcastObject(conversationId, "tool_approval_resolved", Map.of(
|
||||
"pendingId", pendingId == null ? "" : pendingId,
|
||||
"decision", decision,
|
||||
"toolName", toolName == null ? "" : toolName,
|
||||
"timestamp", System.currentTimeMillis()
|
||||
));
|
||||
}
|
||||
streamTracker.broadcastObject(conversationId, "message_start",
|
||||
Map.of("role", "assistant"));
|
||||
streamTracker.broadcastObject(conversationId, "content_delta",
|
||||
Map.of("delta", hint));
|
||||
streamTracker.broadcastObject(conversationId, "message_complete",
|
||||
Map.of("status", "completed"));
|
||||
} catch (Exception e) {
|
||||
// Broadcast is best-effort; a missing run state is the common case.
|
||||
log.debug("[approval-hint] broadcast skipped/failed for conv={}: {}",
|
||||
conversationId, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@ import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.spec.IvParameterSpec;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
@ -26,6 +27,8 @@ import java.time.Duration;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
@ -240,16 +243,40 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
*/
|
||||
private final WeComKeepaliveScheduler keepaliveScheduler;
|
||||
|
||||
/**
|
||||
* In-memory cache of bytes generated by tools like
|
||||
* {@code DocxRenderTool} / {@code PptxRenderTool}. The agent emits a
|
||||
* {@code /api/v1/files/generated/{id}} URL referencing this cache; the
|
||||
* channel layer resolves that URL back to bytes and uploads them as a
|
||||
* native WeCom file message so the user actually receives a tappable
|
||||
* document instead of an unopenable link. Null-tolerant: if missing
|
||||
* (older constructor / test DI gap), URL stays inline as plain markdown
|
||||
* which renders as a non-interactive link in the bubble.
|
||||
*/
|
||||
private final vip.mate.tool.document.GeneratedFileCache generatedFileCache;
|
||||
|
||||
public WeComChannelAdapter(ChannelEntity channelEntity,
|
||||
ChannelMessageRouter messageRouter,
|
||||
ObjectMapper objectMapper,
|
||||
vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService,
|
||||
vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher,
|
||||
WeComKeepaliveScheduler keepaliveScheduler) {
|
||||
this(channelEntity, messageRouter, objectMapper, approvalNotificationService,
|
||||
cardDispatcher, keepaliveScheduler, null);
|
||||
}
|
||||
|
||||
public WeComChannelAdapter(ChannelEntity channelEntity,
|
||||
ChannelMessageRouter messageRouter,
|
||||
ObjectMapper objectMapper,
|
||||
vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService,
|
||||
vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher,
|
||||
WeComKeepaliveScheduler keepaliveScheduler,
|
||||
vip.mate.tool.document.GeneratedFileCache generatedFileCache) {
|
||||
super(channelEntity, messageRouter, objectMapper);
|
||||
this.approvalNotificationService = approvalNotificationService;
|
||||
this.cardDispatcher = cardDispatcher;
|
||||
this.keepaliveScheduler = keepaliveScheduler;
|
||||
this.generatedFileCache = generatedFileCache;
|
||||
// Default to 8 bounded attempts (~4 minutes total at 2s..30s exponential)
|
||||
// so the UI eventually settles in ERROR instead of getting stuck in
|
||||
// RECONNECTING forever. User config still overrides (-1 = infinite).
|
||||
@ -895,15 +922,9 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
Map<String, Object> imgBody = (Map<String, Object>) body.getOrDefault("image", Map.of());
|
||||
String url = (String) imgBody.getOrDefault("url", "");
|
||||
String aesKey = (String) imgBody.getOrDefault("aeskey", "");
|
||||
if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) {
|
||||
String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "image.jpg");
|
||||
if (localPath != null) {
|
||||
contentParts.add(MessageContentPart.image(localPath, url));
|
||||
} else {
|
||||
contentParts.add(MessageContentPart.image(url, url));
|
||||
}
|
||||
} else if (!url.isBlank()) {
|
||||
contentParts.add(MessageContentPart.image(url, url));
|
||||
String inboundConvId = inboundConversationId(senderId, chatId, chatType);
|
||||
if (!url.isBlank()) {
|
||||
contentParts.add(buildInboundImagePart(url, aesKey, msgId, "image.jpg", inboundConvId));
|
||||
}
|
||||
textContent = "[图片]";
|
||||
}
|
||||
@ -922,11 +943,23 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
Map<String, Object> fileBody = (Map<String, Object>) body.getOrDefault("file", Map.of());
|
||||
String url = (String) fileBody.getOrDefault("url", "");
|
||||
String aesKey = (String) fileBody.getOrDefault("aeskey", "");
|
||||
String filename = (String) fileBody.getOrDefault("filename", "file.bin");
|
||||
if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) {
|
||||
String localPath = downloadAndDecryptMedia(url, aesKey, msgId, filename);
|
||||
if (localPath != null) {
|
||||
contentParts.add(MessageContentPart.file(localPath, filename, null));
|
||||
// WeCom sometimes omits filename for forwarded files. Try a
|
||||
// few fallback keys before giving up to "file.bin"; the
|
||||
// magic-byte sniffer in downloadInboundMedia will fix the
|
||||
// extension either way, but having something user-readable
|
||||
// here keeps the bubble title meaningful.
|
||||
String filename = (String) fileBody.getOrDefault("filename",
|
||||
fileBody.getOrDefault("file_name",
|
||||
fileBody.getOrDefault("name", "file.bin")));
|
||||
String fileConvId = inboundConversationId(senderId, chatId, chatType);
|
||||
if (!url.isBlank()) {
|
||||
MessageContentPart filePart = buildInboundFilePart(
|
||||
url, aesKey, msgId, filename, fileConvId);
|
||||
contentParts.add(filePart);
|
||||
// Surface the corrected filename (with proper extension)
|
||||
// back into the [文件: X] text marker the agent sees.
|
||||
if (filePart.getFileName() != null && !filePart.getFileName().isBlank()) {
|
||||
filename = filePart.getFileName();
|
||||
}
|
||||
}
|
||||
textContent = "[文件: " + filename + "]";
|
||||
@ -948,15 +981,10 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
Map<String, Object> img = (Map<String, Object>) item.getOrDefault("image", Map.of());
|
||||
String url = (String) img.getOrDefault("url", "");
|
||||
String aesKey = (String) img.getOrDefault("aeskey", "");
|
||||
if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) {
|
||||
String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "mixed_image.jpg");
|
||||
if (localPath != null) {
|
||||
contentParts.add(MessageContentPart.image(localPath, url));
|
||||
} else {
|
||||
contentParts.add(MessageContentPart.image(url, url));
|
||||
}
|
||||
} else if (!url.isBlank()) {
|
||||
contentParts.add(MessageContentPart.image(url, url));
|
||||
String mixedConvId = inboundConversationId(senderId, chatId, chatType);
|
||||
if (!url.isBlank()) {
|
||||
contentParts.add(buildInboundImagePart(
|
||||
url, aesKey, msgId, "mixed_image.jpg", mixedConvId));
|
||||
}
|
||||
} else if ("voice".equals(itemType)) {
|
||||
Map<String, Object> v = (Map<String, Object>) item.getOrDefault("voice", Map.of());
|
||||
@ -1243,6 +1271,15 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
keepaliveScheduler.stop(ctx.processingStreamId());
|
||||
}
|
||||
|
||||
// Sniff `/api/v1/files/generated/{id}` URLs out of the agent's text
|
||||
// BEFORE rendering. Each hit gets upgraded to a native WeCom file
|
||||
// message via the chunked upload API; the URL in the text is replaced
|
||||
// with a "📎 filename" marker so the bubble doesn't repeat itself.
|
||||
// Without this, a generated docx/pptx would arrive as a markdown link
|
||||
// the user can't open inside WeCom (no public access + JWT required).
|
||||
List<UploadJob> uploadJobs = new ArrayList<>();
|
||||
String rewrittenContent = sniffGeneratedFiles(content, uploadJobs);
|
||||
|
||||
// 先进行正常的内容渲染（过滤 thinking、分割长文本）
|
||||
boolean filterThinking = getConfigBoolean("filter_thinking", true);
|
||||
boolean filterToolMessages = getConfigBoolean("filter_tool_messages", true);
|
||||
@ -1250,7 +1287,7 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
int maxLen = vip.mate.channel.ChannelMessageRenderer.PLATFORM_LIMITS.getOrDefault(getChannelType(), 2048);
|
||||
|
||||
List<String> segments = vip.mate.channel.ChannelMessageRenderer.renderForChannel(
|
||||
content, filterThinking, filterToolMessages, format, maxLen);
|
||||
rewrittenContent, filterThinking, filterToolMessages, format, maxLen);
|
||||
|
||||
boolean first = true;
|
||||
for (String rawSegment : segments) {
|
||||
@ -1265,6 +1302,89 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
sendMessage(targetId, segment);
|
||||
}
|
||||
}
|
||||
|
||||
// Upload + dispatch any generated files we sniffed out. Done after the
|
||||
// text bubble so the order in the IM client mirrors the markdown:
|
||||
// explanatory text first, then the actual file card the user can tap.
|
||||
// Use the original frameReqId once for the first attachment (so it
|
||||
// rides the reply path) and active-push for the rest.
|
||||
if (!uploadJobs.isEmpty()) {
|
||||
String frameReqId = ctx != null ? ctx.frameReqId() : null;
|
||||
for (int i = 0; i < uploadJobs.size(); i++) {
|
||||
UploadJob job = uploadJobs.get(i);
|
||||
String mediaId = uploadMedia(job.bytes(), job.fileName(), job.mediaType());
|
||||
if (mediaId == null) {
|
||||
log.warn("[wecom] Generated-file upload failed: {} ({} bytes)",
|
||||
job.fileName(), job.bytes().length);
|
||||
continue;
|
||||
}
|
||||
// Only the first attachment can use the inbound frameReqId
|
||||
// reply slot; subsequent attachments must go via active-push.
|
||||
String replyReqId = (i == 0) ? frameReqId : null;
|
||||
sendMediaMessage(targetId, mediaId, job.mediaType(), replyReqId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Carries one to-be-uploaded generated file from {@link #sniffGeneratedFiles}. */
|
||||
private record UploadJob(byte[] bytes, String fileName, String mediaType) {}
|
||||
|
||||
/**
|
||||
* URL pattern for the in-memory generated-file cache served by
|
||||
* {@code GeneratedFileController}. Lives in the channel layer because
|
||||
* each adapter rewrites the URL to a channel-native attachment.
|
||||
*/
|
||||
private static final java.util.regex.Pattern GENERATED_URL_PATTERN =
|
||||
java.util.regex.Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)");
|
||||
|
||||
/**
|
||||
* Scan the agent's text for {@code /api/v1/files/generated/{id}} URLs;
|
||||
* for each hit, look up the cached bytes and queue an {@link UploadJob}.
|
||||
* Replaces the URL in the returned text with a "📎 filename" marker so
|
||||
* the bubble shows the file name without dangling an unopenable link.
|
||||
* Cache misses (entry expired or never existed) leave the URL untouched
|
||||
* — the user can still try clicking from the Web mirror's history view.
|
||||
*/
|
||||
private String sniffGeneratedFiles(String text, List<UploadJob> jobs) {
|
||||
if (text == null || text.isEmpty() || generatedFileCache == null) return text;
|
||||
java.util.regex.Matcher m = GENERATED_URL_PATTERN.matcher(text);
|
||||
StringBuilder out = new StringBuilder();
|
||||
while (m.find()) {
|
||||
String id = m.group(1);
|
||||
var entry = generatedFileCache.get(id).orElse(null);
|
||||
if (entry != null) {
|
||||
String mediaType = isImageMime(entry.mimeType()) ? "image" : "file";
|
||||
jobs.add(new UploadJob(entry.bytes(), entry.filename(), mediaType));
|
||||
m.appendReplacement(out,
|
||||
java.util.regex.Matcher.quoteReplacement("📎 " + entry.filename()));
|
||||
} else {
|
||||
// Cache miss has two real-world causes, both surfaced with
|
||||
// the same retry hint so the user just resubmits:
|
||||
// 1) LLM hallucinated a UUID-shaped string instead of
|
||||
// calling a render tool — IDs like
|
||||
// "a1b2c3d4-e5f6-7890-abcd-ef1234567890" with sequential
|
||||
// hex are textbook fakes. {@code GeneratedFileCache}
|
||||
// logs every real {@code put}, so its absence here is
|
||||
// proof the file was never generated this turn.
|
||||
// 2) Cache entry expired (10-min TTL) before the IM
|
||||
// client got around to clicking, or was wiped on
|
||||
// JVM restart.
|
||||
// Without this replacement, users tap a markdown link that
|
||||
// returns 404 and the IM client saves the error body as a
|
||||
// ".docx" — they then "open" what is actually an HTML 404
|
||||
// page and report "file is corrupted".
|
||||
log.warn("[wecom] Generated-file cache miss for id={} — likely LLM skipped the render tool and wrote a fake URL (toolCallCount=0 in this turn). Bubble will show retry hint.",
|
||||
id);
|
||||
m.appendReplacement(out, java.util.regex.Matcher.quoteReplacement(
|
||||
"⚠️ 文件未真正生成（模型未调用文档生成工具），请重新发送请求"));
|
||||
}
|
||||
}
|
||||
m.appendTail(out);
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
private static boolean isImageMime(String mimeType) {
|
||||
return mimeType != null && mimeType.toLowerCase().startsWith("image/");
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -1276,19 +1396,28 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
boolean sentText = false;
|
||||
boolean firstText = true;
|
||||
|
||||
// Mirror renderAndSend's sniff so contentParts-mode replies also
|
||||
// upgrade /api/v1/files/generated/{id} URLs into native WeCom file
|
||||
// attachments. Text parts get the URL replaced with "📎 filename";
|
||||
// the actual bytes are queued for native upload after all parts are
|
||||
// dispatched (preserves the "text first, attachments after" ordering).
|
||||
List<UploadJob> uploadJobs = new ArrayList<>();
|
||||
|
||||
for (MessageContentPart part : parts) {
|
||||
if (part == null) continue;
|
||||
try {
|
||||
switch (part.getType()) {
|
||||
case "text" -> {
|
||||
if (part.getText() != null && !part.getText().isBlank()) {
|
||||
String txt = part.getText();
|
||||
if (txt != null && !txt.isBlank()) {
|
||||
String rewritten = sniffGeneratedFiles(txt, uploadJobs);
|
||||
// 第一条文本用 processingStreamId 覆盖"思考中..."
|
||||
if (firstText && ctx != null && ctx.processingStreamId() != null
|
||||
&& !ctx.processingStreamId().isBlank()) {
|
||||
replyStream(ctx.frameReqId(), ctx.processingStreamId(), part.getText(), true);
|
||||
replyStream(ctx.frameReqId(), ctx.processingStreamId(), rewritten, true);
|
||||
firstText = false;
|
||||
} else {
|
||||
sendMessage(targetId, part.getText());
|
||||
sendMessage(targetId, rewritten);
|
||||
}
|
||||
sentText = true;
|
||||
}
|
||||
@ -1326,6 +1455,26 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
log.debug("[wecom] Failed to clear processing indicator: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// After all parts are dispatched, upload any generated files we
|
||||
// sniffed out of text parts. First attachment rides the inbound
|
||||
// frameReqId reply slot; subsequent ones go via active-push. Mirror
|
||||
// the ordering used in renderAndSend so the user sees text bubble
|
||||
// first, then the actual file card.
|
||||
if (!uploadJobs.isEmpty()) {
|
||||
String frameReqId = ctx != null ? ctx.frameReqId() : null;
|
||||
for (int i = 0; i < uploadJobs.size(); i++) {
|
||||
UploadJob job = uploadJobs.get(i);
|
||||
String mediaId = uploadMedia(job.bytes(), job.fileName(), job.mediaType());
|
||||
if (mediaId == null) {
|
||||
log.warn("[wecom] Generated-file upload failed: {} ({} bytes)",
|
||||
job.fileName(), job.bytes().length);
|
||||
continue;
|
||||
}
|
||||
String replyReqId = (i == 0) ? frameReqId : null;
|
||||
sendMediaMessage(targetId, mediaId, job.mediaType(), replyReqId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1747,7 +1896,13 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
Map<String, Object> chunkBody = new LinkedHashMap<>();
|
||||
chunkBody.put("upload_id", uploadId);
|
||||
chunkBody.put("chunk_index", i);
|
||||
chunkBody.put("data", base64Data);
|
||||
// Field name MUST be "base64_data" — the WeCom AI bot upload
|
||||
// server reads the chunk bytes from this exact key. A previous
|
||||
// version sent "data" which the server silently dropped, so
|
||||
// metadata (filename/size) committed but the bytes never made
|
||||
// it to storage. Receivers then saw the file with the right
|
||||
// name/size but couldn't open it ("文件已损坏").
|
||||
chunkBody.put("base64_data", base64Data);
|
||||
|
||||
Map<String, Object> chunkFrame = Map.of(
|
||||
"cmd", CMD_UPLOAD_CHUNK,
|
||||
@ -2089,7 +2244,383 @@ public class WeComChannelAdapter extends AbstractChannelAdapter {
|
||||
// ==================== 媒体文件下载与 AES 解密 ====================
|
||||
|
||||
/**
|
||||
* 下载并解密企业微信媒体文件
|
||||
* Conversation id for inbound media: matches the format
|
||||
* {@code ChannelMessageRouter.buildConversationId} produces from the same
|
||||
* (channelType, chatId, senderId) tuple. Pre-computing it here lets the
|
||||
* media-download helper write into the right per-conversation directory
|
||||
* <em>before</em> the {@link ChannelMessage} is built.
|
||||
*/
|
||||
private static String inboundConversationId(String senderId, String chatId, String chatType) {
|
||||
boolean isGroup = "group".equals(chatType);
|
||||
return isGroup ? "wecom:group:" + chatId : "wecom:" + senderId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a fully-populated image content part for inbound WeCom media.
|
||||
* <p>
|
||||
* When download is enabled and succeeds, the part carries
|
||||
* {@code path}, {@code fileUrl} (browser-servable), {@code fileName},
|
||||
* {@code storedName}, {@code fileSize}, and a precise {@code contentType}
|
||||
* — every field the chat bubble and the multimodal sidecar inspect. When
|
||||
* download is disabled or fails, the part falls back to URL-only fields
|
||||
* but still sets {@code fileName} so the bubble doesn't render
|
||||
* "未命名 / unknown".
|
||||
*/
|
||||
private MessageContentPart buildInboundImagePart(String url, String aesKey, String msgId,
|
||||
String fileNameHint, String conversationId) {
|
||||
if (getConfigBoolean("media_download_enabled", true)) {
|
||||
InboundMediaResult r = downloadInboundMedia(url, aesKey, msgId, fileNameHint, conversationId);
|
||||
if (r != null) {
|
||||
MessageContentPart part = new MessageContentPart();
|
||||
part.setType("image");
|
||||
part.setFileName(r.fileName());
|
||||
part.setStoredName(r.storedName());
|
||||
part.setPath(r.localPath());
|
||||
part.setFileUrl(r.fileUrl());
|
||||
part.setFileSize(r.fileSize());
|
||||
// Prefer the sniffed contentType (could be image/png) over a
|
||||
// hardcoded image/jpeg. Falls back to image/jpeg only when the
|
||||
// sniff was inconclusive.
|
||||
String ct = r.contentType();
|
||||
part.setContentType((ct != null && ct.startsWith("image/")) ? ct : "image/jpeg");
|
||||
// mediaId mirrors path so callers that prefer it still resolve
|
||||
// to the same on-disk file (matches Web upload's behaviour).
|
||||
part.setMediaId(r.localPath());
|
||||
return part;
|
||||
}
|
||||
}
|
||||
// Fallback: download disabled or failed. Browser preview will be broken
|
||||
// because the WeCom CDN URL carries a short-lived signature, but at
|
||||
// least the bubble shows "image.jpg" instead of "未命名 / unknown".
|
||||
MessageContentPart part = new MessageContentPart();
|
||||
part.setType("image");
|
||||
part.setFileName(fileNameHint);
|
||||
part.setFileUrl(url);
|
||||
part.setMediaId(url);
|
||||
part.setContentType("image/jpeg");
|
||||
return part;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a fully-populated file content part for inbound WeCom media.
|
||||
* <p>
|
||||
* Mirrors {@link #buildInboundImagePart} but for non-image attachments
|
||||
* (PDF, DOCX, ZIP, etc.). The magic-byte sniffer inside
|
||||
* {@link #downloadInboundMedia} fixes generic {@code file.bin} hints to
|
||||
* the real extension so downstream tools (PDF text extractor, magika,
|
||||
* etc.) key off the correct mime.
|
||||
*/
|
||||
private MessageContentPart buildInboundFilePart(String url, String aesKey, String msgId,
|
||||
String fileNameHint, String conversationId) {
|
||||
if (getConfigBoolean("media_download_enabled", true)) {
|
||||
InboundMediaResult r = downloadInboundMedia(url, aesKey, msgId, fileNameHint, conversationId);
|
||||
if (r != null) {
|
||||
MessageContentPart part = new MessageContentPart();
|
||||
part.setType("file");
|
||||
part.setFileName(r.fileName());
|
||||
part.setStoredName(r.storedName());
|
||||
part.setPath(r.localPath());
|
||||
part.setFileUrl(r.fileUrl());
|
||||
part.setFileSize(r.fileSize());
|
||||
part.setContentType(r.contentType());
|
||||
part.setMediaId(r.localPath());
|
||||
return part;
|
||||
}
|
||||
}
|
||||
// Fallback when download is disabled or fails — at least keep the
|
||||
// original hint so the bubble doesn't say "file.bin" for a PDF.
|
||||
MessageContentPart part = new MessageContentPart();
|
||||
part.setType("file");
|
||||
part.setFileName(fileNameHint);
|
||||
part.setFileUrl(url);
|
||||
part.setMediaId(url);
|
||||
return part;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inbound-media download result. Carries every field the bubble renderer
|
||||
* and the multimodal sidecar need so callers don't have to re-derive
|
||||
* storedName / fileUrl from scratch.
|
||||
*
|
||||
* @param localPath absolute filesystem path of the saved file
|
||||
* @param storedName the on-disk filename (matches the last segment of localPath)
|
||||
* @param fileUrl browser-servable URL: {@code /api/v1/chat/files/{convId}/{storedName}}
|
||||
* @param fileSize byte length after decryption
|
||||
* @param fileName human-readable display name (extension corrected by magic-byte sniff)
|
||||
* @param contentType MIME type derived from magic bytes (or {@code application/octet-stream})
|
||||
*/
|
||||
record InboundMediaResult(String localPath, String storedName,
|
||||
String fileUrl, long fileSize, String fileName,
|
||||
String contentType) {}
|
||||
|
||||
/** Magic-byte sniff result. */
|
||||
private record MagicSniff(String extension, String contentType) {
|
||||
static final MagicSniff UNKNOWN = new MagicSniff(".bin", "application/octet-stream");
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort MIME sniff from the first 12 bytes of a file. Covers the
|
||||
* formats users routinely forward to bots (PDF, Office, archives, common
|
||||
* image / audio / video). When nothing matches, returns
|
||||
* {@link MagicSniff#UNKNOWN} so the caller falls back to {@code .bin}.
|
||||
* <p>
|
||||
* This exists because WeCom's {@code aibot_msg_callback} {@code file}
|
||||
* body sometimes omits {@code filename} entirely (forwarded files in
|
||||
* particular), and shipping the agent a part labelled {@code file.bin}
|
||||
* makes downstream tools mis-route the content. Sniffing recovers a
|
||||
* useful extension so PDF tools fire on PDFs.
|
||||
*/
|
||||
private static MagicSniff sniffMagic(byte[] head) {
|
||||
if (head == null || head.length < 4) return MagicSniff.UNKNOWN;
|
||||
// PDF: %PDF
|
||||
if (head[0] == 0x25 && head[1] == 0x50 && head[2] == 0x44 && head[3] == 0x46) {
|
||||
return new MagicSniff(".pdf", "application/pdf");
|
||||
}
|
||||
// PNG: 89 50 4E 47
|
||||
if (head[0] == (byte) 0x89 && head[1] == 0x50 && head[2] == 0x4E && head[3] == 0x47) {
|
||||
return new MagicSniff(".png", "image/png");
|
||||
}
|
||||
// JPEG: FF D8 FF
|
||||
if (head[0] == (byte) 0xFF && head[1] == (byte) 0xD8 && head[2] == (byte) 0xFF) {
|
||||
return new MagicSniff(".jpg", "image/jpeg");
|
||||
}
|
||||
// GIF: "GIF8"
|
||||
if (head[0] == 0x47 && head[1] == 0x49 && head[2] == 0x46 && head[3] == 0x38) {
|
||||
return new MagicSniff(".gif", "image/gif");
|
||||
}
|
||||
// ZIP-based container: PK\x03\x04. Could be a plain ZIP, a JAR,
|
||||
// an OOXML document (DOCX/XLSX/PPTX), an ODF document (ODT/ODS/ODP),
|
||||
// or an EPUB. Magic-byte alone can't tell them apart — caller is
|
||||
// expected to follow up with refineZipKind(fullBytes) to pick a
|
||||
// specific type.
|
||||
if (head[0] == 0x50 && head[1] == 0x4B && head[2] == 0x03 && head[3] == 0x04) {
|
||||
return new MagicSniff(".zip", "application/zip");
|
||||
}
|
||||
// Legacy Office (DOC/XLS/PPT): D0 CF 11 E0 A1 B1 1A E1
|
||||
if (head.length >= 8
|
||||
&& head[0] == (byte) 0xD0 && head[1] == (byte) 0xCF
|
||||
&& head[2] == 0x11 && head[3] == (byte) 0xE0
|
||||
&& head[4] == (byte) 0xA1 && head[5] == (byte) 0xB1
|
||||
&& head[6] == 0x1A && head[7] == (byte) 0xE1) {
|
||||
return new MagicSniff(".doc", "application/msword");
|
||||
}
|
||||
// RTF: "{\rtf"
|
||||
if (head.length >= 5
|
||||
&& head[0] == 0x7B && head[1] == 0x5C
|
||||
&& head[2] == 0x72 && head[3] == 0x74 && head[4] == 0x66) {
|
||||
return new MagicSniff(".rtf", "application/rtf");
|
||||
}
|
||||
// 7z: 37 7A BC AF 27 1C
|
||||
if (head.length >= 6
|
||||
&& head[0] == 0x37 && head[1] == 0x7A && head[2] == (byte) 0xBC
|
||||
&& head[3] == (byte) 0xAF && head[4] == 0x27 && head[5] == 0x1C) {
|
||||
return new MagicSniff(".7z", "application/x-7z-compressed");
|
||||
}
|
||||
// RAR: "Rar!\x1A\x07"
|
||||
if (head.length >= 6
|
||||
&& head[0] == 0x52 && head[1] == 0x61 && head[2] == 0x72
|
||||
&& head[3] == 0x21 && head[4] == 0x1A && head[5] == 0x07) {
|
||||
return new MagicSniff(".rar", "application/x-rar-compressed");
|
||||
}
|
||||
// MP3: ID3v2 ("ID3") or MPEG sync 0xFFFB / 0xFFF3 / 0xFFF2
|
||||
if (head[0] == 0x49 && head[1] == 0x44 && head[2] == 0x33) {
|
||||
return new MagicSniff(".mp3", "audio/mpeg");
|
||||
}
|
||||
// MP4: "....ftyp" — bytes 4..7 == "ftyp"
|
||||
if (head.length >= 8
|
||||
&& head[4] == 0x66 && head[5] == 0x74 && head[6] == 0x79 && head[7] == 0x70) {
|
||||
return new MagicSniff(".mp4", "video/mp4");
|
||||
}
|
||||
// OGG: "OggS"
|
||||
if (head[0] == 0x4F && head[1] == 0x67 && head[2] == 0x67 && head[3] == 0x53) {
|
||||
return new MagicSniff(".ogg", "audio/ogg");
|
||||
}
|
||||
return MagicSniff.UNKNOWN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Peek inside a ZIP container to distinguish OOXML (DOCX/XLSX/PPTX),
|
||||
* ODF (ODT/ODS/ODP), JAR, and EPUB from a plain ZIP. Reads the local
|
||||
* file headers in order via {@link ZipInputStream}; the discriminator
|
||||
* entry is almost always within the first few entries (OOXML places
|
||||
* {@code [Content_Types].xml} first, ODF places {@code mimetype} first),
|
||||
* so we cap iteration at 16 entries to bound CPU.
|
||||
* <p>
|
||||
* Returns the original {@code zipDefault} sniff (plain
|
||||
* {@code application/zip}) when no specific kind is detected — that's
|
||||
* the right answer for actual ZIPs and unknown archive formats.
|
||||
*/
|
||||
private static MagicSniff refineZipKind(byte[] fileData, MagicSniff zipDefault) {
|
||||
if (fileData == null || fileData.length < 30) return zipDefault;
|
||||
String mimetypeContent = null;
|
||||
try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(fileData))) {
|
||||
ZipEntry entry;
|
||||
int seen = 0;
|
||||
while ((entry = zis.getNextEntry()) != null && seen < 16) {
|
||||
String name = entry.getName();
|
||||
// OOXML — Office Open XML (Word/Excel/PowerPoint). Each format
|
||||
// has a distinct top-level directory; we match on prefix
|
||||
// because the entry order isn't guaranteed.
|
||||
if (name.startsWith("word/")) {
|
||||
return new MagicSniff(".docx",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document");
|
||||
}
|
||||
if (name.startsWith("xl/")) {
|
||||
return new MagicSniff(".xlsx",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
}
|
||||
if (name.startsWith("ppt/")) {
|
||||
return new MagicSniff(".pptx",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation");
|
||||
}
|
||||
// Visio (rare but worth catching)
|
||||
if (name.startsWith("visio/")) {
|
||||
return new MagicSniff(".vsdx",
|
||||
"application/vnd.ms-visio.drawing");
|
||||
}
|
||||
// ODF marker: a {@code mimetype} entry that contains the full
|
||||
// application/vnd.oasis.opendocument.* string — read its body
|
||||
// and decide once we have it.
|
||||
if ("mimetype".equals(name)) {
|
||||
byte[] buf = zis.readAllBytes();
|
||||
mimetypeContent = new String(buf, java.nio.charset.StandardCharsets.UTF_8).trim();
|
||||
}
|
||||
// JAR
|
||||
if ("META-INF/MANIFEST.MF".equals(name)) {
|
||||
return new MagicSniff(".jar", "application/java-archive");
|
||||
}
|
||||
// EPUB always has META-INF/container.xml
|
||||
if ("META-INF/container.xml".equals(name)) {
|
||||
return new MagicSniff(".epub", "application/epub+zip");
|
||||
}
|
||||
seen++;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("[wecom] refineZipKind failed (treating as plain zip): {}", e.getMessage());
|
||||
return zipDefault;
|
||||
}
|
||||
if (mimetypeContent != null) {
|
||||
if (mimetypeContent.contains("opendocument.text")) {
|
||||
return new MagicSniff(".odt", "application/vnd.oasis.opendocument.text");
|
||||
}
|
||||
if (mimetypeContent.contains("opendocument.spreadsheet")) {
|
||||
return new MagicSniff(".ods", "application/vnd.oasis.opendocument.spreadsheet");
|
||||
}
|
||||
if (mimetypeContent.contains("opendocument.presentation")) {
|
||||
return new MagicSniff(".odp", "application/vnd.oasis.opendocument.presentation");
|
||||
}
|
||||
if (mimetypeContent.contains("epub")) {
|
||||
return new MagicSniff(".epub", "application/epub+zip");
|
||||
}
|
||||
}
|
||||
return zipDefault;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip a trailing extension from a filename. {@code "image.jpg" → "image"};
|
||||
* {@code "no_ext" → "no_ext"}; {@code "" → ""}.
|
||||
*/
|
||||
private static String stripExtension(String name) {
|
||||
if (name == null || name.isBlank()) return "";
|
||||
int dot = name.lastIndexOf('.');
|
||||
if (dot <= 0) return name;
|
||||
return name.substring(0, dot);
|
||||
}
|
||||
|
||||
/**
|
||||
* Download + decrypt an inbound media attachment and stash it under
|
||||
* {@code data/chat-uploads/{conversationId}/} so the existing
|
||||
* {@code /api/v1/chat/files/...} endpoint can serve it back to the chat
|
||||
* bubble. Returns a fully-populated {@link InboundMediaResult} on success
|
||||
* or null on download/decrypt failure (callers fall back to URL-only).
|
||||
* <p>
|
||||
* Storing under chat-uploads rather than {@code data/media} means
|
||||
* {@link MessageContentPart#getPath()} resolves to a real file for the
|
||||
* vision sidecar AND {@code fileUrl} renders as a thumbnail in the Web
|
||||
* mirror — instead of the WeCom-signed CDN URL whose 5-minute query-string
|
||||
* signature expires before the browser can fetch it.
|
||||
*/
|
||||
private InboundMediaResult downloadInboundMedia(String url, String aesKey, String msgId,
|
||||
String fileNameHint, String conversationId) {
|
||||
try {
|
||||
// Mirror ChatController.uploadRoot ("data/chat-uploads") so the
|
||||
// serve endpoint at /api/v1/chat/files/{convId}/{storedName} works
|
||||
// without any extra wiring. The conversationId may contain ':'
|
||||
// (e.g. "wecom:XuZhanFu" or "wecom:group:abc"); Path resolution
|
||||
// tolerates this on macOS/Linux but Windows would reject the
|
||||
// colon — for now we keep parity with the existing chat-uploads
|
||||
// layout and revisit if Windows support comes up.
|
||||
Path uploadDir = Path.of("data", "chat-uploads", conversationId);
|
||||
Files.createDirectories(uploadDir);
|
||||
|
||||
// 1. HTTP GET 下载文件
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(url))
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.GET()
|
||||
.build();
|
||||
|
||||
HttpResponse<InputStream> response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
|
||||
byte[] encryptedData = response.body().readAllBytes();
|
||||
|
||||
byte[] fileData;
|
||||
// 2. AES 解密（如果提供了 aesKey）
|
||||
if (aesKey != null && !aesKey.isBlank()) {
|
||||
fileData = decryptAes256Cbc(encryptedData, aesKey);
|
||||
} else {
|
||||
fileData = encryptedData;
|
||||
}
|
||||
|
||||
// 3. Magic-byte sniff to recover a real extension when WeCom
|
||||
// didn't include filename in the body (forwarded files often
|
||||
// arrive nameless — saving them as "file.bin" misroutes the
|
||||
// agent because every PDF tool keys off the .pdf extension).
|
||||
byte[] head = new byte[Math.min(12, fileData.length)];
|
||||
System.arraycopy(fileData, 0, head, 0, head.length);
|
||||
MagicSniff sniff = sniffMagic(head);
|
||||
// ZIP container needs a deeper look — DOCX/XLSX/PPTX/ODF/EPUB/JAR
|
||||
// all share the PK\x03\x04 magic. Peek inside the first few
|
||||
// entries to pick the specific kind.
|
||||
if (".zip".equals(sniff.extension())) {
|
||||
sniff = refineZipKind(fileData, sniff);
|
||||
}
|
||||
|
||||
// 4. Compose a URL-safe storedName. If the hint is generic
|
||||
// (e.g. "file.bin"), prefer the sniffed extension.
|
||||
String urlHash = md5Hex(url).substring(0, 8);
|
||||
String hintRaw = (fileNameHint == null ? "media" : fileNameHint).trim();
|
||||
String safeName = hintRaw.replaceAll("[^a-zA-Z0-9._-]", "_");
|
||||
if (safeName.isBlank()) safeName = "media";
|
||||
// "file.bin" is the WeCom-no-filename sentinel; if magic gave us
|
||||
// something better, replace the extension. Same when hint had no
|
||||
// extension at all.
|
||||
boolean hintIsGeneric = safeName.equals("file.bin") || safeName.equals("media")
|
||||
|| !safeName.contains(".");
|
||||
if (hintIsGeneric && !".bin".equals(sniff.extension())) {
|
||||
safeName = stripExtension(safeName) + sniff.extension();
|
||||
}
|
||||
String storedName = "wecom_" + urlHash + "_" + safeName;
|
||||
Path filePath = uploadDir.resolve(storedName);
|
||||
Files.write(filePath, fileData);
|
||||
|
||||
String fileUrl = "/api/v1/chat/files/" + conversationId + "/" + storedName;
|
||||
log.info("[wecom] Inbound media saved: {} ({} bytes, sniffed={}), serve URL={}",
|
||||
filePath, fileData.length, sniff.contentType(), fileUrl);
|
||||
return new InboundMediaResult(
|
||||
filePath.toAbsolutePath().toString(),
|
||||
storedName,
|
||||
fileUrl,
|
||||
fileData.length,
|
||||
safeName,
|
||||
sniff.contentType());
|
||||
} catch (Exception e) {
|
||||
log.error("[wecom] Failed to download inbound media: {}", e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 下载并解密企业微信媒体文件（旧版本，保留给 outbound / 其他场景使用）
|
||||
* <p>
|
||||
* AES-256-CBC 解密：base64 decode aesKey → IV = 前 16 字节 → PKCS#7 去填充
|
||||
*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user