mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 03:55:09 +08:00
feat(channel): forward async tool results to IM channels + slack file upload
This commit is contained in:
parent
134fa1a975
commit
f6b4f7e402
@ -0,0 +1,117 @@
|
|||||||
|
package vip.mate.channel;
|
||||||
|
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.channel.model.ChannelSessionEntity;
|
||||||
|
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Forward async-task results (image / video / music / 3D) generated by tool
|
||||||
|
* pipelines to the IM channel that originated the conversation.
|
||||||
|
* <p>
|
||||||
|
* Without this dispatcher, completion bytes only land in
|
||||||
|
* {@code mate_message} + a Web SSE broadcast — IM users (WeCom / DingTalk /
|
||||||
|
* Feishu / Telegram / etc.) see nothing arrive in their chat client because
|
||||||
|
* the tool pipeline doesn't know about channel adapters. This dispatcher
|
||||||
|
* closes that loop: look up the conversation's bound channel session, get
|
||||||
|
* the live adapter from {@link ChannelManager}, and call
|
||||||
|
* {@link ChannelAdapter#sendContentParts} so the same bytes ride the
|
||||||
|
* channel-native attachment protocol.
|
||||||
|
* <p>
|
||||||
|
* Web / webchat conversations are intentionally skipped because their SSE
|
||||||
|
* stream already carries the result; double-dispatching would render the
|
||||||
|
* image twice.
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
public class AsyncTaskMediaDispatcher {
|
||||||
|
|
||||||
|
private final ChannelSessionStore channelSessionStore;
|
||||||
|
private final ChannelManager channelManager;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Channel types that handle their own UX via SSE (no IM forward needed).
|
||||||
|
* Everything not in this set is treated as an IM channel and gets the
|
||||||
|
* generated parts pushed via the adapter.
|
||||||
|
*/
|
||||||
|
private static final Set<String> WEB_CHANNEL_TYPES = Set.of("web", "webchat");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Forward generated content parts to the IM channel bound to this
|
||||||
|
* conversation, if any.
|
||||||
|
* <p>
|
||||||
|
* Best-effort: missing session, missing adapter, or adapter exception
|
||||||
|
* are all logged at debug/warn and never propagate. The caller has
|
||||||
|
* already persisted the message to {@code mate_message} and broadcast
|
||||||
|
* to Web SSE before invoking this — IM forwarding is additive.
|
||||||
|
*
|
||||||
|
* @param conversationId the conversation id used by the agent (e.g.
|
||||||
|
* {@code wecom:XuZhanFu}, {@code dingtalk:cid_xxx},
|
||||||
|
* {@code conv_xxx} for Web)
|
||||||
|
* @param parts assistant content parts to dispatch (typically a
|
||||||
|
* single image / video / audio / file part)
|
||||||
|
*/
|
||||||
|
public void forwardToImIfBound(String conversationId, List<MessageContentPart> parts) {
|
||||||
|
if (conversationId == null || conversationId.isBlank() || parts == null || parts.isEmpty()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ChannelSessionEntity session = channelSessionStore.getSession(conversationId);
|
||||||
|
if (session == null) {
|
||||||
|
// Common case for Web-only conversations — the session is never
|
||||||
|
// populated because Web doesn't write to ChannelSessionStore.
|
||||||
|
log.debug("[async-forward] No channel session for conv={}, skipping IM forward",
|
||||||
|
conversationId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String channelType = session.getChannelType();
|
||||||
|
if (channelType == null || WEB_CHANNEL_TYPES.contains(channelType)) {
|
||||||
|
log.debug("[async-forward] conv={} is web-class ({}), skipping IM forward",
|
||||||
|
conversationId, channelType);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
Long channelId = session.getChannelId();
|
||||||
|
if (channelId == null) {
|
||||||
|
log.debug("[async-forward] conv={} session has no channelId, skipping",
|
||||||
|
conversationId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
ChannelAdapter adapter = channelManager.getAdapter(channelId).orElse(null);
|
||||||
|
if (adapter == null) {
|
||||||
|
log.warn("[async-forward] conv={} channelId={} has no live adapter (channel disabled?), skipping",
|
||||||
|
conversationId, channelId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String targetId = session.getTargetId();
|
||||||
|
if (targetId == null || targetId.isBlank()) {
|
||||||
|
log.warn("[async-forward] conv={} session has no targetId, skipping",
|
||||||
|
conversationId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
adapter.sendContentParts(targetId, parts);
|
||||||
|
log.info("[async-forward] Dispatched {} part(s) to {} adapter (conv={}, target={})",
|
||||||
|
parts.size(), channelType, conversationId, targetId);
|
||||||
|
} catch (UnsupportedOperationException uoe) {
|
||||||
|
// Adapter doesn't override sendContentParts — fall through to
|
||||||
|
// text-only fallback. Most adapters that handle media (wecom /
|
||||||
|
// feishu / dingtalk) override; the rest will stay text-only
|
||||||
|
// until they implement the part dispatcher.
|
||||||
|
log.info("[async-forward] {} adapter does not implement sendContentParts, skipping (conv={})",
|
||||||
|
channelType, conversationId);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("[async-forward] Failed to dispatch to {} adapter for conv={}: {}",
|
||||||
|
channelType, conversationId, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -7,6 +7,7 @@ import com.slack.api.bolt.AppConfig;
|
|||||||
import com.slack.api.bolt.socket_mode.SocketModeApp;
|
import com.slack.api.bolt.socket_mode.SocketModeApp;
|
||||||
import com.slack.api.methods.SlackApiException;
|
import com.slack.api.methods.SlackApiException;
|
||||||
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
|
import com.slack.api.methods.response.chat.ChatPostMessageResponse;
|
||||||
|
import com.slack.api.methods.response.files.FilesUploadV2Response;
|
||||||
import com.slack.api.model.event.MessageEvent;
|
import com.slack.api.model.event.MessageEvent;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import vip.mate.channel.AbstractChannelAdapter;
|
import vip.mate.channel.AbstractChannelAdapter;
|
||||||
@ -14,8 +15,16 @@ import vip.mate.channel.ChannelMessage;
|
|||||||
import vip.mate.channel.ChannelMessageRouter;
|
import vip.mate.channel.ChannelMessageRouter;
|
||||||
import vip.mate.channel.ExponentialBackoff;
|
import vip.mate.channel.ExponentialBackoff;
|
||||||
import vip.mate.channel.model.ChannelEntity;
|
import vip.mate.channel.model.ChannelEntity;
|
||||||
|
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.http.HttpClient;
|
||||||
|
import java.net.http.HttpRequest;
|
||||||
|
import java.net.http.HttpResponse;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.time.Duration;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@ -295,4 +304,199 @@ public class SlackChannelAdapter extends AbstractChannelAdapter {
|
|||||||
result = result.replaceAll("(?m)^#{1,6}\\s+(.+)$", "*$1*");
|
result = result.replaceAll("(?m)^#{1,6}\\s+(.+)$", "*$1*");
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lazily-initialised JDK HTTP client for fetching media bytes from
|
||||||
|
* fully-qualified {@code fileUrl} fields. Only used when
|
||||||
|
* {@link MessageContentPart#getPath()} isn't set.
|
||||||
|
*/
|
||||||
|
private volatile HttpClient httpClient;
|
||||||
|
|
||||||
|
private HttpClient getHttpClient() {
|
||||||
|
HttpClient hc = httpClient;
|
||||||
|
if (hc == null) {
|
||||||
|
synchronized (this) {
|
||||||
|
hc = httpClient;
|
||||||
|
if (hc == null) {
|
||||||
|
hc = HttpClient.newBuilder()
|
||||||
|
.connectTimeout(Duration.ofSeconds(10))
|
||||||
|
.build();
|
||||||
|
httpClient = hc;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hc;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dispatch a list of {@link MessageContentPart}s to Slack. Text parts
|
||||||
|
* fall through to the existing {@link #sendMessage} chat-post path;
|
||||||
|
* media parts (image / audio / video / file / model3d) ride
|
||||||
|
* {@code filesUploadV2} so users see a native file card with thumbnail
|
||||||
|
* preview rather than an unopenable markdown link.
|
||||||
|
* <p>
|
||||||
|
* Wired by {@link vip.mate.channel.AsyncTaskMediaDispatcher} so async
|
||||||
|
* tool results (image generation / video generation / etc.) reach
|
||||||
|
* Slack channels the same way they reach WeCom / DingTalk / Feishu.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void sendContentParts(String targetId, List<MessageContentPart> parts) {
|
||||||
|
if (parts == null || parts.isEmpty()) return;
|
||||||
|
|
||||||
|
for (MessageContentPart part : parts) {
|
||||||
|
if (part == null) continue;
|
||||||
|
String type = part.getType();
|
||||||
|
try {
|
||||||
|
switch (type == null ? "" : type) {
|
||||||
|
case "text", "thinking" -> {
|
||||||
|
String text = part.getText();
|
||||||
|
if (text != null && !text.isBlank()) {
|
||||||
|
sendMessage(targetId, text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "refusal" -> {
|
||||||
|
String text = part.getText();
|
||||||
|
if (text != null && !text.isBlank()) {
|
||||||
|
sendMessage(targetId, "⚠️ " + text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "image", "audio", "video", "file", "model3d" -> uploadFilePart(targetId, part);
|
||||||
|
default -> {
|
||||||
|
// Unknown part type — fall back to its text body if any.
|
||||||
|
if (part.getText() != null && !part.getText().isBlank()) {
|
||||||
|
sendMessage(targetId, part.getText());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("[slack] Failed to send {} part to {}: {}", type, targetId, e.getMessage());
|
||||||
|
sendFallbackText(targetId, part);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a media part to Slack via {@code files.uploadV2}. Resolves
|
||||||
|
* bytes from the part's local {@code path} first (set by image / video
|
||||||
|
* / music / 3D generation services), falling back to an HTTP fetch of
|
||||||
|
* a fully-qualified {@code fileUrl}. Returns silently after sending a
|
||||||
|
* markdown fallback if no bytes are recoverable.
|
||||||
|
*/
|
||||||
|
private void uploadFilePart(String targetId, MessageContentPart part) {
|
||||||
|
byte[] bytes = resolveBytes(part);
|
||||||
|
if (bytes == null || bytes.length == 0) {
|
||||||
|
log.warn("[slack] No bytes resolvable for {} part (path={}, fileUrl={}), falling back to text",
|
||||||
|
part.getType(), part.getPath(), part.getFileUrl());
|
||||||
|
sendFallbackText(targetId, part);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String botToken = getConfigString("bot_token");
|
||||||
|
if (botToken == null || botToken.isBlank()) {
|
||||||
|
log.warn("[slack] Missing bot_token, cannot upload {} part", part.getType());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
String filename = part.getFileName();
|
||||||
|
if (filename == null || filename.isBlank()) {
|
||||||
|
filename = defaultFilenameFor(part.getType());
|
||||||
|
}
|
||||||
|
String threadTs = lookupThreadTs(targetId);
|
||||||
|
// Slack derives the MIME from the filename's extension; passing
|
||||||
|
// contentType here is unnecessary and not part of the V2 API.
|
||||||
|
final String finalFilename = filename;
|
||||||
|
final byte[] finalBytes = bytes;
|
||||||
|
try {
|
||||||
|
FilesUploadV2Response response = slack.methods(botToken).filesUploadV2(req -> {
|
||||||
|
var b = req
|
||||||
|
.channel(targetId)
|
||||||
|
.fileData(finalBytes)
|
||||||
|
.filename(finalFilename);
|
||||||
|
if (threadTs != null && !threadTs.isBlank()) {
|
||||||
|
b.threadTs(threadTs);
|
||||||
|
}
|
||||||
|
return b;
|
||||||
|
});
|
||||||
|
if (!response.isOk()) {
|
||||||
|
log.warn("[slack] filesUploadV2 failed for {} ({}): {}",
|
||||||
|
finalFilename, finalBytes.length, response.getError());
|
||||||
|
sendFallbackText(targetId, part);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
log.info("[slack] Uploaded {} part ({} bytes) to {}", part.getType(), finalBytes.length, targetId);
|
||||||
|
} catch (IOException | SlackApiException e) {
|
||||||
|
log.warn("[slack] filesUploadV2 error for {}: {}", finalFilename, e.getMessage());
|
||||||
|
sendFallbackText(targetId, part);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the part's bytes from disk first (paths set by the generation
|
||||||
|
* services + the WeCom inbound pipeline both populate this), or fetch
|
||||||
|
* via HTTP when only an external {@code fileUrl} is available. Returns
|
||||||
|
* null when neither path nor URL yields bytes.
|
||||||
|
*/
|
||||||
|
private byte[] resolveBytes(MessageContentPart part) {
|
||||||
|
String path = part.getPath();
|
||||||
|
if (path != null && !path.isBlank()) {
|
||||||
|
try {
|
||||||
|
Path p = Path.of(path);
|
||||||
|
if (Files.exists(p)) {
|
||||||
|
return Files.readAllBytes(p);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("[slack] Reading local path failed ({}): {}", path, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
String url = part.getFileUrl();
|
||||||
|
if (url != null && (url.startsWith("http://") || url.startsWith("https://"))) {
|
||||||
|
try {
|
||||||
|
HttpRequest req = HttpRequest.newBuilder()
|
||||||
|
.uri(URI.create(url))
|
||||||
|
.timeout(Duration.ofSeconds(30))
|
||||||
|
.GET()
|
||||||
|
.build();
|
||||||
|
HttpResponse<byte[]> resp = getHttpClient().send(req, HttpResponse.BodyHandlers.ofByteArray());
|
||||||
|
if (resp.statusCode() == 200) {
|
||||||
|
return resp.body();
|
||||||
|
}
|
||||||
|
log.debug("[slack] HTTP fetch returned status {} for {}", resp.statusCode(), url);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("[slack] HTTP fetch failed for {}: {}", url, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String defaultFilenameFor(String type) {
|
||||||
|
return switch (type == null ? "" : type) {
|
||||||
|
case "image" -> "image.png";
|
||||||
|
case "audio" -> "audio.mp3";
|
||||||
|
case "video" -> "video.mp4";
|
||||||
|
case "model3d" -> "model.glb";
|
||||||
|
default -> "file.bin";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Thread-aware reply: same lookup pattern as {@link #sendMessage}. */
|
||||||
|
private String lookupThreadTs(String channelId) {
|
||||||
|
for (var entry : threadTsCache.entrySet()) {
|
||||||
|
if (entry.getKey().contains(channelId)) {
|
||||||
|
return entry.getValue();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Final fallback when both upload and resolve fail — keep the user
|
||||||
|
* informed instead of dropping the message silently. */
|
||||||
|
private void sendFallbackText(String targetId, MessageContentPart part) {
|
||||||
|
String url = part.getFileUrl();
|
||||||
|
String fileName = part.getFileName() != null ? part.getFileName() : part.getType();
|
||||||
|
if (url != null && !url.isBlank()) {
|
||||||
|
sendMessage(targetId, "📎 " + fileName + ": " + url);
|
||||||
|
} else {
|
||||||
|
sendMessage(targetId, "[" + fileName + "]");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import vip.mate.channel.AsyncTaskMediaDispatcher;
|
||||||
import vip.mate.channel.web.ChatStreamTracker;
|
import vip.mate.channel.web.ChatStreamTracker;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.system.service.SystemSettingService;
|
import vip.mate.system.service.SystemSettingService;
|
||||||
@ -39,6 +40,14 @@ public class ImageGenerationService {
|
|||||||
private final ImageFileDownloader fileDownloader;
|
private final ImageFileDownloader fileDownloader;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
private final ChatStreamTracker streamTracker;
|
private final ChatStreamTracker streamTracker;
|
||||||
|
/**
|
||||||
|
* Forward completion to the conversation's bound IM channel adapter so
|
||||||
|
* WeCom / DingTalk / Feishu / etc. users actually receive the generated
|
||||||
|
* image as a native attachment. Web SSE handling continues unchanged
|
||||||
|
* via {@link ChatStreamTracker} — the dispatcher is additive and skips
|
||||||
|
* Web channels to avoid double-rendering.
|
||||||
|
*/
|
||||||
|
private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher;
|
||||||
|
|
||||||
private static final String TASK_TYPE = "image_generation";
|
private static final String TASK_TYPE = "image_generation";
|
||||||
|
|
||||||
@ -180,7 +189,15 @@ public class ImageGenerationService {
|
|||||||
|
|
||||||
MessageContentPart imagePart = MessageContentPart.image(null, servingUrl);
|
MessageContentPart imagePart = MessageContentPart.image(null, servingUrl);
|
||||||
imagePart.setFileName(localPath.getFileName().toString());
|
imagePart.setFileName(localPath.getFileName().toString());
|
||||||
|
imagePart.setStoredName(localPath.getFileName().toString());
|
||||||
imagePart.setContentType("image/png");
|
imagePart.setContentType("image/png");
|
||||||
|
// Set absolute disk path so IM channel adapters can read the
|
||||||
|
// bytes locally instead of round-tripping through the
|
||||||
|
// /api/v1/chat/files endpoint (which would require auth).
|
||||||
|
imagePart.setPath(localPath.toAbsolutePath().toString());
|
||||||
|
try {
|
||||||
|
imagePart.setFileSize(java.nio.file.Files.size(localPath));
|
||||||
|
} catch (Exception ignored) { /* size is best-effort */ }
|
||||||
contentParts.add(imagePart);
|
contentParts.add(imagePart);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -207,6 +224,11 @@ public class ImageGenerationService {
|
|||||||
streamTracker.broadcastObject(conversationId, "async_task_completed", data);
|
streamTracker.broadcastObject(conversationId, "async_task_completed", data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Forward to the conversation's bound IM channel adapter so
|
||||||
|
// WeCom / DingTalk / Feishu etc. users receive a native attachment
|
||||||
|
// (the SSE broadcast above only reaches Web subscribers).
|
||||||
|
asyncTaskMediaDispatcher.forwardToImIfBound(conversationId, contentParts);
|
||||||
|
|
||||||
log.info("[ImageGen] Sync generation completed, {} image(s) saved for conversation {}",
|
log.info("[ImageGen] Sync generation completed, {} image(s) saved for conversation {}",
|
||||||
servingUrls.size(), conversationId);
|
servingUrls.size(), conversationId);
|
||||||
|
|
||||||
@ -248,17 +270,31 @@ public class ImageGenerationService {
|
|||||||
// 保存 assistant 消息
|
// 保存 assistant 消息
|
||||||
MessageContentPart imagePart = MessageContentPart.image(null, servingUrl);
|
MessageContentPart imagePart = MessageContentPart.image(null, servingUrl);
|
||||||
imagePart.setFileName(localPath.getFileName().toString());
|
imagePart.setFileName(localPath.getFileName().toString());
|
||||||
|
imagePart.setStoredName(localPath.getFileName().toString());
|
||||||
imagePart.setContentType("image/png");
|
imagePart.setContentType("image/png");
|
||||||
|
// Set absolute disk path so IM channel adapters can read the
|
||||||
|
// bytes locally instead of round-tripping through the
|
||||||
|
// /api/v1/chat/files endpoint (which would require auth).
|
||||||
|
imagePart.setPath(localPath.toAbsolutePath().toString());
|
||||||
|
try {
|
||||||
|
imagePart.setFileSize(java.nio.file.Files.size(localPath));
|
||||||
|
} catch (Exception ignored) { /* size is best-effort */ }
|
||||||
|
|
||||||
|
List<MessageContentPart> parts = List.of(imagePart);
|
||||||
conversationService.saveMessage(
|
conversationService.saveMessage(
|
||||||
task.getConversationId(), "assistant",
|
task.getConversationId(), "assistant",
|
||||||
"图片已生成完毕",
|
"图片已生成完毕",
|
||||||
List.of(imagePart), "completed");
|
parts, "completed");
|
||||||
|
|
||||||
// SSE 广播(使用 imageUrl 字段)
|
// SSE 广播(使用 imageUrl 字段)
|
||||||
asyncTaskService.broadcastTaskEvent(task, "async_task_completed",
|
asyncTaskService.broadcastTaskEvent(task, "async_task_completed",
|
||||||
true, null, servingUrl, null);
|
true, null, servingUrl, null);
|
||||||
|
|
||||||
|
// Forward to the conversation's bound IM channel adapter so
|
||||||
|
// WeCom / DingTalk / Feishu etc. users receive a native
|
||||||
|
// attachment (the SSE broadcast above only reaches Web).
|
||||||
|
asyncTaskMediaDispatcher.forwardToImIfBound(task.getConversationId(), parts);
|
||||||
|
|
||||||
log.info("[ImageGen] Task {} completed, image saved: {}", task.getTaskId(), servingUrl);
|
log.info("[ImageGen] Task {} completed, image saved: {}", task.getTaskId(), servingUrl);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[ImageGen] Completion handling failed for task {}: {}",
|
log.error("[ImageGen] Completion handling failed for task {}: {}",
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import vip.mate.channel.AsyncTaskMediaDispatcher;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.system.service.SystemSettingService;
|
import vip.mate.system.service.SystemSettingService;
|
||||||
import vip.mate.task.AsyncTaskService;
|
import vip.mate.task.AsyncTaskService;
|
||||||
@ -44,6 +45,12 @@ public class Model3dGenerationService {
|
|||||||
private final ConversationService conversationService;
|
private final ConversationService conversationService;
|
||||||
private final Model3dFileDownloader fileDownloader;
|
private final Model3dFileDownloader fileDownloader;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
/**
|
||||||
|
* Forward async-task completion to the conversation's bound IM channel
|
||||||
|
* adapter so users on WeCom / DingTalk / Feishu / etc. receive the
|
||||||
|
* generated 3D model as a native attachment. SSE remains the Web path.
|
||||||
|
*/
|
||||||
|
private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher;
|
||||||
|
|
||||||
private static final String TASK_TYPE = "model3d_generation";
|
private static final String TASK_TYPE = "model3d_generation";
|
||||||
|
|
||||||
@ -173,6 +180,7 @@ public class Model3dGenerationService {
|
|||||||
String fileName = localPath.getFileName().toString();
|
String fileName = localPath.getFileName().toString();
|
||||||
MessageContentPart modelPart = MessageContentPart.model3d(null, fileName);
|
MessageContentPart modelPart = MessageContentPart.model3d(null, fileName);
|
||||||
modelPart.setFileUrl(servingUrl);
|
modelPart.setFileUrl(servingUrl);
|
||||||
|
modelPart.setStoredName(fileName);
|
||||||
// model/gltf-binary for .glb is the iana-registered MIME; downstream
|
// model/gltf-binary for .glb is the iana-registered MIME; downstream
|
||||||
// <model-viewer> only cares about the URL, not the MIME header.
|
// <model-viewer> only cares about the URL, not the MIME header.
|
||||||
if (fileName.endsWith(".glb")) {
|
if (fileName.endsWith(".glb")) {
|
||||||
@ -184,11 +192,18 @@ public class Model3dGenerationService {
|
|||||||
} else if (fileName.endsWith(".usdz")) {
|
} else if (fileName.endsWith(".usdz")) {
|
||||||
modelPart.setContentType("model/vnd.usdz+zip");
|
modelPart.setContentType("model/vnd.usdz+zip");
|
||||||
}
|
}
|
||||||
|
// Set absolute disk path so IM adapters can read bytes locally
|
||||||
|
// instead of round-tripping through /api/v1/chat/files (auth).
|
||||||
|
modelPart.setPath(localPath.toAbsolutePath().toString());
|
||||||
|
try {
|
||||||
|
modelPart.setFileSize(java.nio.file.Files.size(localPath));
|
||||||
|
} catch (Exception ignored) { /* best-effort */ }
|
||||||
|
|
||||||
|
List<MessageContentPart> parts = List.of(modelPart);
|
||||||
conversationService.saveMessage(
|
conversationService.saveMessage(
|
||||||
task.getConversationId(), "assistant",
|
task.getConversationId(), "assistant",
|
||||||
"3D 模型已生成完毕",
|
"3D 模型已生成完毕",
|
||||||
List.of(modelPart), "completed");
|
parts, "completed");
|
||||||
|
|
||||||
Map<String, Object> extra = new LinkedHashMap<>();
|
Map<String, Object> extra = new LinkedHashMap<>();
|
||||||
extra.put("modelUrl", servingUrl);
|
extra.put("modelUrl", servingUrl);
|
||||||
@ -196,6 +211,14 @@ public class Model3dGenerationService {
|
|||||||
asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed",
|
asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed",
|
||||||
true, extra, null);
|
true, extra, null);
|
||||||
|
|
||||||
|
// Forward to the conversation's bound IM channel adapter so
|
||||||
|
// WeCom / DingTalk / Feishu etc. users receive the model as a
|
||||||
|
// native attachment (the SSE broadcast above only reaches Web).
|
||||||
|
// Most IM channels will fall back to a markdown link via
|
||||||
|
// sendFallbackText if their adapter doesn't natively support
|
||||||
|
// model/* media — that's fine, the dispatcher logs and continues.
|
||||||
|
asyncTaskMediaDispatcher.forwardToImIfBound(task.getConversationId(), parts);
|
||||||
|
|
||||||
log.info("[Model3dGen] Task {} completed, model saved: {}", task.getTaskId(), servingUrl);
|
log.info("[Model3dGen] Task {} completed, model saved: {}", task.getTaskId(), servingUrl);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[Model3dGen] Completion handling failed for task {}: {}",
|
log.error("[Model3dGen] Completion handling failed for task {}: {}",
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import vip.mate.channel.AsyncTaskMediaDispatcher;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.system.service.SystemSettingService;
|
import vip.mate.system.service.SystemSettingService;
|
||||||
import vip.mate.task.AsyncTaskService;
|
import vip.mate.task.AsyncTaskService;
|
||||||
@ -48,6 +49,12 @@ public class MusicGenerationService {
|
|||||||
private final AsyncTaskService asyncTaskService;
|
private final AsyncTaskService asyncTaskService;
|
||||||
private final ConversationService conversationService;
|
private final ConversationService conversationService;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
/**
|
||||||
|
* Forward async-task completion to the conversation's bound IM channel
|
||||||
|
* adapter so WeCom / DingTalk / Feishu / etc. users receive the generated
|
||||||
|
* audio as a native attachment. Web-class channels keep using SSE only.
|
||||||
|
*/
|
||||||
|
private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher;
|
||||||
|
|
||||||
private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads");
|
private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads");
|
||||||
private static final String TASK_TYPE = "music_generation";
|
private static final String TASK_TYPE = "music_generation";
|
||||||
@ -130,9 +137,10 @@ public class MusicGenerationService {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
String audioUrl = persistAudio(conversationId, task.getTaskId(), result);
|
PersistedAudio persisted = persistAudio(conversationId, task.getTaskId(), result);
|
||||||
|
String audioUrl = persisted.servingUrl();
|
||||||
|
|
||||||
saveAssistantMessage(conversationId, audioUrl, result);
|
List<MessageContentPart> parts = saveAssistantMessage(conversationId, persisted, result);
|
||||||
|
|
||||||
ObjectNode resultJson = objectMapper.createObjectNode();
|
ObjectNode resultJson = objectMapper.createObjectNode();
|
||||||
resultJson.put("audioUrl", audioUrl);
|
resultJson.put("audioUrl", audioUrl);
|
||||||
@ -152,6 +160,11 @@ public class MusicGenerationService {
|
|||||||
asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed",
|
asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed",
|
||||||
true, extra, null);
|
true, extra, null);
|
||||||
|
|
||||||
|
// Forward to the conversation's bound IM channel adapter so
|
||||||
|
// WeCom / DingTalk / Feishu etc. users receive the audio as a
|
||||||
|
// native attachment (the SSE broadcast above only reaches Web).
|
||||||
|
asyncTaskMediaDispatcher.forwardToImIfBound(conversationId, parts);
|
||||||
|
|
||||||
log.info("[Music] Task {} succeeded, audio at {}", task.getTaskId(), audioUrl);
|
log.info("[Music] Task {} succeeded, audio at {}", task.getTaskId(), audioUrl);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[Music] Task {} worker failed: {}", task.getTaskId(), e.getMessage(), e);
|
log.error("[Music] Task {} worker failed: {}", task.getTaskId(), e.getMessage(), e);
|
||||||
@ -167,29 +180,48 @@ public class MusicGenerationService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String persistAudio(String conversationId, String taskId,
|
/**
|
||||||
MusicGenerationResult result) throws IOException {
|
* Stash audio bytes on disk and surface both the absolute local path and
|
||||||
|
* the browser-servable URL so callers can hand both to the
|
||||||
|
* {@link MessageContentPart}. The local path is what IM channel adapters
|
||||||
|
* read directly (faster, no auth round-trip); the serving URL is what
|
||||||
|
* the Web bubble renders.
|
||||||
|
*/
|
||||||
|
private record PersistedAudio(Path localPath, String servingUrl, String fileName) {}
|
||||||
|
|
||||||
|
private PersistedAudio persistAudio(String conversationId, String taskId,
|
||||||
|
MusicGenerationResult result) throws IOException {
|
||||||
Path dir = UPLOAD_ROOT.resolve(conversationId);
|
Path dir = UPLOAD_ROOT.resolve(conversationId);
|
||||||
Files.createDirectories(dir);
|
Files.createDirectories(dir);
|
||||||
String fileName = "music_" + taskId + "." + result.getFormat();
|
String fileName = "music_" + taskId + "." + result.getFormat();
|
||||||
Path filePath = dir.resolve(fileName);
|
Path filePath = dir.resolve(fileName);
|
||||||
Files.write(filePath, result.getAudioData());
|
Files.write(filePath, result.getAudioData());
|
||||||
return "/api/v1/chat/files/" + conversationId + "/" + fileName;
|
String servingUrl = "/api/v1/chat/files/" + conversationId + "/" + fileName;
|
||||||
|
return new PersistedAudio(filePath, servingUrl, fileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void saveAssistantMessage(String conversationId, String audioUrl,
|
private List<MessageContentPart> saveAssistantMessage(String conversationId,
|
||||||
MusicGenerationResult result) {
|
PersistedAudio persisted,
|
||||||
MessageContentPart audioPart = MessageContentPart.audio(null,
|
MusicGenerationResult result) {
|
||||||
audioUrl.substring(audioUrl.lastIndexOf('/') + 1));
|
MessageContentPart audioPart = MessageContentPart.audio(null, persisted.fileName());
|
||||||
audioPart.setFileUrl(audioUrl);
|
audioPart.setFileUrl(persisted.servingUrl());
|
||||||
|
audioPart.setStoredName(persisted.fileName());
|
||||||
audioPart.setContentType(result.getContentType());
|
audioPart.setContentType(result.getContentType());
|
||||||
|
// Set absolute disk path so IM adapters can read bytes locally
|
||||||
|
// instead of round-tripping through /api/v1/chat/files (auth).
|
||||||
|
audioPart.setPath(persisted.localPath().toAbsolutePath().toString());
|
||||||
|
try {
|
||||||
|
audioPart.setFileSize(Files.size(persisted.localPath()));
|
||||||
|
} catch (Exception ignored) { /* best-effort */ }
|
||||||
|
|
||||||
StringBuilder content = new StringBuilder("音乐生成完成");
|
StringBuilder content = new StringBuilder("音乐生成完成");
|
||||||
if (result.getLyrics() != null && !result.getLyrics().isBlank()) {
|
if (result.getLyrics() != null && !result.getLyrics().isBlank()) {
|
||||||
content.append("\n\n歌词:\n").append(result.getLyrics());
|
content.append("\n\n歌词:\n").append(result.getLyrics());
|
||||||
}
|
}
|
||||||
|
List<MessageContentPart> parts = List.of(audioPart);
|
||||||
conversationService.saveMessage(conversationId, "assistant",
|
conversationService.saveMessage(conversationId, "assistant",
|
||||||
content.toString(), List.of(audioPart), "completed");
|
content.toString(), parts, "completed");
|
||||||
|
return parts;
|
||||||
}
|
}
|
||||||
|
|
||||||
private MusicGenerationResult generateWithFallback(MusicGenerationRequest request,
|
private MusicGenerationResult generateWithFallback(MusicGenerationRequest request,
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
|
|||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
|
import vip.mate.channel.AsyncTaskMediaDispatcher;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.system.service.SystemSettingService;
|
import vip.mate.system.service.SystemSettingService;
|
||||||
import vip.mate.task.AsyncTaskService;
|
import vip.mate.task.AsyncTaskService;
|
||||||
@ -34,6 +35,12 @@ public class VideoGenerationService {
|
|||||||
private final ConversationService conversationService;
|
private final ConversationService conversationService;
|
||||||
private final VideoFileDownloader fileDownloader;
|
private final VideoFileDownloader fileDownloader;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
/**
|
||||||
|
* Forward async-task completion to the conversation's bound IM channel
|
||||||
|
* adapter so WeCom / DingTalk / Feishu / etc. users actually receive the
|
||||||
|
* generated video as a native attachment. SSE remains the Web path.
|
||||||
|
*/
|
||||||
|
private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher;
|
||||||
|
|
||||||
private static final String TASK_TYPE = "video_generation";
|
private static final String TASK_TYPE = "video_generation";
|
||||||
|
|
||||||
@ -176,19 +183,33 @@ public class VideoGenerationService {
|
|||||||
String servingUrl = fileDownloader.toServingUrl(task.getConversationId(), localPath);
|
String servingUrl = fileDownloader.toServingUrl(task.getConversationId(), localPath);
|
||||||
|
|
||||||
// 保存 assistant 消息(含 video content part)
|
// 保存 assistant 消息(含 video content part)
|
||||||
MessageContentPart videoPart = MessageContentPart.video(null, localPath.getFileName().toString());
|
String videoFileName = localPath.getFileName().toString();
|
||||||
|
MessageContentPart videoPart = MessageContentPart.video(null, videoFileName);
|
||||||
videoPart.setFileUrl(servingUrl);
|
videoPart.setFileUrl(servingUrl);
|
||||||
|
videoPart.setStoredName(videoFileName);
|
||||||
videoPart.setContentType("video/mp4");
|
videoPart.setContentType("video/mp4");
|
||||||
|
// Set absolute disk path so IM adapters can read bytes locally
|
||||||
|
// instead of round-tripping through /api/v1/chat/files (auth).
|
||||||
|
videoPart.setPath(localPath.toAbsolutePath().toString());
|
||||||
|
try {
|
||||||
|
videoPart.setFileSize(java.nio.file.Files.size(localPath));
|
||||||
|
} catch (Exception ignored) { /* best-effort */ }
|
||||||
|
|
||||||
|
List<MessageContentPart> parts = List.of(videoPart);
|
||||||
conversationService.saveMessage(
|
conversationService.saveMessage(
|
||||||
task.getConversationId(), "assistant",
|
task.getConversationId(), "assistant",
|
||||||
"视频已生成完毕",
|
"视频已生成完毕",
|
||||||
List.of(videoPart), "completed");
|
parts, "completed");
|
||||||
|
|
||||||
// SSE 广播
|
// SSE 广播
|
||||||
asyncTaskService.broadcastTaskEvent(task, "async_task_completed",
|
asyncTaskService.broadcastTaskEvent(task, "async_task_completed",
|
||||||
true, servingUrl, null);
|
true, servingUrl, null);
|
||||||
|
|
||||||
|
// Forward to the conversation's bound IM channel adapter so
|
||||||
|
// WeCom / DingTalk / Feishu etc. users receive the video as a
|
||||||
|
// native attachment (the SSE broadcast above only reaches Web).
|
||||||
|
asyncTaskMediaDispatcher.forwardToImIfBound(task.getConversationId(), parts);
|
||||||
|
|
||||||
log.info("[VideoGen] Task {} completed, video saved: {}", task.getTaskId(), servingUrl);
|
log.info("[VideoGen] Task {} completed, video saved: {}", task.getTaskId(), servingUrl);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[VideoGen] Completion handling failed for task {}: {}",
|
log.error("[VideoGen] Completion handling failed for task {}: {}",
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user