From 43acd2bf94a05579cb0dda4cf1d5f518f6dc3242 Mon Sep 17 00:00:00 2001 From: matevip Date: Tue, 7 Apr 2026 22:14:31 +0800 Subject: [PATCH] feat(image,tts): add image generation with 4 providers and TTS with 3 providers --- mateclaw-server/nul | 1 + .../mate/system/model/SystemSettingsDTO.java | 22 ++ .../system/service/SystemSettingService.java | 58 ++++ .../java/vip/mate/task/AsyncTaskService.java | 19 +- .../mate/tool/builtin/ImageGenerateTool.java | 171 ++++++++++++ .../vip/mate/tool/image/ImageCapability.java | 15 + .../mate/tool/image/ImageFileDownloader.java | 74 +++++ .../tool/image/ImageGenerationProvider.java | 65 +++++ .../tool/image/ImageGenerationRequest.java | 43 +++ .../tool/image/ImageGenerationResult.java | 62 +++++ .../tool/image/ImageGenerationService.java | 262 ++++++++++++++++++ .../tool/image/ImageProviderCapabilities.java | 90 ++++++ .../tool/image/ImageProviderRegistry.java | 84 ++++++ .../mate/tool/image/ImageSubmitResult.java | 64 +++++ .../provider/DashScopeImageProvider.java | 196 +++++++++++++ .../tool/image/provider/FalImageProvider.java | 205 ++++++++++++++ .../image/provider/OpenAiImageProvider.java | 173 ++++++++++++ .../image/provider/ZhipuImageProvider.java | 147 ++++++++++ .../main/java/vip/mate/tts/TtsController.java | 54 ++++ .../main/java/vip/mate/tts/TtsProvider.java | 43 +++ .../vip/mate/tts/TtsProviderRegistry.java | 73 +++++ .../main/java/vip/mate/tts/TtsRequest.java | 31 +++ .../src/main/java/vip/mate/tts/TtsResult.java | 45 +++ .../main/java/vip/mate/tts/TtsService.java | 206 ++++++++++++++ .../tts/provider/DashScopeTtsProvider.java | 133 +++++++++ .../mate/tts/provider/EdgeTtsProvider.java | 241 ++++++++++++++++ .../mate/tts/provider/OpenAiTtsProvider.java | 138 +++++++++ .../src/main/resources/db/data-en.sql | 4 + .../src/main/resources/db/data-zh.sql | 4 + .../src/components/chat/MessageBubble.vue | 88 ++++++ mateclaw-ui/src/composables/chat/useChat.ts | 92 +++++- mateclaw-ui/src/composables/chat/useStream.ts | 2 + mateclaw-ui/src/i18n/locales/en-US.ts | 53 ++++ mateclaw-ui/src/i18n/locales/zh-CN.ts | 57 ++++ mateclaw-ui/src/router/index.ts | 12 + .../src/views/Settings/Image/index.vue | 262 ++++++++++++++++++ mateclaw-ui/src/views/Settings/Layout.vue | 12 + mateclaw-ui/src/views/Settings/Tts/index.vue | 243 ++++++++++++++++ 38 files changed, 3525 insertions(+), 19 deletions(-) create mode 100644 mateclaw-server/nul create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageFileDownloader.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderRegistry.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/ImageSubmitResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/provider/FalImageProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/provider/OpenAiImageProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/provider/ZhipuImageProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/TtsController.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/TtsProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/TtsProviderRegistry.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/TtsRequest.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/TtsResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/TtsService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tts/provider/OpenAiTtsProvider.java create mode 100644 mateclaw-ui/src/views/Settings/Image/index.vue create mode 100644 mateclaw-ui/src/views/Settings/Tts/index.vue diff --git a/mateclaw-server/nul b/mateclaw-server/nul new file mode 100644 index 00000000..96ee2339 --- /dev/null +++ b/mateclaw-server/nul @@ -0,0 +1 @@ +/bin/sh: wmic: command not found diff --git a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java index d1636dcb..be27040b 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java @@ -60,4 +60,26 @@ public class SystemSettingsDTO { private String klingSecretKey; private String klingAccessKeyMasked; private String klingSecretKeyMasked; + + // ===== 图片生成配置 ===== + /** 是否启用图片生成能力 */ + private Boolean imageEnabled; + /** 首选图片 provider: auto / dashscope / openai / fal / zhipu-cogview */ + private String imageProvider; + /** 是否启用 provider 级 fallback */ + private Boolean imageFallbackEnabled; + + // ===== TTS 语音合成配置 ===== + /** 是否启用 TTS */ + private Boolean ttsEnabled; + /** 首选 TTS provider: auto / edge-tts / openai / dashscope */ + private String ttsProvider; + /** 是否启用 provider 级 fallback */ + private Boolean ttsFallbackEnabled; + /** 自动 TTS 模式: off / always */ + private String ttsAutoMode; + /** 默认语音 */ + private String ttsDefaultVoice; + /** 默认语速 0.5-2.0 */ + private Double ttsSpeed; } diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index b797cf98..b6006b73 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -31,6 +31,19 @@ public class SystemSettingService { private static final String VIDEO_ENABLED_KEY = "videoEnabled"; private static final String VIDEO_PROVIDER_KEY = "videoProvider"; private static final String VIDEO_FALLBACK_ENABLED_KEY = "videoFallbackEnabled"; + + // 图片生成配置 keys + private static final String IMAGE_ENABLED_KEY = "imageEnabled"; + private static final String IMAGE_PROVIDER_KEY = "imageProvider"; + private static final String IMAGE_FALLBACK_ENABLED_KEY = "imageFallbackEnabled"; + + // TTS 配置 keys + private static final String TTS_ENABLED_KEY = "ttsEnabled"; + private static final String TTS_PROVIDER_KEY = "ttsProvider"; + private static final String TTS_FALLBACK_ENABLED_KEY = "ttsFallbackEnabled"; + private static final String TTS_AUTO_MODE_KEY = "ttsAutoMode"; + private static final String TTS_DEFAULT_VOICE_KEY = "ttsDefaultVoice"; + private static final String TTS_SPEED_KEY = "ttsSpeed"; private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey"; private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl"; private static final String FAL_API_KEY_KEY = "falApiKey"; @@ -68,6 +81,20 @@ public class SystemSettingService { dto.setFalApiKeyMasked(maskApiKey(getValue(FAL_API_KEY_KEY, ""))); dto.setKlingAccessKeyMasked(maskApiKey(getValue(KLING_ACCESS_KEY_KEY, ""))); dto.setKlingSecretKeyMasked(maskApiKey(getValue(KLING_SECRET_KEY_KEY, ""))); + + // 图片生成配置 + dto.setImageEnabled(Boolean.parseBoolean(getValue(IMAGE_ENABLED_KEY, "false"))); + dto.setImageProvider(getValue(IMAGE_PROVIDER_KEY, "auto")); + dto.setImageFallbackEnabled(Boolean.parseBoolean(getValue(IMAGE_FALLBACK_ENABLED_KEY, "true"))); + + // TTS 配置 + dto.setTtsEnabled(Boolean.parseBoolean(getValue(TTS_ENABLED_KEY, "false"))); + dto.setTtsProvider(getValue(TTS_PROVIDER_KEY, "auto")); + dto.setTtsFallbackEnabled(Boolean.parseBoolean(getValue(TTS_FALLBACK_ENABLED_KEY, "true"))); + dto.setTtsAutoMode(getValue(TTS_AUTO_MODE_KEY, "off")); + dto.setTtsDefaultVoice(getValue(TTS_DEFAULT_VOICE_KEY, "")); + String speedStr = getValue(TTS_SPEED_KEY, "1.0"); + try { dto.setTtsSpeed(Double.parseDouble(speedStr)); } catch (NumberFormatException e) { dto.setTtsSpeed(1.0); } return dto; } @@ -166,6 +193,37 @@ public class SystemSettingService { if (dto.getKlingSecretKey() != null && !dto.getKlingSecretKey().isBlank()) { saveValue(KLING_SECRET_KEY_KEY, dto.getKlingSecretKey(), "快手可灵 Secret Key"); } + + // 图片生成配置 + if (dto.getImageEnabled() != null) { + saveValue(IMAGE_ENABLED_KEY, String.valueOf(dto.getImageEnabled()), "是否启用图片生成"); + } + if (dto.getImageProvider() != null) { + saveValue(IMAGE_PROVIDER_KEY, dto.getImageProvider(), "图片生成首选 Provider"); + } + if (dto.getImageFallbackEnabled() != null) { + saveValue(IMAGE_FALLBACK_ENABLED_KEY, String.valueOf(dto.getImageFallbackEnabled()), "图片 Provider 级 Fallback"); + } + + // TTS 配置 + if (dto.getTtsEnabled() != null) { + saveValue(TTS_ENABLED_KEY, String.valueOf(dto.getTtsEnabled()), "是否启用 TTS 语音合成"); + } + if (dto.getTtsProvider() != null) { + saveValue(TTS_PROVIDER_KEY, dto.getTtsProvider(), "TTS 首选 Provider"); + } + if (dto.getTtsFallbackEnabled() != null) { + saveValue(TTS_FALLBACK_ENABLED_KEY, String.valueOf(dto.getTtsFallbackEnabled()), "TTS Provider 级 Fallback"); + } + if (dto.getTtsAutoMode() != null) { + saveValue(TTS_AUTO_MODE_KEY, dto.getTtsAutoMode(), "TTS 自动模式（off/always）"); + } + if (dto.getTtsDefaultVoice() != null) { + saveValue(TTS_DEFAULT_VOICE_KEY, dto.getTtsDefaultVoice(), "TTS 默认语音"); + } + if (dto.getTtsSpeed() != null) { + saveValue(TTS_SPEED_KEY, String.valueOf(dto.getTtsSpeed()), "TTS 默认语速"); + } return getSettings(); } diff --git a/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java b/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java index c8ef8462..0c3a8c50 100644 --- a/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java +++ b/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java @@ -262,11 +262,17 @@ public class AsyncTaskService implements ApplicationRunner { public void broadcastTaskEvent(AsyncTaskEntity task, String eventName, boolean success, String videoUrl, String errorMessage) { + broadcastTaskEvent(task, eventName, success, videoUrl, null, errorMessage); + } + + public void broadcastTaskEvent(AsyncTaskEntity task, String eventName, + boolean success, String videoUrl, String imageUrl, String errorMessage) { Map data = new HashMap<>(); data.put("taskId", task.getTaskId()); data.put("taskType", task.getTaskType()); data.put("success", success); if (videoUrl != null) data.put("videoUrl", videoUrl); + if (imageUrl != null) data.put("imageUrl", imageUrl); if (errorMessage != null) data.put("errorMessage", errorMessage); streamTracker.broadcastObject(task.getConversationId(), eventName, data); } @@ -310,6 +316,7 @@ public class AsyncTaskService implements ApplicationRunner { Integer progress, // 0-100, nullable String videoUrl, // 成功时的视频 URL String coverImageUrl,// 可选封面图 + String imageUrl, // 成功时的图片 URL（图片生成场景） String resultJson, // 完成时的完整结果 JSON String errorMessage // 失败时的错误信息 ) { @@ -322,19 +329,23 @@ public class AsyncTaskService implements ApplicationRunner { } public static TaskPollResult pending(Integer progress) { - return new TaskPollResult("pending", progress, null, null, null, null); + return new TaskPollResult("pending", progress, null, null, null, null, null); } public static TaskPollResult running(Integer progress) { - return new TaskPollResult("running", progress, null, null, null, null); + return new TaskPollResult("running", progress, null, null, null, null, null); } public static TaskPollResult succeeded(String videoUrl, String coverImageUrl, String resultJson) { - return new TaskPollResult("succeeded", 100, videoUrl, coverImageUrl, resultJson, null); + return new TaskPollResult("succeeded", 100, videoUrl, coverImageUrl, null, resultJson, null); + } + + public static TaskPollResult imageSucceeded(String imageUrl, String resultJson) { + return new TaskPollResult("succeeded", 100, null, null, imageUrl, resultJson, null); } public static TaskPollResult failed(String errorMessage) { - return new TaskPollResult("failed", null, null, null, null, errorMessage); + return new TaskPollResult("failed", null, null, null, null, null, errorMessage); } } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java new file mode 100644 index 00000000..64cb673d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java @@ -0,0 +1,171 @@ +package vip.mate.tool.builtin; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.task.AsyncTaskService; +import vip.mate.task.model.AsyncTaskInfo; +import vip.mate.tool.image.*; + +import java.util.List; +import java.util.StringJoiner; + +/** + * 图片生成工具 — Agent 可调用的 @Tool，提交图片生成任务 + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ImageGenerateTool { + + private final ImageGenerationService imageGenerationService; + private final ImageProviderRegistry providerRegistry; + private final SystemSettingService systemSettingService; + private final AsyncTaskService asyncTaskService; + + @Tool(description = "图片生成工具，支持以下 action：\n" + + "- generate（默认）：生成图片。提供 prompt 描述图片内容，可选 size/aspectRatio/model/count\n" + + "- list：列出所有可用的图片 Provider 及其支持的模型和能力\n" + + "- status：查看当前会话中正在进行的图片生成任务状态\n" + + "部分 Provider 是异步生成（30秒-2分钟），完成后自动显示在对话中。") + public String image_generate( + @ToolParam(description = "操作类型: generate（生成图片）、list（列出可用 Provider）、status（查看任务状态），默认 generate", required = false) String action, + @ToolParam(description = "图片内容描述，尽量详细（generate 时必填）", required = false) String prompt, + @ToolParam(description = "图片尺寸: 1024x1024 / 1024x1792 / 1792x1024", required = false) String size, + @ToolParam(description = "画面比例: 1:1 / 16:9 / 9:16，默认 1:1", required = false) String aspectRatio, + @ToolParam(description = "生成数量（1-4），默认 1", required = false) Integer count, + @ToolParam(description = "指定模型名称（可选）", required = false) String model, + @ToolParam(description = "查询指定任务 ID 的状态（status 模式时使用）", required = false) String taskId + ) { + String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase(); + + return switch (normalizedAction) { + case "list" -> handleListAction(); + case "status" -> handleStatusAction(taskId); + default -> handleGenerateAction(prompt, size, aspectRatio, count, model); + }; + } + + // ==================== action=list ==================== + + private String handleListAction() { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + List providers = providerRegistry.allSorted(); + + if (providers.isEmpty()) { + return "当前没有注册的图片生成 Provider。"; + } + + StringJoiner sb = new StringJoiner("\n\n"); + sb.add("## 可用的图片生成 Provider\n"); + + for (ImageGenerationProvider p : providers) { + boolean available = p.isAvailable(config); + ImageProviderCapabilities caps = p.detailedCapabilities(); + + StringJoiner entry = new StringJoiner("\n"); + entry.add("### " + p.label() + " (" + p.id() + ") " + (available ? "[已配置]" : "[未配置]")); + + if (caps != null) { + if (caps.getModels() != null && !caps.getModels().isEmpty()) { + entry.add("- 模型: " + String.join(", ", caps.getModels())); + } + entry.add("- 支持尺寸: " + String.join(", ", caps.getSupportedSizes())); + entry.add("- 最大数量: " + caps.getMaxCount()); + } + sb.add(entry.toString()); + } + return sb.toString(); + } + + // ==================== action=status ==================== + + private String handleStatusAction(String taskId) { + String conversationId = ToolExecutionContext.conversationId(); + + if (taskId != null && !taskId.isBlank()) { + AsyncTaskInfo info = imageGenerationService.checkTaskStatus(taskId); + if (info == null) { + return "未找到任务 ID: " + taskId; + } + return formatTaskStatus(info); + } + + if (conversationId == null) { + return "无法获取当前会话信息"; + } + List activeTasks = asyncTaskService.listActiveTasks(conversationId); + List imageTasks = activeTasks.stream() + .filter(t -> "image_generation".equals(t.getTaskType())) + .toList(); + if (imageTasks.isEmpty()) { + return "当前会话没有进行中的图片生成任务。"; + } + + StringJoiner sb = new StringJoiner("\n"); + sb.add("当前会话有 " + imageTasks.size() + " 个进行中的任务："); + for (AsyncTaskInfo task : imageTasks) { + sb.add("- 任务 " + task.getTaskId() + ": " + formatTaskStatus(task)); + } + return sb.toString(); + } + + // ==================== action=generate ==================== + + private String handleGenerateAction(String prompt, String size, String aspectRatio, + Integer count, String model) { + String conversationId = ToolExecutionContext.conversationId(); + String username = ToolExecutionContext.username(); + + if (conversationId == null || conversationId.isBlank()) { + return "错误：无法获取当前会话信息，请重试"; + } + + if (prompt == null || prompt.isBlank()) { + return "错误：prompt 为必填参数，请描述你想要生成的图片内容"; + } + + ImageGenerationRequest request = ImageGenerationRequest.builder() + .prompt(prompt) + .size(size) + .aspectRatio(aspectRatio != null ? aspectRatio : "1:1") + .count(count != null ? count : 1) + .model(model) + .build(); + + ImageGenerationResult result = imageGenerationService.submitGeneration( + request, conversationId, username != null ? username : "system"); + + if (result.isCompleted()) { + // 同步模式：图片已生成 + return result.getMessage(); + } else if (result.isSubmitted()) { + // 异步模式：已提交 + return result.getMessage(); + } else { + return "图片生成失败：" + result.getMessage(); + } + } + + // ==================== 辅助方法 ==================== + + private String formatTaskStatus(AsyncTaskInfo info) { + return switch (info.getStatus()) { + case "pending" -> "排队中，请稍候..."; + case "running" -> { + String progressStr = info.getProgress() != null && info.getProgress() > 0 + ? "（进度: " + info.getProgress() + "%）" : ""; + yield "生成中" + progressStr + "（" + info.getProviderName() + "）"; + } + case "succeeded" -> "已完成，图片已显示在对话中"; + case "failed" -> "失败：" + (info.getErrorMessage() != null ? info.getErrorMessage() : "未知错误"); + default -> "状态: " + info.getStatus(); + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java new file mode 100644 index 00000000..93eaa4e4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java @@ -0,0 +1,15 @@ +package vip.mate.tool.image; + +/** + * 图片生成能力枚举 + * + * @author MateClaw Team + */ +public enum ImageCapability { + + /** 文字生成图片 */ + TEXT_TO_IMAGE, + + /** 图片编辑 / 风格转换 */ + IMAGE_EDIT +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageFileDownloader.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageFileDownloader.java new file mode 100644 index 00000000..02964ef1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageFileDownloader.java @@ -0,0 +1,74 @@ +package vip.mate.tool.image; + +import cn.hutool.http.HttpUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Base64; + +/** + * 图片文件下载器 — 从 provider CDN 下载图片到本地存储，或解码 Base64 图片 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class ImageFileDownloader { + + private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads"); + + /** + * 从 URL 下载图片到本地 + */ + public Path download(String imageUrl, String conversationId, String taskId, int index) throws IOException { + Path dir = UPLOAD_ROOT.resolve(conversationId); + Files.createDirectories(dir); + + String extension = guessExtension(imageUrl); + String fileName = "image_" + taskId + "_" + index + extension; + Path targetFile = dir.resolve(fileName); + + log.info("[ImageDownloader] Downloading image from {} to {}", imageUrl, targetFile); + long size = HttpUtil.downloadFile(imageUrl, targetFile.toFile()); + log.info("[ImageDownloader] Downloaded {} bytes to {}", size, targetFile); + + return targetFile; + } + + /** + * 将 Base64 编码的图片保存到本地 + */ + public Path saveBase64(String base64Data, String conversationId, String taskId, int index) throws IOException { + Path dir = UPLOAD_ROOT.resolve(conversationId); + Files.createDirectories(dir); + + String fileName = "image_" + taskId + "_" + index + ".png"; + Path targetFile = dir.resolve(fileName); + + byte[] imageBytes = Base64.getDecoder().decode(base64Data); + Files.write(targetFile, imageBytes); + log.info("[ImageDownloader] Saved base64 image ({} bytes) to {}", imageBytes.length, targetFile); + + return targetFile; + } + + /** + * 构造文件的 API 访问 URL + */ + public String toServingUrl(String conversationId, Path localPath) { + return "/api/v1/chat/files/" + conversationId + "/" + localPath.getFileName().toString(); + } + + private String guessExtension(String url) { + String lower = url.toLowerCase().split("\\?")[0]; + if (lower.endsWith(".png")) return ".png"; + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return ".jpg"; + if (lower.endsWith(".webp")) return ".webp"; + if (lower.endsWith(".gif")) return ".gif"; + return ".png"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationProvider.java new file mode 100644 index 00000000..f5c514fa --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationProvider.java @@ -0,0 +1,65 @@ +package vip.mate.tool.image; + +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; + +import java.util.Set; + +/** + * 图片生成提供商接口 — 所有图片 provider 统一实现此接口 + *

+ * 设计参考 {@link vip.mate.tool.video.VideoGenerationProvider}。 + * 与视频不同，图片 Provider 分同步和异步两种模式。 + * + * @author MateClaw Team + */ +public interface ImageGenerationProvider { + + /** 提供商唯一 ID，如 "dashscope"、"openai"、"fal"、"zhipu-cogview" */ + String id(); + + /** 显示名称 */ + String label(); + + /** 是否需要 API Key / Credential */ + boolean requiresCredential(); + + /** + * 自动探测排序优先级（升序）。 + */ + int autoDetectOrder(); + + /** 该 provider 支持的能力集 */ + Set capabilities(); + + /** 细粒度能力声明（支持的 size、aspectRatio、模型列表等） */ + ImageProviderCapabilities detailedCapabilities(); + + /** + * 判断该 provider 在当前配置下是否可用 + */ + boolean isAvailable(SystemSettingsDTO config); + + /** + * 提交图片生成任务 + *

+ * 同步 Provider 在此方法内完成生成并返回 imageUrls（async=false）。 + * 异步 Provider 返回 providerTaskId（async=true），需后续轮询。 + * + * @param request 统一请求参数 + * @param config 系统配置 + * @return 提交结果 + */ + ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config); + + /** + * 轮询任务状态（仅异步 Provider 需要实现） + * + * @param providerTaskId provider 返回的任务 ID + * @param config 系统配置 + * @return 轮询结果，同步 Provider 可返回 null + */ + default TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + return null; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java new file mode 100644 index 00000000..752ddb6a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java @@ -0,0 +1,43 @@ +package vip.mate.tool.image; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +/** + * 图片生成统一请求 + * + * @author MateClaw Team + */ +@Data +@Builder +public class ImageGenerationRequest { + + /** 图片内容描述 */ + private String prompt; + + /** 生成模式（由 runtime 自动推断） */ + private ImageCapability mode; + + /** 指定模型名称（可选，provider 有默认值） */ + private String model; + + /** 图片尺寸：1024x1024 / 1024x1792 / 1792x1024 等 */ + @Builder.Default + private String size = "1024x1024"; + + /** 画面比例：1:1 / 16:9 / 9:16 */ + @Builder.Default + private String aspectRatio = "1:1"; + + /** 生成数量 */ + @Builder.Default + private Integer count = 1; + + /** 参考图片 URL（IMAGE_EDIT 模式） */ + private String referenceImageUrl; + + /** provider 特有的额外参数 */ + private Map extraParams; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationResult.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationResult.java new file mode 100644 index 00000000..72cc2096 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationResult.java @@ -0,0 +1,62 @@ +package vip.mate.tool.image; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * 图片生成服务提交结果（面向 Tool 层） + * + * @author MateClaw Team + */ +@Data +@Builder +public class ImageGenerationResult { + + /** 内部任务 ID（异步模式，供 Agent 查询状态用） */ + private String taskId; + + /** 处理该任务的 provider 名称 */ + private String providerName; + + /** 是否成功提交（异步）或生成完成（同步） */ + private boolean submitted; + + /** 同步模式：图片是否已生成完毕 */ + private boolean completed; + + /** 同步模式：生成的图片本地 serving URL 列表 */ + private List imageUrls; + + /** 面向 Agent 的说明文本 */ + private String message; + + public static ImageGenerationResult asyncSuccess(String taskId, String providerName) { + return ImageGenerationResult.builder() + .taskId(taskId) + .providerName(providerName) + .submitted(true) + .completed(false) + .message("图片生成任务已提交（任务 ID: " + taskId + "）。预计 30 秒 - 2 分钟完成，完成后会自动显示在对话中。") + .build(); + } + + public static ImageGenerationResult syncSuccess(String providerName, List imageUrls) { + return ImageGenerationResult.builder() + .providerName(providerName) + .submitted(true) + .completed(true) + .imageUrls(imageUrls) + .message("图片已生成完毕，共 " + imageUrls.size() + " 张。") + .build(); + } + + public static ImageGenerationResult failure(String message) { + return ImageGenerationResult.builder() + .submitted(false) + .completed(false) + .message(message) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java new file mode 100644 index 00000000..9d6c6249 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java @@ -0,0 +1,262 @@ +package vip.mate.tool.image; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.task.AsyncTaskService; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.task.model.AsyncTaskEntity; +import vip.mate.task.model.AsyncTaskInfo; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * 图片生成服务 — 统一入口，处理 provider 选择、参数归一化、fallback、同步/异步提交 + *

+ * 与 VideoGenerationService 结构一致，额外处理同步模式（部分 Provider 直接返回图片 URL）。 + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageGenerationService { + + private final SystemSettingService systemSettingService; + private final ImageProviderRegistry providerRegistry; + private final AsyncTaskService asyncTaskService; + private final ConversationService conversationService; + private final ImageFileDownloader fileDownloader; + private final ObjectMapper objectMapper; + + private static final String TASK_TYPE = "image_generation"; + + /** + * 提交图片生成任务 + */ + public ImageGenerationResult submitGeneration(ImageGenerationRequest request, + String conversationId, + String createdBy) { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + + // 1. 检查图片功能是否启用 + if (!Boolean.TRUE.equals(config.getImageEnabled())) { + return ImageGenerationResult.failure("图片生成功能未启用，请在系统设置中开启"); + } + + // 2. 模式推断 + if (request.getMode() == null) { + request.setMode(inferMode(request)); + } + + // 3. Provider 选择 + ImageProviderRegistry.ResolvedProvider resolved = + providerRegistry.resolve(config, request.getMode()); + if (resolved == null) { + return ImageGenerationResult.failure( + "没有可用的图片生成 Provider，请在系统设置中配置（支持 DashScope、OpenAI、fal.ai、智谱）"); + } + + // 4. 提交（含 fallback） + return submitWithFallback(request, config, resolved, conversationId, createdBy); + } + + /** + * 查询任务状态 + */ + public AsyncTaskInfo checkTaskStatus(String taskId) { + return asyncTaskService.getTaskInfo(taskId); + } + + // ==================== 内部逻辑 ==================== + + private ImageGenerationResult submitWithFallback(ImageGenerationRequest request, + SystemSettingsDTO config, + ImageProviderRegistry.ResolvedProvider primary, + String conversationId, + String createdBy) { + // 尝试 primary + normalizeForProvider(request, primary.provider()); + ImageSubmitResult submitResult = primary.provider().submit(request, config); + if (submitResult.isAccepted()) { + return handleSubmitResult(submitResult, request, conversationId, createdBy, config); + } + + // Fallback + List attemptErrors = new ArrayList<>(); + attemptErrors.add(primary.provider().id() + ": " + submitResult.getErrorMessage()); + + if (Boolean.TRUE.equals(config.getImageFallbackEnabled())) { + List fallbacks = + providerRegistry.fallbackCandidates(config, request.getMode(), primary.provider().id()); + for (ImageGenerationProvider fb : fallbacks) { + log.info("[ImageGen] Trying fallback provider: {}", fb.id()); + normalizeForProvider(request, fb); + submitResult = fb.submit(request, config); + if (submitResult.isAccepted()) { + return handleSubmitResult(submitResult, request, conversationId, createdBy, config); + } + attemptErrors.add(fb.id() + ": " + submitResult.getErrorMessage()); + log.warn("[ImageGen] Fallback provider {} failed: {}", fb.id(), submitResult.getErrorMessage()); + } + } + + return ImageGenerationResult.failure( + "所有 Provider 均提交失败\n" + String.join("\n", attemptErrors)); + } + + private ImageGenerationResult handleSubmitResult(ImageSubmitResult submitResult, + ImageGenerationRequest request, + String conversationId, + String createdBy, + SystemSettingsDTO config) { + if (submitResult.isAsync()) { + // 异步模式：创建任务 + 启动轮询 + return createAsyncTask(submitResult, request, conversationId, createdBy, config); + } else { + // 同步模式：直接下载图片、保存消息 + return handleSyncCompletion(submitResult, conversationId, createdBy); + } + } + + private ImageGenerationResult createAsyncTask(ImageSubmitResult submitResult, + ImageGenerationRequest request, + String conversationId, + String createdBy, + SystemSettingsDTO config) { + try { + String requestJson = objectMapper.writeValueAsString(request); + AsyncTaskEntity task = asyncTaskService.createTask( + TASK_TYPE, conversationId, null, + submitResult.getProviderName(), + submitResult.getProviderTaskId(), + requestJson, createdBy); + + ImageGenerationProvider provider = providerRegistry.getById(submitResult.getProviderName()); + if (provider == null) { + return ImageGenerationResult.failure("Provider 不存在: " + submitResult.getProviderName()); + } + + asyncTaskService.startPolling( + task.getTaskId(), + providerTaskId -> provider.checkStatus(providerTaskId, systemSettingService.getAllSettings()), + (completedTask, pollResult) -> handleAsyncCompletion(completedTask, pollResult) + ); + + return ImageGenerationResult.asyncSuccess(task.getTaskId(), submitResult.getProviderName()); + } catch (Exception e) { + log.error("[ImageGen] Failed to create async task: {}", e.getMessage(), e); + return ImageGenerationResult.failure("创建任务失败: " + e.getMessage()); + } + } + + /** + * 同步 Provider 完成后：下载图片 → 保存消息 + */ + private ImageGenerationResult handleSyncCompletion(ImageSubmitResult submitResult, + String conversationId, + String createdBy) { + try { + List imageUrls = submitResult.getImageUrls(); + List servingUrls = new ArrayList<>(); + String taskId = java.util.UUID.randomUUID().toString().replace("-", "").substring(0, 16); + + List contentParts = new ArrayList<>(); + for (int i = 0; i < imageUrls.size(); i++) { + Path localPath = fileDownloader.download(imageUrls.get(i), conversationId, taskId, i); + String servingUrl = fileDownloader.toServingUrl(conversationId, localPath); + servingUrls.add(servingUrl); + + MessageContentPart imagePart = MessageContentPart.image(null, servingUrl); + imagePart.setFileName(localPath.getFileName().toString()); + imagePart.setContentType("image/png"); + contentParts.add(imagePart); + } + + // 保存 assistant 消息 + conversationService.saveMessage( + conversationId, "assistant", + "图片已生成完毕", + contentParts, "completed"); + + log.info("[ImageGen] Sync generation completed, {} image(s) saved for conversation {}", + servingUrls.size(), conversationId); + + return ImageGenerationResult.syncSuccess(submitResult.getProviderName(), servingUrls); + } catch (Exception e) { + log.error("[ImageGen] Sync completion handling failed: {}", e.getMessage(), e); + return ImageGenerationResult.failure("图片下载或保存失败: " + e.getMessage()); + } + } + + /** + * 异步任务完成时的回写逻辑：下载图片 → 保存消息 → 广播 SSE + */ + private void handleAsyncCompletion(AsyncTaskEntity task, TaskPollResult result) { + if (result.succeeded()) { + try { + String imageUrl = result.imageUrl(); + if (imageUrl == null) { + log.warn("[ImageGen] Task {} succeeded but no image URL", task.getTaskId()); + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + false, null, null, "图片生成成功但未返回图片 URL"); + return; + } + + // 下载图片到本地 + Path localPath = fileDownloader.download(imageUrl, task.getConversationId(), task.getTaskId(), 0); + String servingUrl = fileDownloader.toServingUrl(task.getConversationId(), localPath); + + // 保存 assistant 消息 + MessageContentPart imagePart = MessageContentPart.image(null, servingUrl); + imagePart.setFileName(localPath.getFileName().toString()); + imagePart.setContentType("image/png"); + + conversationService.saveMessage( + task.getConversationId(), "assistant", + "图片已生成完毕", + List.of(imagePart), "completed"); + + // SSE 广播（使用 imageUrl 字段） + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + true, null, servingUrl, null); + + log.info("[ImageGen] Task {} completed, image saved: {}", task.getTaskId(), servingUrl); + } catch (Exception e) { + log.error("[ImageGen] Completion handling failed for task {}: {}", + task.getTaskId(), e.getMessage(), e); + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + false, null, null, "图片下载或保存失败: " + e.getMessage()); + } + } else { + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + false, null, null, result.errorMessage()); + log.warn("[ImageGen] Task {} failed: {}", task.getTaskId(), result.errorMessage()); + } + } + + private ImageCapability inferMode(ImageGenerationRequest request) { + if (request.getReferenceImageUrl() != null && !request.getReferenceImageUrl().isBlank()) { + return ImageCapability.IMAGE_EDIT; + } + return ImageCapability.TEXT_TO_IMAGE; + } + + private void normalizeForProvider(ImageGenerationRequest request, ImageGenerationProvider provider) { + ImageProviderCapabilities caps = provider.detailedCapabilities(); + if (caps == null) return; + + request.setSize(caps.normalizeSize(request.getSize())); + request.setAspectRatio(caps.normalizeAspectRatio(request.getAspectRatio())); + if (request.getCount() != null) { + request.setCount(caps.normalizeCount(request.getCount())); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java new file mode 100644 index 00000000..078880de --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java @@ -0,0 +1,90 @@ +package vip.mate.tool.image; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; +import java.util.Set; + +/** + * 图片生成 Provider 细粒度能力声明 + * + * @author MateClaw Team + */ +@Data +@Builder +public class ImageProviderCapabilities { + + /** 支持的生成模式 */ + @Builder.Default + private Set modes = Set.of(ImageCapability.TEXT_TO_IMAGE); + + /** 支持的图片尺寸，如 ["1024x1024", "1024x1792"] */ + @Builder.Default + private List supportedSizes = List.of("1024x1024"); + + /** 支持的画面比例 */ + @Builder.Default + private List aspectRatios = List.of("1:1", "16:9", "9:16"); + + /** 最大生成数量 */ + @Builder.Default + private int maxCount = 1; + + /** 默认模型 */ + private String defaultModel; + + /** 可用模型列表 */ + @Builder.Default + private List models = List.of(); + + /** + * 将请求的 size 就近匹配到 provider 支持的值 + */ + public String normalizeSize(String requested) { + if (requested == null || requested.isBlank()) { + return supportedSizes.isEmpty() ? "1024x1024" : supportedSizes.get(0); + } + if (supportedSizes.contains(requested)) { + return requested; + } + // 就近匹配：解析面积，找最接近的 + long reqArea = parseArea(requested); + String closest = supportedSizes.get(0); + long minDiff = Math.abs(reqArea - parseArea(closest)); + for (String s : supportedSizes) { + long diff = Math.abs(reqArea - parseArea(s)); + if (diff < minDiff) { + minDiff = diff; + closest = s; + } + } + return closest; + } + + /** + * 将请求的 aspectRatio 就近匹配或回退到默认 + */ + public String normalizeAspectRatio(String requested) { + if (aspectRatios.contains(requested)) { + return requested; + } + return aspectRatios.isEmpty() ? "1:1" : aspectRatios.get(0); + } + + /** + * 将请求的 count 限制在 provider 支持范围内 + */ + public int normalizeCount(int requested) { + return Math.min(Math.max(requested, 1), maxCount); + } + + private long parseArea(String size) { + try { + String[] parts = size.toLowerCase().split("x"); + return Long.parseLong(parts[0].trim()) * Long.parseLong(parts[1].trim()); + } catch (Exception e) { + return 1024L * 1024L; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderRegistry.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderRegistry.java new file mode 100644 index 00000000..c61360b0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderRegistry.java @@ -0,0 +1,84 @@ +package vip.mate.tool.image; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 图片生成提供商注册表 — 收集所有 {@link ImageGenerationProvider} 实现，提供优先级排序与自动探测 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class ImageProviderRegistry { + + private final List sortedProviders; + private final Map providerMap; + + public ImageProviderRegistry(List providers) { + this.sortedProviders = providers.stream() + .sorted(Comparator.comparingInt(ImageGenerationProvider::autoDetectOrder)) + .toList(); + this.providerMap = providers.stream() + .collect(Collectors.toMap(ImageGenerationProvider::id, Function.identity())); + log.info("注册图片生成提供商 {} 个: {}", sortedProviders.size(), + sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList()); + } + + /** 按 ID 获取指定 provider */ + public ImageGenerationProvider getById(String id) { + return providerMap.get(id); + } + + /** 获取按 autoDetectOrder 排序的全部 provider 列表 */ + public List allSorted() { + return sortedProviders; + } + + /** + * 根据当前配置，解析应使用的 provider + */ + public ResolvedProvider resolve(SystemSettingsDTO config, ImageCapability requiredCapability) { + // 1. 用户显式配置的 primary provider + String configuredId = config.getImageProvider(); + if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) { + ImageGenerationProvider configured = providerMap.get(configuredId); + if (configured != null && configured.isAvailable(config) + && configured.capabilities().contains(requiredCapability)) { + return new ResolvedProvider(configured, "configured"); + } + } + + // 2. 自动探测：按优先级遍历，找第一个可用且支持该能力的 + for (ImageGenerationProvider p : sortedProviders) { + if (p.isAvailable(config) && p.capabilities().contains(requiredCapability)) { + return new ResolvedProvider(p, "auto-detect"); + } + } + + return null; + } + + /** + * 获取所有可用于 fallback 的 provider（按优先级排序，排除 primary） + */ + public List fallbackCandidates(SystemSettingsDTO config, + ImageCapability requiredCapability, + String excludeId) { + return sortedProviders.stream() + .filter(p -> !p.id().equals(excludeId)) + .filter(p -> p.isAvailable(config)) + .filter(p -> p.capabilities().contains(requiredCapability)) + .toList(); + } + + public record ResolvedProvider(ImageGenerationProvider provider, String source) { + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageSubmitResult.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageSubmitResult.java new file mode 100644 index 00000000..7487658d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageSubmitResult.java @@ -0,0 +1,64 @@ +package vip.mate.tool.image; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * Provider 提交图片生成任务的结果 + *

+ * 与视频不同，图片 Provider 分同步和异步两种： + * - 同步（async=false）：submit 时已完成生成，imageUrls 直接包含结果 + * - 异步（async=true）：返回 providerTaskId，需后续轮询 + * + * @author MateClaw Team + */ +@Data +@Builder +public class ImageSubmitResult { + + /** 是否被接受 */ + private boolean accepted; + + /** true=异步需轮询, false=同步已完成 */ + private boolean async; + + /** 提交的 provider 名称 */ + private String providerName; + + /** 异步模式：provider 返回的任务 ID */ + private String providerTaskId; + + /** 同步模式：直接返回的图片 URL 列表 */ + private List imageUrls; + + /** 错误信息（仅 accepted=false 时） */ + private String errorMessage; + + public static ImageSubmitResult syncSuccess(String providerName, List imageUrls) { + return ImageSubmitResult.builder() + .accepted(true) + .async(false) + .providerName(providerName) + .imageUrls(imageUrls) + .build(); + } + + public static ImageSubmitResult asyncSuccess(String providerTaskId, String providerName) { + return ImageSubmitResult.builder() + .accepted(true) + .async(true) + .providerName(providerName) + .providerTaskId(providerTaskId) + .build(); + } + + public static ImageSubmitResult failure(String providerName, String errorMessage) { + return ImageSubmitResult.builder() + .providerName(providerName) + .accepted(false) + .errorMessage(errorMessage) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java new file mode 100644 index 00000000..27fda229 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java @@ -0,0 +1,196 @@ +package vip.mate.tool.image.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.image.*; + +import java.util.List; +import java.util.Set; + +/** + * DashScope 图片生成 Provider — 支持通义万相 Wanx 系列 + *

+ * 异步模式：提交后返回 taskId，需轮询获取结果。 + * 复用已有的 DashScope LLM provider 的 API Key。 + * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/text-to-image + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DashScopeImageProvider implements ImageGenerationProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1"; + private static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo"; + + @Override + public String id() { + return "dashscope"; + } + + @Override + public String label() { + return "DashScope (通义万相)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 100; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.TEXT_TO_IMAGE); + } + + @Override + public ImageProviderCapabilities detailedCapabilities() { + return ImageProviderCapabilities.builder() + .modes(capabilities()) + .supportedSizes(List.of("1024x1024", "720x1280", "1280x720")) + .aspectRatios(List.of("1:1", "16:9", "9:16")) + .maxCount(4) + .defaultModel(DEFAULT_MODEL) + .models(List.of("wanx2.1-t2i-turbo", "wanx-v1")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { + return modelProviderService.isProviderConfigured("dashscope"); + } catch (Exception e) { + return false; + } + } + + @Override + public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { + String apiKey = getDashScopeApiKey(); + if (apiKey == null) { + return ImageSubmitResult.failure(id(), "DashScope API Key 未配置"); + } + + try { + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt()); + + ObjectNode parameters = body.putObject("parameters"); + String size = aspectRatioToSize(request.getAspectRatio()); + if (size != null) { + parameters.put("size", size); + } + int count = request.getCount() != null ? Math.min(request.getCount(), 4) : 1; + parameters.put("n", count); + + HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/text2image/image-synthesis") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .header("X-DashScope-Async", "enable") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("output")) { + String taskId = result.path("output").path("task_id").asText(); + log.info("[DashScope Image] Submitted task: {} (model={})", taskId, model); + return ImageSubmitResult.asyncSuccess(taskId, id()); + } else { + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[DashScope Image] Submit failed: {}", errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[DashScope Image] Submit error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + String apiKey = getDashScopeApiKey(); + if (apiKey == null) { + return TaskPollResult.failed("DashScope API Key 未配置"); + } + + try { + HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId) + .header("Authorization", "Bearer " + apiKey) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + JsonNode output = result.path("output"); + String taskStatus = output.path("task_status").asText(); + + return switch (taskStatus) { + case "SUCCEEDED" -> { + String imageUrl = extractImageUrl(output); + yield TaskPollResult.imageSucceeded(imageUrl, output.toString()); + } + case "FAILED" -> { + String errMsg = output.has("message") ? output.get("message").asText() : "任务失败"; + yield TaskPollResult.failed(errMsg); + } + case "RUNNING" -> TaskPollResult.running(null); + default -> TaskPollResult.pending(null); + }; + } catch (Exception e) { + log.error("[DashScope Image] Poll error for task {}: {}", providerTaskId, e.getMessage()); + return null; + } + } + + private String getDashScopeApiKey() { + try { + var providerEntity = modelProviderService.getProviderConfig("dashscope"); + return providerEntity.getApiKey(); + } catch (Exception e) { + return null; + } + } + + private String aspectRatioToSize(String aspectRatio) { + if (aspectRatio == null) return "1024*1024"; + return switch (aspectRatio) { + case "16:9" -> "1280*720"; + case "9:16" -> "720*1280"; + default -> "1024*1024"; + }; + } + + private String extractImageUrl(JsonNode output) { + JsonNode results = output.path("results"); + if (results.isArray() && !results.isEmpty()) { + return results.get(0).path("url").asText(null); + } + return null; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/FalImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/FalImageProvider.java new file mode 100644 index 00000000..b3b33a3b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/FalImageProvider.java @@ -0,0 +1,205 @@ +package vip.mate.tool.image.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.image.*; + +import java.util.List; +import java.util.Set; + +/** + * fal.ai 图片生成 Provider — 支持 Flux 系列模型 + *

+ * 异步队列模式：提交到 queue，轮询获取结果。 + * API 文档: https://fal.ai/models/fal-ai/flux/dev/api + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class FalImageProvider implements ImageGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String DEFAULT_MODEL = "fal-ai/flux/dev"; + private static final String QUEUE_BASE = "https://queue.fal.run"; + + @Override + public String id() { + return "fal"; + } + + @Override + public String label() { + return "fal.ai (Flux)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 300; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.TEXT_TO_IMAGE); + } + + @Override + public ImageProviderCapabilities detailedCapabilities() { + return ImageProviderCapabilities.builder() + .modes(capabilities()) + .supportedSizes(List.of("1024x1024", "1024x1536", "1536x1024")) + .aspectRatios(List.of("1:1", "16:9", "9:16", "4:3", "3:4")) + .maxCount(4) + .defaultModel(DEFAULT_MODEL) + .models(List.of("fal-ai/flux/dev", "fal-ai/flux/schnell", "fal-ai/flux-pro/v1.1")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return config.getFalApiKey() != null && !config.getFalApiKey().isBlank(); + } + + @Override + public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { + String apiKey = config.getFalApiKey(); + if (apiKey == null || apiKey.isBlank()) { + return ImageSubmitResult.failure(id(), "fal.ai API Key 未配置"); + } + + try { + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("prompt", request.getPrompt()); + + // fal.ai 使用 image_size 对象或字符串 + String size = normalizeSize(request.getSize(), request.getAspectRatio()); + ObjectNode imageSize = body.putObject("image_size"); + String[] parts = size.split("x"); + imageSize.put("width", Integer.parseInt(parts[0])); + imageSize.put("height", Integer.parseInt(parts[1])); + + int count = request.getCount() != null ? Math.min(request.getCount(), 4) : 1; + body.put("num_images", count); + + String url = QUEUE_BASE + "/" + model; + + HttpResponse response = HttpRequest.post(url) + .header("Authorization", "Key " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("request_id")) { + String requestId = result.get("request_id").asText(); + // 复合 taskId: model|requestId + String compositeId = model + "|" + requestId; + log.info("[Fal Image] Submitted task: {} (model={})", requestId, model); + return ImageSubmitResult.asyncSuccess(compositeId, id()); + } else { + String errMsg = result.has("detail") ? result.get("detail").asText() + : "HTTP " + response.getStatus(); + log.warn("[Fal Image] Submit failed: {}", errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[Fal Image] Submit error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + String apiKey = config.getFalApiKey(); + if (apiKey == null || apiKey.isBlank()) { + return TaskPollResult.failed("fal.ai API Key 未配置"); + } + + try { + // 解析复合 ID: model|requestId + String[] parts = providerTaskId.split("\\|", 2); + if (parts.length != 2) { + return TaskPollResult.failed("Invalid fal task ID format"); + } + String model = parts[0]; + String requestId = parts[1]; + + // 先检查状态 + String statusUrl = QUEUE_BASE + "/" + model + "/requests/" + requestId + "/status"; + HttpResponse statusResp = HttpRequest.get(statusUrl) + .header("Authorization", "Key " + apiKey) + .timeout(15_000) + .execute(); + + JsonNode statusResult = objectMapper.readTree(statusResp.body()); + String status = statusResult.has("status") ? statusResult.get("status").asText() : "UNKNOWN"; + + return switch (status) { + case "COMPLETED" -> { + // 获取结果 + String resultUrl = QUEUE_BASE + "/" + model + "/requests/" + requestId; + HttpResponse resultResp = HttpRequest.get(resultUrl) + .header("Authorization", "Key " + apiKey) + .timeout(15_000) + .execute(); + JsonNode resultData = objectMapper.readTree(resultResp.body()); + String imageUrl = extractImageUrl(resultData); + yield TaskPollResult.imageSucceeded(imageUrl, resultData.toString()); + } + case "FAILED" -> { + String errMsg = statusResult.has("error") ? statusResult.get("error").asText() : "任务失败"; + yield TaskPollResult.failed(errMsg); + } + case "IN_PROGRESS" -> TaskPollResult.running(null); + default -> TaskPollResult.pending(null); + }; + } catch (Exception e) { + log.error("[Fal Image] Poll error for task {}: {}", providerTaskId, e.getMessage()); + return null; + } + } + + private String normalizeSize(String size, String aspectRatio) { + if (size != null && !size.isBlank()) { + return size; + } + if (aspectRatio != null) { + return switch (aspectRatio) { + case "16:9" -> "1536x1024"; + case "9:16" -> "1024x1536"; + case "4:3" -> "1024x768"; + case "3:4" -> "768x1024"; + default -> "1024x1024"; + }; + } + return "1024x1024"; + } + + private String extractImageUrl(JsonNode result) { + JsonNode images = result.path("images"); + if (images.isArray() && !images.isEmpty()) { + return images.get(0).path("url").asText(null); + } + return null; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/OpenAiImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/OpenAiImageProvider.java new file mode 100644 index 00000000..21e46bc9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/OpenAiImageProvider.java @@ -0,0 +1,173 @@ +package vip.mate.tool.image.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * OpenAI 图片生成 Provider — 支持 DALL-E 3 / DALL-E 2 / gpt-image-1 + *

+ * 同步模式：直接返回图片 URL。 + * 复用已有的 OpenAI LLM provider 的 API Key。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OpenAiImageProvider implements ImageGenerationProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String DEFAULT_MODEL = "dall-e-3"; + + @Override + public String id() { + return "openai"; + } + + @Override + public String label() { + return "OpenAI (DALL-E)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 200; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.TEXT_TO_IMAGE); + } + + @Override + public ImageProviderCapabilities detailedCapabilities() { + return ImageProviderCapabilities.builder() + .modes(capabilities()) + .supportedSizes(List.of("1024x1024", "1024x1792", "1792x1024")) + .aspectRatios(List.of("1:1", "9:16", "16:9")) + .maxCount(1) // DALL-E 3 只支持 n=1 + .defaultModel(DEFAULT_MODEL) + .models(List.of("dall-e-3", "dall-e-2", "gpt-image-1")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { + return modelProviderService.isProviderConfigured("openai"); + } catch (Exception e) { + return false; + } + } + + @Override + public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { + String apiKey = getOpenAiApiKey(); + String baseUrl = getOpenAiBaseUrl(); + if (apiKey == null) { + return ImageSubmitResult.failure(id(), "OpenAI API Key 未配置"); + } + + try { + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("prompt", request.getPrompt()); + body.put("size", normalizeSize(request.getSize(), request.getAspectRatio())); + body.put("n", 1); + body.put("response_format", "url"); + + String url = (baseUrl != null ? baseUrl : "https://api.openai.com") + "/v1/images/generations"; + + HttpResponse response = HttpRequest.post(url) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(60_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("data")) { + List imageUrls = new ArrayList<>(); + for (JsonNode item : result.get("data")) { + String imageUrl = item.has("url") ? item.get("url").asText() : null; + if (imageUrl != null) { + imageUrls.add(imageUrl); + } + } + if (imageUrls.isEmpty()) { + return ImageSubmitResult.failure(id(), "API 返回成功但未包含图片 URL"); + } + log.info("[OpenAI Image] Generated {} image(s) (model={})", imageUrls.size(), model); + return ImageSubmitResult.syncSuccess(id(), imageUrls); + } else { + String errMsg = result.has("error") + ? result.path("error").path("message").asText("Unknown error") + : "HTTP " + response.getStatus(); + log.warn("[OpenAI Image] Submit failed: {}", errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[OpenAI Image] Submit error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), e.getMessage()); + } + } + + private String getOpenAiApiKey() { + try { + var providerEntity = modelProviderService.getProviderConfig("openai"); + return providerEntity.getApiKey(); + } catch (Exception e) { + return null; + } + } + + private String getOpenAiBaseUrl() { + try { + var providerEntity = modelProviderService.getProviderConfig("openai"); + return providerEntity.getBaseUrl(); + } catch (Exception e) { + return null; + } + } + + private String normalizeSize(String size, String aspectRatio) { + // 优先使用 size + if (size != null && !size.isBlank()) { + List supported = List.of("1024x1024", "1024x1792", "1792x1024"); + if (supported.contains(size)) return size; + } + // 根据 aspectRatio 推断 + if (aspectRatio != null) { + return switch (aspectRatio) { + case "9:16" -> "1024x1792"; + case "16:9" -> "1792x1024"; + default -> "1024x1024"; + }; + } + return "1024x1024"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/ZhipuImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/ZhipuImageProvider.java new file mode 100644 index 00000000..eb294543 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/ZhipuImageProvider.java @@ -0,0 +1,147 @@ +package vip.mate.tool.image.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * 智谱 CogView 图片生成 Provider — 支持 CogView-4 / CogView-3-Flash + *

+ * 同步模式：直接返回图片 URL。 + * CogView-3-Flash 模型免费。 + * API 文档: https://open.bigmodel.cn/dev/api/image-generate/cogview + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ZhipuImageProvider implements ImageGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String DEFAULT_MODEL = "cogview-3-flash"; + + @Override + public String id() { + return "zhipu-cogview"; + } + + @Override + public String label() { + return "智谱 CogView"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 150; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.TEXT_TO_IMAGE); + } + + @Override + public ImageProviderCapabilities detailedCapabilities() { + return ImageProviderCapabilities.builder() + .modes(capabilities()) + .supportedSizes(List.of("1024x1024", "768x1344", "1344x768", + "864x1152", "1152x864", "1440x720", "720x1440")) + .aspectRatios(List.of("1:1", "16:9", "9:16", "4:3", "3:4")) + .maxCount(1) + .defaultModel(DEFAULT_MODEL) + .models(List.of("cogview-4", "cogview-3-flash", "cogview-3-plus")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return config.getZhipuApiKey() != null && !config.getZhipuApiKey().isBlank(); + } + + @Override + public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { + String apiKey = config.getZhipuApiKey(); + if (apiKey == null || apiKey.isBlank()) { + return ImageSubmitResult.failure(id(), "智谱 API Key 未配置"); + } + + try { + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + + String baseUrl = config.getZhipuBaseUrl() != null && !config.getZhipuBaseUrl().isBlank() + ? config.getZhipuBaseUrl() : "https://open.bigmodel.cn"; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("prompt", request.getPrompt()); + + String size = aspectRatioToSize(request.getAspectRatio()); + if (size != null) { + body.put("size", size); + } + + HttpResponse response = HttpRequest.post(baseUrl + "/api/paas/v4/images/generations") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(60_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("data")) { + List imageUrls = new ArrayList<>(); + for (JsonNode item : result.get("data")) { + String imageUrl = item.has("url") ? item.get("url").asText() : null; + if (imageUrl != null) { + imageUrls.add(imageUrl); + } + } + if (imageUrls.isEmpty()) { + return ImageSubmitResult.failure(id(), "API 返回成功但未包含图片 URL"); + } + log.info("[Zhipu Image] Generated {} image(s) (model={})", imageUrls.size(), model); + return ImageSubmitResult.syncSuccess(id(), imageUrls); + } else { + String errMsg = result.has("error") + ? result.path("error").path("message").asText("Unknown error") + : "HTTP " + response.getStatus(); + log.warn("[Zhipu Image] Submit failed: {}", errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[Zhipu Image] Submit error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), e.getMessage()); + } + } + + private String aspectRatioToSize(String aspectRatio) { + if (aspectRatio == null) return "1024x1024"; + return switch (aspectRatio) { + case "16:9" -> "1344x768"; + case "9:16" -> "768x1344"; + case "4:3" -> "1152x864"; + case "3:4" -> "864x1152"; + default -> "1024x1024"; + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/TtsController.java b/mateclaw-server/src/main/java/vip/mate/tts/TtsController.java new file mode 100644 index 00000000..f41c812e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/TtsController.java @@ -0,0 +1,54 @@ +package vip.mate.tts; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; +import java.util.Map; + +/** + * TTS 语音合成 REST 端点 + * + * @author MateClaw Team + */ +@RestController +@RequestMapping("/api/v1/tts") +@RequiredArgsConstructor +public class TtsController { + + private final TtsService ttsService; + + /** + * 合成语音 — 前端"朗读"按钮调用 + */ + @PostMapping("/synthesize") + public ResponseEntity> synthesize(@RequestBody SynthesizeRequest req) { + Map result = ttsService.synthesize( + req.getConversationId(), + req.getText(), + req.getVoice(), + req.getSpeed(), + req.getFormat() + ); + return ResponseEntity.ok(result); + } + + /** + * 列出所有可用语音 + */ + @GetMapping("/voices") + public ResponseEntity>> listVoices() { + return ResponseEntity.ok(ttsService.listVoices()); + } + + @Data + public static class SynthesizeRequest { + private String conversationId; + private String text; + private String voice; + private Double speed; + private String format; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/TtsProvider.java b/mateclaw-server/src/main/java/vip/mate/tts/TtsProvider.java new file mode 100644 index 00000000..b7f923ca --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/TtsProvider.java @@ -0,0 +1,43 @@ +package vip.mate.tts; + +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.List; + +/** + * TTS 语音合成提供商接口 + * + * @author MateClaw Team + */ +public interface TtsProvider { + + /** 提供商唯一 ID，如 "edge-tts"、"openai"、"dashscope" */ + String id(); + + /** 显示名称 */ + String label(); + + /** 是否需要 API Key */ + boolean requiresCredential(); + + /** 自动探测排序优先级（升序），免费 Provider 优先 */ + int autoDetectOrder(); + + /** 判断该 provider 在当前配置下是否可用 */ + boolean isAvailable(SystemSettingsDTO config); + + /** 可用语音列表 */ + List availableVoices(); + + /** 默认语音 */ + String defaultVoice(); + + /** + * 合成语音 + * + * @param request 请求参数 + * @param config 系统配置 + * @return 合成结果（含音频字节） + */ + TtsResult synthesize(TtsRequest request, SystemSettingsDTO config); +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/TtsProviderRegistry.java b/mateclaw-server/src/main/java/vip/mate/tts/TtsProviderRegistry.java new file mode 100644 index 00000000..5e10b2bc --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/TtsProviderRegistry.java @@ -0,0 +1,73 @@ +package vip.mate.tts; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * TTS 提供商注册表 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class TtsProviderRegistry { + + private final List sortedProviders; + private final Map providerMap; + + public TtsProviderRegistry(List providers) { + this.sortedProviders = providers.stream() + .sorted(Comparator.comparingInt(TtsProvider::autoDetectOrder)) + .toList(); + this.providerMap = providers.stream() + .collect(Collectors.toMap(TtsProvider::id, Function.identity())); + log.info("注册 TTS 提供商 {} 个: {}", sortedProviders.size(), + sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList()); + } + + public TtsProvider getById(String id) { + return providerMap.get(id); + } + + public List allSorted() { + return sortedProviders; + } + + /** + * 根据当前配置，解析应使用的 provider + */ + public TtsProvider resolve(SystemSettingsDTO config) { + String configuredId = config.getTtsProvider(); + if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) { + TtsProvider configured = providerMap.get(configuredId); + if (configured != null && configured.isAvailable(config)) { + return configured; + } + } + + // 自动探测 + for (TtsProvider p : sortedProviders) { + if (p.isAvailable(config)) { + return p; + } + } + return null; + } + + /** + * 获取可用于 fallback 的 provider 列表 + */ + public List fallbackCandidates(SystemSettingsDTO config, String excludeId) { + return sortedProviders.stream() + .filter(p -> !p.id().equals(excludeId)) + .filter(p -> p.isAvailable(config)) + .toList(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/TtsRequest.java b/mateclaw-server/src/main/java/vip/mate/tts/TtsRequest.java new file mode 100644 index 00000000..afd43035 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/TtsRequest.java @@ -0,0 +1,31 @@ +package vip.mate.tts; + +import lombok.Builder; +import lombok.Data; + +/** + * TTS 语音合成统一请求 + * + * @author MateClaw Team + */ +@Data +@Builder +public class TtsRequest { + + /** 待合成的文本 */ + private String text; + + /** 语音 ID（可选，使用 Provider 默认） */ + private String voice; + + /** 模型名称（可选） */ + private String model; + + /** 语速 0.5-2.0，默认 1.0 */ + @Builder.Default + private Double speed = 1.0; + + /** 输出格式：mp3 / ogg / wav，默认 mp3 */ + @Builder.Default + private String format = "mp3"; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/TtsResult.java b/mateclaw-server/src/main/java/vip/mate/tts/TtsResult.java new file mode 100644 index 00000000..0cf974db --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/TtsResult.java @@ -0,0 +1,45 @@ +package vip.mate.tts; + +import lombok.Builder; +import lombok.Data; + +/** + * TTS 语音合成结果 + * + * @author MateClaw Team + */ +@Data +@Builder +public class TtsResult { + + /** 是否合成成功 */ + private boolean success; + + /** 音频字节数据 */ + private byte[] audioData; + + /** MIME 类型，如 audio/mpeg, audio/ogg */ + private String contentType; + + /** 音频格式：mp3, ogg, wav */ + private String format; + + /** 错误信息（仅 success=false 时） */ + private String errorMessage; + + public static TtsResult success(byte[] audioData, String contentType, String format) { + return TtsResult.builder() + .success(true) + .audioData(audioData) + .contentType(contentType) + .format(format) + .build(); + } + + public static TtsResult failure(String errorMessage) { + return TtsResult.builder() + .success(false) + .errorMessage(errorMessage) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/TtsService.java b/mateclaw-server/src/main/java/vip/mate/tts/TtsService.java new file mode 100644 index 00000000..912b4668 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/TtsService.java @@ -0,0 +1,206 @@ +package vip.mate.tts; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * TTS 语音合成服务 — 核心编排，处理 provider 选择、文本预处理、fallback、文件保存 + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class TtsService { + + private final SystemSettingService systemSettingService; + private final TtsProviderRegistry providerRegistry; + private final ChatStreamTracker streamTracker; + + private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads"); + private static final int MAX_TEXT_LENGTH = 4096; + + /** 用于自动 TTS 的异步线程池 */ + private final ExecutorService ttsExecutor = Executors.newFixedThreadPool(2, r -> { + Thread t = new Thread(r, "tts-worker"); + t.setDaemon(true); + return t; + }); + + /** + * 合成语音并保存为文件 + * + * @return { success, audioUrl, contentType, providerName } + */ + public Map synthesize(String conversationId, String text, + String voice, Double speed, String format) { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + + if (!Boolean.TRUE.equals(config.getTtsEnabled())) { + return Map.of("success", false, "error", "TTS 功能未启用，请在系统设置中开启"); + } + + // 文本预处理 + String cleanText = preprocessText(text); + if (cleanText.isBlank()) { + return Map.of("success", false, "error", "待合成的文本为空"); + } + + // 构建请求 + TtsRequest request = TtsRequest.builder() + .text(cleanText) + .voice(voice) + .speed(speed != null ? speed : config.getTtsSpeed()) + .format(format != null ? format : "mp3") + .build(); + + // Provider 选择 + fallback + TtsResult result = synthesizeWithFallback(request, config); + if (!result.isSuccess()) { + return Map.of("success", false, "error", result.getErrorMessage()); + } + + // 保存文件 + try { + String fileId = UUID.randomUUID().toString().replace("-", "").substring(0, 12); + Path filePath = saveAudioFile(conversationId, fileId, result.getAudioData(), result.getFormat()); + String audioUrl = "/api/v1/chat/files/" + conversationId + "/" + filePath.getFileName(); + + return Map.of( + "success", true, + "audioUrl", audioUrl, + "contentType", result.getContentType(), + "format", result.getFormat() + ); + } catch (IOException e) { + log.error("[TTS] Failed to save audio file: {}", e.getMessage(), e); + return Map.of("success", false, "error", "音频文件保存失败: " + e.getMessage()); + } + } + + /** + * 自动 TTS：消息完成后异步触发，通过 SSE 广播结果 + */ + public void autoSynthesize(String conversationId, String text) { + ttsExecutor.submit(() -> { + try { + Map result = synthesize(conversationId, text, null, null, null); + if (Boolean.TRUE.equals(result.get("success"))) { + Map event = new HashMap<>(); + event.put("audioUrl", result.get("audioUrl")); + event.put("contentType", result.get("contentType")); + streamTracker.broadcastObject(conversationId, "tts_ready", event); + log.info("[TTS] Auto-synthesized for conversation {}", conversationId); + } + } catch (Exception e) { + log.error("[TTS] Auto-synthesize failed: {}", e.getMessage(), e); + } + }); + } + + /** + * 获取当前配置是否开启自动 TTS + */ + public boolean isAutoModeEnabled() { + SystemSettingsDTO config = systemSettingService.getSettings(); + return Boolean.TRUE.equals(config.getTtsEnabled()) + && "always".equals(config.getTtsAutoMode()); + } + + /** + * 列出所有可用的语音 + */ + public List> listVoices() { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + List> voices = new ArrayList<>(); + + for (TtsProvider provider : providerRegistry.allSorted()) { + boolean available = provider.isAvailable(config); + for (String voice : provider.availableVoices()) { + Map info = new LinkedHashMap<>(); + info.put("voice", voice); + info.put("provider", provider.id()); + info.put("providerLabel", provider.label()); + info.put("available", available); + info.put("isDefault", voice.equals(provider.defaultVoice())); + voices.add(info); + } + } + return voices; + } + + // ==================== 内部逻辑 ==================== + + private TtsResult synthesizeWithFallback(TtsRequest request, SystemSettingsDTO config) { + TtsProvider primary = providerRegistry.resolve(config); + if (primary == null) { + return TtsResult.failure("没有可用的 TTS Provider，请检查配置"); + } + + TtsResult result = primary.synthesize(request, config); + if (result.isSuccess()) { + return result; + } + + // Fallback + List errors = new ArrayList<>(); + errors.add(primary.id() + ": " + result.getErrorMessage()); + + if (Boolean.TRUE.equals(config.getTtsFallbackEnabled())) { + for (TtsProvider fb : providerRegistry.fallbackCandidates(config, primary.id())) { + log.info("[TTS] Trying fallback provider: {}", fb.id()); + result = fb.synthesize(request, config); + if (result.isSuccess()) { + return result; + } + errors.add(fb.id() + ": " + result.getErrorMessage()); + } + } + + return TtsResult.failure("所有 TTS Provider 均失败\n" + String.join("\n", errors)); + } + + private String preprocessText(String text) { + if (text == null) return ""; + // 去除 Markdown 格式 + String clean = text + .replaceAll("```[\\s\\S]*?```", "") // 代码块 + .replaceAll("`[^`]+`", "") // 行内代码 + .replaceAll("!?\\[([^\\]]*)\\]\\([^)]+\\)", "$1") // 链接/图片 + .replaceAll("[*_~]{1,3}", "") // 加粗/斜体/删除线 + .replaceAll("^#{1,6}\\s+", "") // 标题 + .replaceAll("^[\\-*+]\\s+", "") // 列表 + .replaceAll("^>\\s+", "") // 引用 + .replaceAll("\\|[^|]+\\|", "") // 表格 + .replaceAll("\n{3,}", "\n\n") // 多余空行 + .trim(); + + // 截断 + if (clean.length() > MAX_TEXT_LENGTH) { + clean = clean.substring(0, MAX_TEXT_LENGTH); + } + return clean; + } + + private Path saveAudioFile(String conversationId, String fileId, byte[] data, String format) + throws IOException { + Path dir = UPLOAD_ROOT.resolve(conversationId); + Files.createDirectories(dir); + String fileName = "tts_" + fileId + "." + format; + Path filePath = dir.resolve(fileName); + Files.write(filePath, data); + return filePath; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java b/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java new file mode 100644 index 00000000..5bfa1a34 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java @@ -0,0 +1,133 @@ +package vip.mate.tts.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tts.TtsProvider; +import vip.mate.tts.TtsRequest; +import vip.mate.tts.TtsResult; + +import java.util.List; + +/** + * DashScope TTS Provider — 使用 CosyVoice（OpenAI 兼容接口） + *

+ * 同步模式，直接返回音频流。 + * 复用已有的 DashScope LLM provider 的 API Key。 + * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-openai-compatible + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DashScopeTtsProvider implements TtsProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; + private static final String DEFAULT_MODEL = "cosyvoice-v2"; + private static final String DEFAULT_VOICE = "longxiaochun"; + + @Override + public String id() { + return "dashscope"; + } + + @Override + public String label() { + return "DashScope (CosyVoice)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 150; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { + return modelProviderService.isProviderConfigured("dashscope"); + } catch (Exception e) { + return false; + } + } + + @Override + public List availableVoices() { + return List.of( + "longxiaochun", "longxiaoxia", "longlaotie", "longshu", + "longhua", "longshuo", "longjielidou", "longmiao", + "longyue", "longfei", "longtong", "longxiang" + ); + } + + @Override + public String defaultVoice() { + return DEFAULT_VOICE; + } + + @Override + public TtsResult synthesize(TtsRequest request, SystemSettingsDTO config) { + String apiKey = getDashScopeApiKey(); + if (apiKey == null) { + return TtsResult.failure("DashScope API Key 未配置"); + } + + try { + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + String voice = request.getVoice() != null && !request.getVoice().isBlank() + ? request.getVoice() : DEFAULT_VOICE; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("input", request.getText()); + body.put("voice", voice); + body.put("response_format", "mp3"); + if (request.getSpeed() != null && request.getSpeed() != 1.0) { + body.put("speed", request.getSpeed()); + } + + HttpResponse response = HttpRequest.post(BASE_URL + "/audio/speech") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(60_000) + .execute(); + + if (response.getStatus() == 200) { + byte[] audioData = response.bodyBytes(); + log.info("[DashScope TTS] Synthesized {} bytes (model={}, voice={})", audioData.length, model, voice); + return TtsResult.success(audioData, "audio/mpeg", "mp3"); + } else { + String errBody = response.body(); + log.warn("[DashScope TTS] Failed: HTTP {} - {}", response.getStatus(), errBody); + return TtsResult.failure("DashScope TTS 失败: HTTP " + response.getStatus()); + } + } catch (Exception e) { + log.error("[DashScope TTS] Error: {}", e.getMessage(), e); + return TtsResult.failure("DashScope TTS 异常: " + e.getMessage()); + } + } + + private String getDashScopeApiKey() { + try { + return modelProviderService.getProviderConfig("dashscope").getApiKey(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java b/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java new file mode 100644 index 00000000..168d5815 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java @@ -0,0 +1,241 @@ +package vip.mate.tts.provider; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tts.TtsProvider; +import vip.mate.tts.TtsRequest; +import vip.mate.tts.TtsResult; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.TimeUnit; + +/** + * Microsoft Edge TTS Provider — 免费，无需 API Key + *

+ * 使用 Edge 浏览器内置的 TTS WebSocket 协议。 + * 自动根据文本语言选择中文或英文语音。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class EdgeTtsProvider implements TtsProvider { + + private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud"; + private static final String TRUSTED_CLIENT_TOKEN = "6A5AA1D4EAFF4E9FB37E23D68491D6F4"; + private static final String DEFAULT_VOICE_ZH = "zh-CN-XiaoxiaoNeural"; + private static final String DEFAULT_VOICE_EN = "en-US-MichelleNeural"; + private static final String OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"; + + @Override + public String id() { + return "edge-tts"; + } + + @Override + public String label() { + return "Edge TTS (免费)"; + } + + @Override + public boolean requiresCredential() { + return false; + } + + @Override + public int autoDetectOrder() { + return 100; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return true; // 始终可用，无需 Key + } + + @Override + public List availableVoices() { + return List.of( + "zh-CN-XiaoxiaoNeural", "zh-CN-YunxiNeural", "zh-CN-YunjianNeural", + "zh-CN-XiaoyiNeural", "zh-CN-YunyangNeural", + "en-US-MichelleNeural", "en-US-GuyNeural", "en-US-JennyNeural", + "en-US-AriaNeural", "en-US-DavisNeural", + "ja-JP-NanamiNeural", "ko-KR-SunHiNeural" + ); + } + + @Override + public String defaultVoice() { + return DEFAULT_VOICE_ZH; + } + + @Override + public TtsResult synthesize(TtsRequest request, SystemSettingsDTO config) { + String voice = resolveVoice(request.getVoice(), request.getText(), config); + String rate = speedToRate(request.getSpeed()); + + try { + byte[] audioData = synthesizeViaWebSocket(request.getText(), voice, rate); + if (audioData == null || audioData.length == 0) { + return TtsResult.failure("Edge TTS 未返回音频数据"); + } + log.info("[Edge TTS] Synthesized {} bytes (voice={})", audioData.length, voice); + return TtsResult.success(audioData, "audio/mpeg", "mp3"); + } catch (Exception e) { + log.error("[Edge TTS] Synthesis error: {}", e.getMessage(), e); + return TtsResult.failure("Edge TTS 合成失败: " + e.getMessage()); + } + } + + private byte[] synthesizeViaWebSocket(String text, String voice, String rate) throws Exception { + String requestId = UUID.randomUUID().toString().replace("-", ""); + String wsUrl = WS_URL + "?TrustedClientToken=" + TRUSTED_CLIENT_TOKEN + + "&ConnectionId=" + requestId; + + ByteArrayOutputStream audioBuffer = new ByteArrayOutputStream(); + CompletableFuture resultFuture = new CompletableFuture<>(); + + HttpClient client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + WebSocket ws = client.newWebSocketBuilder() + .header("Origin", "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold") + .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + .buildAsync(URI.create(wsUrl), new WebSocket.Listener() { + private final StringBuilder textBuffer = new StringBuilder(); + + @Override + public void onOpen(WebSocket webSocket) { + webSocket.request(Long.MAX_VALUE); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + textBuffer.append(data); + if (last) { + String msg = textBuffer.toString(); + textBuffer.setLength(0); + if (msg.contains("turn.end")) { + resultFuture.complete(audioBuffer.toByteArray()); + } + } + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + // 二进制帧：前 2 字节是 header 长度（大端），跳过 header + byte[] bytes = new byte[data.remaining()]; + data.get(bytes); + // 查找 "Path:audio\r\n" 后的音频数据 + int headerEnd = findHeaderEnd(bytes); + if (headerEnd >= 0 && headerEnd < bytes.length) { + audioBuffer.write(bytes, headerEnd, bytes.length - headerEnd); + } + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onClose(WebSocket webSocket, int statusCode, String reason) { + if (!resultFuture.isDone()) { + resultFuture.complete(audioBuffer.toByteArray()); + } + return null; + } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + if (!resultFuture.isDone()) { + resultFuture.completeExceptionally(error); + } + } + }).join(); + + // 发送配置消息 + String configMsg = "Content-Type:application/json; charset=utf-8\r\n" + + "Path:speech.config\r\n\r\n" + + "{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{" + + "\"sentenceBoundaryEnabled\":false,\"wordBoundaryEnabled\":false}," + + "\"outputFormat\":\"" + OUTPUT_FORMAT + "\"}}}}\r\n"; + ws.sendText(configMsg, true); + + // 发送 SSML + String escapedText = escapeXml(text); + String ssml = "" + + "" + + "" + + escapedText + + ""; + + String ssmlMsg = "X-RequestId:" + requestId + "\r\n" + + "Content-Type:application/ssml+xml\r\n" + + "Path:ssml\r\n\r\n" + ssml; + ws.sendText(ssmlMsg, true); + + // 等待结果（最多 60 秒） + return resultFuture.get(60, TimeUnit.SECONDS); + } + + private int findHeaderEnd(byte[] data) { + // 二进制帧格式：2 字节 header 长度（大端） + header + 音频数据 + if (data.length < 2) return -1; + int headerLen = ((data[0] & 0xFF) << 8) | (data[1] & 0xFF); + return 2 + headerLen; + } + + private String resolveVoice(String requestedVoice, String text, SystemSettingsDTO config) { + if (requestedVoice != null && !requestedVoice.isBlank()) { + return requestedVoice; + } + String configVoice = config.getTtsDefaultVoice(); + if (configVoice != null && !configVoice.isBlank()) { + return configVoice; + } + // 自动语言检测：CJK 字符比例 > 30% 使用中文语音 + return isCjkDominant(text) ? DEFAULT_VOICE_ZH : DEFAULT_VOICE_EN; + } + + private boolean isCjkDominant(String text) { + if (text == null || text.isEmpty()) return true; + int cjkCount = 0; + int totalAlphaNum = 0; + for (char c : text.toCharArray()) { + if (Character.isLetterOrDigit(c)) { + totalAlphaNum++; + if (Character.UnicodeScript.of(c) == Character.UnicodeScript.HAN + || Character.UnicodeScript.of(c) == Character.UnicodeScript.HIRAGANA + || Character.UnicodeScript.of(c) == Character.UnicodeScript.KATAKANA + || Character.UnicodeScript.of(c) == Character.UnicodeScript.HANGUL) { + cjkCount++; + } + } + } + return totalAlphaNum == 0 || (double) cjkCount / totalAlphaNum > 0.3; + } + + private String speedToRate(Double speed) { + if (speed == null || speed == 1.0) return "+0%"; + int percent = (int) ((speed - 1.0) * 100); + return (percent >= 0 ? "+" : "") + percent + "%"; + } + + private String escapeXml(String text) { + return text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """) + .replace("'", "'"); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tts/provider/OpenAiTtsProvider.java b/mateclaw-server/src/main/java/vip/mate/tts/provider/OpenAiTtsProvider.java new file mode 100644 index 00000000..60a6984f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tts/provider/OpenAiTtsProvider.java @@ -0,0 +1,138 @@ +package vip.mate.tts.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tts.TtsProvider; +import vip.mate.tts.TtsRequest; +import vip.mate.tts.TtsResult; + +import java.util.List; + +/** + * OpenAI TTS Provider — 支持 tts-1 / tts-1-hd / gpt-4o-mini-tts + *

+ * 同步模式，直接返回音频流。 + * 复用已有的 OpenAI LLM provider 的 API Key。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OpenAiTtsProvider implements TtsProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String DEFAULT_MODEL = "tts-1"; + private static final String DEFAULT_VOICE = "alloy"; + + @Override + public String id() { + return "openai"; + } + + @Override + public String label() { + return "OpenAI TTS"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 200; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { + return modelProviderService.isProviderConfigured("openai"); + } catch (Exception e) { + return false; + } + } + + @Override + public List availableVoices() { + return List.of("alloy", "ash", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer"); + } + + @Override + public String defaultVoice() { + return DEFAULT_VOICE; + } + + @Override + public TtsResult synthesize(TtsRequest request, SystemSettingsDTO config) { + String apiKey = getApiKey(); + String baseUrl = getBaseUrl(); + if (apiKey == null) { + return TtsResult.failure("OpenAI API Key 未配置"); + } + + try { + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + String voice = request.getVoice() != null && !request.getVoice().isBlank() + ? request.getVoice() : DEFAULT_VOICE; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("input", request.getText()); + body.put("voice", voice); + body.put("response_format", "mp3"); + if (request.getSpeed() != null && request.getSpeed() != 1.0) { + body.put("speed", request.getSpeed()); + } + + String url = (baseUrl != null ? baseUrl : "https://api.openai.com") + "/v1/audio/speech"; + + HttpResponse response = HttpRequest.post(url) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(60_000) + .execute(); + + if (response.getStatus() == 200) { + byte[] audioData = response.bodyBytes(); + log.info("[OpenAI TTS] Synthesized {} bytes (model={}, voice={})", audioData.length, model, voice); + return TtsResult.success(audioData, "audio/mpeg", "mp3"); + } else { + String errBody = response.body(); + log.warn("[OpenAI TTS] Failed: HTTP {} - {}", response.getStatus(), errBody); + return TtsResult.failure("OpenAI TTS 失败: HTTP " + response.getStatus()); + } + } catch (Exception e) { + log.error("[OpenAI TTS] Error: {}", e.getMessage(), e); + return TtsResult.failure("OpenAI TTS 异常: " + e.getMessage()); + } + } + + private String getApiKey() { + try { + return modelProviderService.getProviderConfig("openai").getApiKey(); + } catch (Exception e) { + return null; + } + } + + private String getBaseUrl() { + try { + return modelProviderService.getProviderConfig("openai").getBaseUrl(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index dcb6720f..7b31fabd 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -377,6 +377,10 @@ MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, KEY (id) VALUES (1000000015, 'VideoGenerateTool', 'Video Generation', 'Generate videos using AI. Supports text-to-video and image-to-video modes. Video generation is asynchronous and will appear in conversation when complete.', 'builtin', 'videoGenerateTool', '🎬', TRUE, TRUE, NOW(), NOW(), 0); +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000016, 'ImageGenerateTool', 'Image Generation', 'Generate images using AI. Supports text-to-image mode with multiple providers: DashScope, OpenAI DALL-E, fal.ai Flux, Zhipu CogView. Auto-fallback between providers.', 'builtin', 'imageGenerateTool', '🎨', TRUE, TRUE, NOW(), NOW(), 0); + -- Example MCP Server: Filesystem (see MateClaw docs mcpServers.filesystem) MERGE INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index 6f690fd2..5965ca5a 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -383,6 +383,10 @@ MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, KEY (id) VALUES (1000000015, 'VideoGenerateTool', '视频生成', '使用 AI 生成视频，支持文字生成视频和图片生成视频两种模式。视频生成是异步过程，完成后自动显示在对话中。', 'builtin', 'videoGenerateTool', '🎬', TRUE, TRUE, NOW(), NOW(), 0); +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000016, 'ImageGenerateTool', '图片生成', '使用 AI 生成图片，支持文字生成图片。支持 DashScope 通义万相、OpenAI DALL-E、fal.ai Flux、智谱 CogView 等多个 Provider，自动回退。', 'builtin', 'imageGenerateTool', '🎨', TRUE, TRUE, NOW(), NOW(), 0); + -- 示例 MCP Server：Filesystem（参考 MateClaw 文档中的 mcpServers.filesystem） MERGE INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-ui/src/components/chat/MessageBubble.vue b/mateclaw-ui/src/components/chat/MessageBubble.vue index ba0551dc..80196f68 100644 --- a/mateclaw-ui/src/components/chat/MessageBubble.vue +++ b/mateclaw-ui/src/components/chat/MessageBubble.vue @@ -275,6 +275,29 @@ + +