From 7007ea844de3091965035d865dd2e62a389e832b Mon Sep 17 00:00:00 2001 From: matevip Date: Wed, 8 Apr 2026 08:49:09 +0800 Subject: [PATCH] feat: add TTS, STT, music generation, video providers (Runway/MiniMax), image providers (Google Imagen/MiniMax), and tool registry fix --- .../main/java/vip/mate/stt/SttController.java | 34 ++++ .../main/java/vip/mate/stt/SttProvider.java | 19 ++ .../vip/mate/stt/SttProviderRegistry.java | 51 +++++ .../main/java/vip/mate/stt/SttRequest.java | 22 +++ .../src/main/java/vip/mate/stt/SttResult.java | 28 +++ .../main/java/vip/mate/stt/SttService.java | 70 +++++++ .../stt/provider/DashScopeSttProvider.java | 73 +++++++ .../mate/stt/provider/OpenAiSttProvider.java | 74 +++++++ .../mate/system/model/SystemSettingsDTO.java | 22 +++ .../system/service/SystemSettingService.java | 54 +++++ .../main/java/vip/mate/tool/ToolRegistry.java | 17 +- .../image/provider/GoogleImagenProvider.java | 187 ++++++++++++++++++ .../image/provider/MiniMaxImageProvider.java | 151 ++++++++++++++ .../mate/tool/music/MusicGenerateTool.java | 52 +++++ .../tool/music/MusicGenerationProvider.java | 23 +++ .../tool/music/MusicGenerationRequest.java | 17 ++ .../tool/music/MusicGenerationResult.java | 27 +++ .../tool/music/MusicGenerationService.java | 85 ++++++++ .../tool/music/MusicProviderRegistry.java | 45 +++++ .../music/provider/GoogleLyriaProvider.java | 133 +++++++++++++ .../music/provider/MiniMaxMusicProvider.java | 140 +++++++++++++ .../video/provider/MiniMaxVideoProvider.java | 176 +++++++++++++++++ .../video/provider/RunwayVideoProvider.java | 171 ++++++++++++++++ mateclaw-ui/src/composables/chat/useChat.ts | 62 +++--- mateclaw-ui/src/i18n/locales/en-US.ts | 32 +++ mateclaw-ui/src/i18n/locales/zh-CN.ts | 36 ++++ mateclaw-ui/src/router/index.ts | 12 ++ .../src/views/Settings/Image/index.vue | 32 +++ mateclaw-ui/src/views/Settings/Layout.vue | 12 ++ .../src/views/Settings/Music/index.vue | 133 +++++++++++++ mateclaw-ui/src/views/Settings/Stt/index.vue | 133 +++++++++++++ .../src/views/Settings/Video/index.vue | 62 ++++++ 32 files changed, 2151 insertions(+), 34 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/SttController.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/SttProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/SttProviderRegistry.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/SttRequest.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/SttResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/SttService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/provider/DashScopeSttProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/provider/MiniMaxImageProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerateTool.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationRequest.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/MusicProviderRegistry.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/provider/GoogleLyriaProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/music/provider/MiniMaxMusicProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/provider/MiniMaxVideoProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/provider/RunwayVideoProvider.java create mode 100644 mateclaw-ui/src/views/Settings/Music/index.vue create mode 100644 mateclaw-ui/src/views/Settings/Stt/index.vue diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttController.java b/mateclaw-server/src/main/java/vip/mate/stt/SttController.java new file mode 100644 index 00000000..631e29d7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttController.java @@ -0,0 +1,34 @@ +package vip.mate.stt; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.util.Map; + +/** + * STT 语音识别 REST 端点 + */ +@RestController +@RequestMapping("/api/v1/stt") +@RequiredArgsConstructor +public class SttController { + + private final SttService sttService; + + @PostMapping(value = "/transcribe", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity> transcribe( + @RequestPart("file") MultipartFile file, + @RequestParam(required = false) String language) throws Exception { + + Map result = sttService.transcribe( + file.getBytes(), + file.getOriginalFilename(), + file.getContentType(), + language + ); + return ResponseEntity.ok(result); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttProvider.java b/mateclaw-server/src/main/java/vip/mate/stt/SttProvider.java new file mode 100644 index 00000000..63c2e2a4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttProvider.java @@ -0,0 +1,19 @@ +package vip.mate.stt; + +import vip.mate.system.model.SystemSettingsDTO; + +/** + * STT 语音识别提供商接口 + */ +public interface SttProvider { + String id(); + String label(); + boolean requiresCredential(); + int autoDetectOrder(); + boolean isAvailable(SystemSettingsDTO config); + + /** + * 转写音频 + */ + SttResult transcribe(SttRequest request, SystemSettingsDTO config); +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttProviderRegistry.java b/mateclaw-server/src/main/java/vip/mate/stt/SttProviderRegistry.java new file mode 100644 index 00000000..63468dca --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttProviderRegistry.java @@ -0,0 +1,51 @@ +package vip.mate.stt; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * STT 提供商注册表 + */ +@Slf4j +@Component +public class SttProviderRegistry { + + private final List sortedProviders; + private final Map providerMap; + + public SttProviderRegistry(List providers) { + this.sortedProviders = providers.stream() + .sorted(Comparator.comparingInt(SttProvider::autoDetectOrder)) + .toList(); + this.providerMap = providers.stream() + .collect(Collectors.toMap(SttProvider::id, Function.identity())); + log.info("注册 STT 提供商 {} 个: {}", sortedProviders.size(), + sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList()); + } + + public SttProvider resolve(SystemSettingsDTO config) { + String configuredId = config.getSttProvider(); + if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) { + SttProvider p = providerMap.get(configuredId); + if (p != null && p.isAvailable(config)) return p; + } + for (SttProvider p : sortedProviders) { + if (p.isAvailable(config)) return p; + } + return null; + } + + public List fallbackCandidates(SystemSettingsDTO config, String excludeId) { + return sortedProviders.stream() + .filter(p -> !p.id().equals(excludeId)) + .filter(p -> p.isAvailable(config)) + .toList(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttRequest.java b/mateclaw-server/src/main/java/vip/mate/stt/SttRequest.java new file mode 100644 index 00000000..6a07bcd5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttRequest.java @@ -0,0 +1,22 @@ +package vip.mate.stt; + +import lombok.Builder; +import lombok.Data; + +/** + * STT 语音识别统一请求 + */ +@Data +@Builder +public class SttRequest { + /** 音频二进制数据 */ + private byte[] audioData; + /** 原始文件名 */ + private String fileName; + /** MIME 类型（audio/ogg, audio/mpeg 等） */ + private String contentType; + /** 目标语言（可选，如 "zh", "en"） */ + private String language; + /** 模型（可选） */ + private String model; +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttResult.java b/mateclaw-server/src/main/java/vip/mate/stt/SttResult.java new file mode 100644 index 00000000..a5dbb345 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttResult.java @@ -0,0 +1,28 @@ +package vip.mate.stt; + +import lombok.Builder; +import lombok.Data; + +/** + * STT 语音识别结果 + */ +@Data +@Builder +public class SttResult { + private boolean success; + private String text; + private String language; + private String errorMessage; + + public static SttResult success(String text) { + return SttResult.builder().success(true).text(text).build(); + } + + public static SttResult success(String text, String language) { + return SttResult.builder().success(true).text(text).language(language).build(); + } + + public static SttResult failure(String errorMessage) { + return SttResult.builder().success(false).errorMessage(errorMessage).build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttService.java b/mateclaw-server/src/main/java/vip/mate/stt/SttService.java new file mode 100644 index 00000000..44432bac --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttService.java @@ -0,0 +1,70 @@ +package vip.mate.stt; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; + +import java.util.*; + +/** + * STT 语音识别服务 + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class SttService { + + private final SystemSettingService systemSettingService; + private final SttProviderRegistry providerRegistry; + + public Map transcribe(byte[] audioData, String fileName, String contentType, String language) { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + + if (!Boolean.TRUE.equals(config.getSttEnabled())) { + return Map.of("success", false, "error", "STT 功能未启用，请在系统设置中开启"); + } + + SttRequest request = SttRequest.builder() + .audioData(audioData) + .fileName(fileName) + .contentType(contentType) + .language(language) + .build(); + + // Provider 选择 + fallback + SttResult result = transcribeWithFallback(request, config); + + if (result.isSuccess()) { + return Map.of("success", true, "text", result.getText(), + "language", result.getLanguage() != null ? result.getLanguage() : ""); + } else { + return Map.of("success", false, "error", result.getErrorMessage()); + } + } + + private SttResult transcribeWithFallback(SttRequest request, SystemSettingsDTO config) { + SttProvider primary = providerRegistry.resolve(config); + if (primary == null) { + return SttResult.failure("没有可用的 STT Provider，请检查配置"); + } + + SttResult result = primary.transcribe(request, config); + if (result.isSuccess()) return result; + + List errors = new ArrayList<>(); + errors.add(primary.id() + ": " + result.getErrorMessage()); + + if (Boolean.TRUE.equals(config.getSttFallbackEnabled())) { + for (SttProvider fb : providerRegistry.fallbackCandidates(config, primary.id())) { + log.info("[STT] Trying fallback provider: {}", fb.id()); + result = fb.transcribe(request, config); + if (result.isSuccess()) return result; + errors.add(fb.id() + ": " + result.getErrorMessage()); + } + } + + return SttResult.failure("所有 STT Provider 均失败\n" + String.join("\n", errors)); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/provider/DashScopeSttProvider.java b/mateclaw-server/src/main/java/vip/mate/stt/provider/DashScopeSttProvider.java new file mode 100644 index 00000000..18e9c5d3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/provider/DashScopeSttProvider.java @@ -0,0 +1,73 @@ +package vip.mate.stt.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.stt.SttProvider; +import vip.mate.stt.SttRequest; +import vip.mate.stt.SttResult; +import vip.mate.system.model.SystemSettingsDTO; + +/** + * DashScope STT Provider — Paraformer（OpenAI 兼容接口） + *

+ * 复用模型管理中的 DashScope API Key。中文识别效果优秀。 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DashScopeSttProvider implements SttProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; + private static final String DEFAULT_MODEL = "paraformer-v2"; + + @Override public String id() { return "dashscope"; } + @Override public String label() { return "DashScope (Paraformer)"; } + @Override public boolean requiresCredential() { return true; } + @Override public int autoDetectOrder() { return 150; } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { return modelProviderService.isProviderConfigured("dashscope"); } + catch (Exception e) { return false; } + } + + @Override + public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + try { + String apiKey = modelProviderService.getProviderConfig("dashscope").getApiKey(); + if (apiKey == null) return SttResult.failure("DashScope API Key 未配置"); + + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + String fileName = request.getFileName() != null ? request.getFileName() : "audio.ogg"; + + HttpResponse response = HttpRequest.post(BASE_URL + "/audio/transcriptions") + .header("Authorization", "Bearer " + apiKey) + .form("model", model) + .form("file", request.getAudioData(), request.getContentType(), fileName) + .timeout(60_000) + .execute(); + + if (response.getStatus() == 200) { + JsonNode result = objectMapper.readTree(response.body()); + String text = result.path("text").asText(""); + log.info("[DashScope STT] Transcribed {} chars (model={})", text.length(), model); + return SttResult.success(text); + } else { + log.warn("[DashScope STT] Failed: HTTP {} - {}", response.getStatus(), response.body()); + return SttResult.failure("DashScope STT 失败: HTTP " + response.getStatus()); + } + } catch (Exception e) { + log.error("[DashScope STT] Error: {}", e.getMessage(), e); + return SttResult.failure("DashScope STT 异常: " + e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java b/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java new file mode 100644 index 00000000..923dd261 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java @@ -0,0 +1,74 @@ +package vip.mate.stt.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.stt.SttProvider; +import vip.mate.stt.SttRequest; +import vip.mate.stt.SttResult; +import vip.mate.system.model.SystemSettingsDTO; + +/** + * OpenAI STT Provider — Whisper / gpt-4o-mini-transcribe + *

+ * 复用模型管理中的 OpenAI API Key。 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OpenAiSttProvider implements SttProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String DEFAULT_MODEL = "whisper-1"; + + @Override public String id() { return "openai"; } + @Override public String label() { return "OpenAI Whisper"; } + @Override public boolean requiresCredential() { return true; } + @Override public int autoDetectOrder() { return 100; } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { return modelProviderService.isProviderConfigured("openai"); } + catch (Exception e) { return false; } + } + + @Override + public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + try { + String apiKey = modelProviderService.getProviderConfig("openai").getApiKey(); + String baseUrl = modelProviderService.getProviderConfig("openai").getBaseUrl(); + if (apiKey == null) return SttResult.failure("OpenAI API Key 未配置"); + + String url = (baseUrl != null ? baseUrl : "https://api.openai.com") + "/v1/audio/transcriptions"; + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + String fileName = request.getFileName() != null ? request.getFileName() : "audio.ogg"; + + HttpResponse response = HttpRequest.post(url) + .header("Authorization", "Bearer " + apiKey) + .form("model", model) + .form("file", request.getAudioData(), request.getContentType(), fileName) + .timeout(60_000) + .execute(); + + if (response.getStatus() == 200) { + JsonNode result = objectMapper.readTree(response.body()); + String text = result.path("text").asText(""); + log.info("[OpenAI STT] Transcribed {} chars (model={})", text.length(), model); + return SttResult.success(text); + } else { + log.warn("[OpenAI STT] Failed: HTTP {} - {}", response.getStatus(), response.body()); + return SttResult.failure("OpenAI STT 失败: HTTP " + response.getStatus()); + } + } catch (Exception e) { + log.error("[OpenAI STT] Error: {}", e.getMessage(), e); + return SttResult.failure("OpenAI STT 异常: " + e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java index be27040b..9d0471a8 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java @@ -61,6 +61,16 @@ public class SystemSettingsDTO { private String klingAccessKeyMasked; private String klingSecretKeyMasked; + // --- Runway --- + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private String runwayApiKey; + private String runwayApiKeyMasked; + + // --- MiniMax (Hailuo) --- + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private String minimaxApiKey; + private String minimaxApiKeyMasked; + // ===== 图片生成配置 ===== /** 是否启用图片生成能力 */ private Boolean imageEnabled; @@ -82,4 +92,16 @@ public class SystemSettingsDTO { private String ttsDefaultVoice; /** 默认语速 0.5-2.0 */ private Double ttsSpeed; + + // ===== STT 语音识别配置 ===== + private Boolean sttEnabled; + /** 首选 STT provider: auto / openai / dashscope */ + private String sttProvider; + private Boolean sttFallbackEnabled; + + // ===== 音乐生成配置 ===== + private Boolean musicEnabled; + /** 首选音乐 provider: auto / google-lyria / minimax */ + private String musicProvider; + private Boolean musicFallbackEnabled; } diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index b6006b73..8471102c 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -44,11 +44,23 @@ public class SystemSettingService { private static final String TTS_AUTO_MODE_KEY = "ttsAutoMode"; private static final String TTS_DEFAULT_VOICE_KEY = "ttsDefaultVoice"; private static final String TTS_SPEED_KEY = "ttsSpeed"; + + // STT 配置 keys + private static final String STT_ENABLED_KEY = "sttEnabled"; + private static final String STT_PROVIDER_KEY = "sttProvider"; + private static final String STT_FALLBACK_ENABLED_KEY = "sttFallbackEnabled"; + + // 音乐生成配置 keys + private static final String MUSIC_ENABLED_KEY = "musicEnabled"; + private static final String MUSIC_PROVIDER_KEY = "musicProvider"; + private static final String MUSIC_FALLBACK_ENABLED_KEY = "musicFallbackEnabled"; private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey"; private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl"; private static final String FAL_API_KEY_KEY = "falApiKey"; private static final String KLING_ACCESS_KEY_KEY = "klingAccessKey"; private static final String KLING_SECRET_KEY_KEY = "klingSecretKey"; + private static final String RUNWAY_API_KEY_KEY = "runwayApiKey"; + private static final String MINIMAX_API_KEY_KEY = "minimaxApiKey"; private final SystemSettingMapper systemSettingMapper; @@ -81,6 +93,8 @@ public class SystemSettingService { dto.setFalApiKeyMasked(maskApiKey(getValue(FAL_API_KEY_KEY, ""))); dto.setKlingAccessKeyMasked(maskApiKey(getValue(KLING_ACCESS_KEY_KEY, ""))); dto.setKlingSecretKeyMasked(maskApiKey(getValue(KLING_SECRET_KEY_KEY, ""))); + dto.setRunwayApiKeyMasked(maskApiKey(getValue(RUNWAY_API_KEY_KEY, ""))); + dto.setMinimaxApiKeyMasked(maskApiKey(getValue(MINIMAX_API_KEY_KEY, ""))); // 图片生成配置 dto.setImageEnabled(Boolean.parseBoolean(getValue(IMAGE_ENABLED_KEY, "false"))); @@ -95,6 +109,16 @@ public class SystemSettingService { dto.setTtsDefaultVoice(getValue(TTS_DEFAULT_VOICE_KEY, "")); String speedStr = getValue(TTS_SPEED_KEY, "1.0"); try { dto.setTtsSpeed(Double.parseDouble(speedStr)); } catch (NumberFormatException e) { dto.setTtsSpeed(1.0); } + + // STT 配置 + dto.setSttEnabled(Boolean.parseBoolean(getValue(STT_ENABLED_KEY, "false"))); + dto.setSttProvider(getValue(STT_PROVIDER_KEY, "auto")); + dto.setSttFallbackEnabled(Boolean.parseBoolean(getValue(STT_FALLBACK_ENABLED_KEY, "true"))); + + // 音乐生成配置 + dto.setMusicEnabled(Boolean.parseBoolean(getValue(MUSIC_ENABLED_KEY, "false"))); + dto.setMusicProvider(getValue(MUSIC_PROVIDER_KEY, "auto")); + dto.setMusicFallbackEnabled(Boolean.parseBoolean(getValue(MUSIC_FALLBACK_ENABLED_KEY, "true"))); return dto; } @@ -111,6 +135,8 @@ public class SystemSettingService { dto.setFalApiKey(getValue(FAL_API_KEY_KEY, "")); dto.setKlingAccessKey(getValue(KLING_ACCESS_KEY_KEY, "")); dto.setKlingSecretKey(getValue(KLING_SECRET_KEY_KEY, "")); + dto.setRunwayApiKey(getValue(RUNWAY_API_KEY_KEY, "")); + dto.setMinimaxApiKey(getValue(MINIMAX_API_KEY_KEY, "")); return dto; } @@ -193,6 +219,12 @@ public class SystemSettingService { if (dto.getKlingSecretKey() != null && !dto.getKlingSecretKey().isBlank()) { saveValue(KLING_SECRET_KEY_KEY, dto.getKlingSecretKey(), "快手可灵 Secret Key"); } + if (dto.getRunwayApiKey() != null && !dto.getRunwayApiKey().isBlank()) { + saveValue(RUNWAY_API_KEY_KEY, dto.getRunwayApiKey(), "Runway API Key"); + } + if (dto.getMinimaxApiKey() != null && !dto.getMinimaxApiKey().isBlank()) { + saveValue(MINIMAX_API_KEY_KEY, dto.getMinimaxApiKey(), "MiniMax API Key"); + } // 图片生成配置 if (dto.getImageEnabled() != null) { @@ -224,6 +256,28 @@ public class SystemSettingService { if (dto.getTtsSpeed() != null) { saveValue(TTS_SPEED_KEY, String.valueOf(dto.getTtsSpeed()), "TTS 默认语速"); } + + // STT 配置 + if (dto.getSttEnabled() != null) { + saveValue(STT_ENABLED_KEY, String.valueOf(dto.getSttEnabled()), "是否启用 STT 语音识别"); + } + if (dto.getSttProvider() != null) { + saveValue(STT_PROVIDER_KEY, dto.getSttProvider(), "STT 首选 Provider"); + } + if (dto.getSttFallbackEnabled() != null) { + saveValue(STT_FALLBACK_ENABLED_KEY, String.valueOf(dto.getSttFallbackEnabled()), "STT Provider 级 Fallback"); + } + + // 音乐生成配置 + if (dto.getMusicEnabled() != null) { + saveValue(MUSIC_ENABLED_KEY, String.valueOf(dto.getMusicEnabled()), "是否启用音乐生成"); + } + if (dto.getMusicProvider() != null) { + saveValue(MUSIC_PROVIDER_KEY, dto.getMusicProvider(), "音乐生成首选 Provider"); + } + if (dto.getMusicFallbackEnabled() != null) { + saveValue(MUSIC_FALLBACK_ENABLED_KEY, String.valueOf(dto.getMusicFallbackEnabled()), "音乐 Provider 级 Fallback"); + } return getSettings(); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/ToolRegistry.java b/mateclaw-server/src/main/java/vip/mate/tool/ToolRegistry.java index 9dad64c9..f15c89d0 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/ToolRegistry.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/ToolRegistry.java @@ -38,10 +38,12 @@ public class ToolRegistry { * 通过数据库 enabled 标志过滤，确保 UI 开关真正生效 */ public List getEnabledTools() { - // 1. 从数据库获取已启用的 beanName 集合 - Set enabledBeanNames = toolMapper.selectList( + // 1. 从数据库获取明确禁用的 beanName 黑名单 + // 逻辑：只有 DB 中存在记录且 enabled=false 的才跳过 + // DB 中没有记录的 bean 默认启用（向后兼容 + 新工具自动可用） + Set disabledBeanNames = toolMapper.selectList( new LambdaQueryWrapper() - .eq(ToolEntity::getEnabled, true) + .eq(ToolEntity::getEnabled, false) .isNotNull(ToolEntity::getBeanName) ).stream() .map(ToolEntity::getBeanName) @@ -59,13 +61,12 @@ public class ToolRegistry { .anyMatch(m -> m.isAnnotationPresent(Tool.class)); if (hasToolMethod) { - // 3. 如果 DB 中有该 beanName 的记录，则按 DB enabled 状态决定是否加入 - // 如果 DB 中没有记录（未注册），默认加入（保持向后兼容） - if (enabledBeanNames.isEmpty() || enabledBeanNames.contains(beanName)) { + // 3. 只有 DB 中明确 enabled=false 的才跳过，其余全部启用 + if (disabledBeanNames.contains(beanName)) { + log.debug("Skipped disabled tool bean: {} (beanName={})", bean.getClass().getSimpleName(), beanName); + } else { tools.add(bean); log.debug("Registered tool bean: {} (beanName={})", bean.getClass().getSimpleName(), beanName); - } else { - log.debug("Skipped disabled tool bean: {} (beanName={})", bean.getClass().getSimpleName(), beanName); } } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java new file mode 100644 index 00000000..eef4bf1f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/GoogleImagenProvider.java @@ -0,0 +1,187 @@ +package vip.mate.tool.image.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.*; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Set; + +/** + * Google Imagen 图片生成 Provider — 使用 Gemini API 的图片生成能力 + *

+ * 同步模式：直接返回 Base64 图片数据。 + * 复用已有的 Google/Gemini LLM provider 的 API Key。 + *

+ * API: POST https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class GoogleImagenProvider implements ImageGenerationProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://generativelanguage.googleapis.com"; + private static final String DEFAULT_MODEL = "gemini-2.0-flash-preview-image-generation"; + + @Override + public String id() { + return "google-imagen"; + } + + @Override + public String label() { + return "Google Imagen"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 250; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.TEXT_TO_IMAGE); + } + + @Override + public ImageProviderCapabilities detailedCapabilities() { + return ImageProviderCapabilities.builder() + .modes(capabilities()) + .supportedSizes(List.of("1024x1024", "1024x1536", "1536x1024", "1024x1792", "1792x1024")) + .aspectRatios(List.of("1:1", "3:4", "4:3", "9:16", "16:9")) + .maxCount(4) + .defaultModel(DEFAULT_MODEL) + .models(List.of("gemini-2.0-flash-preview-image-generation", "imagen-4.0-generate-preview", "imagen-4.0-ultra-generate-preview")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { + return modelProviderService.isProviderConfigured("google"); + } catch (Exception e) { + return false; + } + } + + @Override + public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { + try { + String apiKey = getApiKey(); + if (apiKey == null) { + return ImageSubmitResult.failure(id(), "Google API Key 未配置"); + } + + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + + // 构建请求体 + ObjectNode body = objectMapper.createObjectNode(); + + // contents + ArrayNode contents = body.putArray("contents"); + ObjectNode content = contents.addObject(); + content.put("role", "user"); + ArrayNode parts = content.putArray("parts"); + parts.addObject().put("text", request.getPrompt()); + + // generationConfig + ObjectNode genConfig = body.putObject("generationConfig"); + ArrayNode modalities = genConfig.putArray("responseModalities"); + modalities.add("TEXT"); + modalities.add("IMAGE"); + + if (request.getAspectRatio() != null) { + ObjectNode imageConfig = genConfig.putObject("imageConfig"); + imageConfig.put("aspectRatio", request.getAspectRatio()); + } + + String url = BASE_URL + "/v1beta/models/" + model + ":generateContent?key=" + apiKey; + + HttpResponse response = HttpRequest.post(url) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(60_000) + .execute(); + + if (response.getStatus() != 200) { + String errBody = response.body(); + log.warn("[Google Imagen] Failed: HTTP {} - {}", response.getStatus(), errBody); + return ImageSubmitResult.failure(id(), "Google Imagen 失败: HTTP " + response.getStatus()); + } + + JsonNode result = objectMapper.readTree(response.body()); + List imageUrls = extractImagesFromResponse(result); + + if (imageUrls.isEmpty()) { + return ImageSubmitResult.failure(id(), "Google Imagen 未返回图片"); + } + + log.info("[Google Imagen] Generated {} images (model={})", imageUrls.size(), model); + return ImageSubmitResult.syncSuccess(id(), imageUrls); + + } catch (Exception e) { + log.error("[Google Imagen] Error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), "Google Imagen 异常: " + e.getMessage()); + } + } + + /** + * 从 Gemini 响应中提取 Base64 图片，转换为 data URI + */ + private List extractImagesFromResponse(JsonNode result) { + List images = new ArrayList<>(); + + JsonNode candidates = result.path("candidates"); + if (candidates.isArray()) { + for (JsonNode candidate : candidates) { + JsonNode parts = candidate.path("content").path("parts"); + if (parts.isArray()) { + for (JsonNode part : parts) { + // 尝试 inlineData 或 inline_data + JsonNode inlineData = part.has("inlineData") ? part.get("inlineData") + : part.path("inline_data"); + if (inlineData.has("data")) { + String mimeType = inlineData.has("mimeType") + ? inlineData.get("mimeType").asText("image/png") + : inlineData.path("mime_type").asText("image/png"); + String base64Data = inlineData.get("data").asText(); + // 返回 data URI 格式 + images.add("data:" + mimeType + ";base64," + base64Data); + } + } + } + } + } + return images; + } + + private String getApiKey() { + try { + return modelProviderService.getProviderConfig("google").getApiKey(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/MiniMaxImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/MiniMaxImageProvider.java new file mode 100644 index 00000000..e3f61c96 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/MiniMaxImageProvider.java @@ -0,0 +1,151 @@ +package vip.mate.tool.image.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * MiniMax 图片生成 Provider — image-01 模型 + *

+ * 同步模式：返回 Base64 图片。 + * 复用视频生成中的 MiniMax API Key。 + *

+ * API: POST https://api.minimax.io/v1/image_generation + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class MiniMaxImageProvider implements ImageGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://api.minimax.io"; + private static final String DEFAULT_MODEL = "image-01"; + + @Override + public String id() { + return "minimax"; + } + + @Override + public String label() { + return "MiniMax Image"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 350; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.TEXT_TO_IMAGE); + } + + @Override + public ImageProviderCapabilities detailedCapabilities() { + return ImageProviderCapabilities.builder() + .modes(capabilities()) + .supportedSizes(List.of("1024x1024")) + .aspectRatios(List.of("1:1", "16:9", "4:3", "3:2", "2:3", "3:4", "9:16")) + .maxCount(9) + .defaultModel(DEFAULT_MODEL) + .models(List.of("image-01")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + // 复用视频生成的 MiniMax API Key + return StringUtils.hasText(config.getMinimaxApiKey()); + } + + @Override + public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { + try { + String apiKey = config.getMinimaxApiKey(); + String model = request.getModel() != null && !request.getModel().isBlank() + ? request.getModel() : DEFAULT_MODEL; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("prompt", request.getPrompt()); + body.put("response_format", "url"); + body.put("n", request.getCount() != null ? request.getCount() : 1); + + if (request.getAspectRatio() != null) { + body.put("aspect_ratio", request.getAspectRatio()); + } + + HttpResponse response = HttpRequest.post(BASE_URL + "/v1/image_generation") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(60_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + int statusCode = result.path("base_resp").path("status_code").asInt(-1); + if (statusCode != 0) { + String errMsg = result.path("base_resp").path("status_msg").asText("未知错误"); + log.warn("[MiniMax Image] Failed: {}", errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + + List imageUrls = new ArrayList<>(); + + // 尝试从 data.image_base64 提取 + JsonNode imageBase64Array = result.path("data").path("image_base64"); + if (imageBase64Array.isArray()) { + for (JsonNode b64 : imageBase64Array) { + String base64Str = b64.asText(); + if (StringUtils.hasText(base64Str)) { + imageUrls.add("data:image/png;base64," + base64Str); + } + } + } + + // 尝试从 data.image_urls 提取（response_format=url 时） + JsonNode imageUrlsNode = result.path("data").path("image_urls"); + if (imageUrlsNode.isArray()) { + for (JsonNode urlNode : imageUrlsNode) { + String url = urlNode.asText(null); + if (StringUtils.hasText(url)) { + imageUrls.add(url); + } + } + } + + if (imageUrls.isEmpty()) { + return ImageSubmitResult.failure(id(), "MiniMax 未返回图片数据"); + } + + log.info("[MiniMax Image] Generated {} images (model={})", imageUrls.size(), model); + return ImageSubmitResult.syncSuccess(id(), imageUrls); + + } catch (Exception e) { + log.error("[MiniMax Image] Error: {}", e.getMessage(), e); + return ImageSubmitResult.failure(id(), "MiniMax Image 异常: " + e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerateTool.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerateTool.java new file mode 100644 index 00000000..a02e48ef --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerateTool.java @@ -0,0 +1,52 @@ +package vip.mate.tool.music; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.tool.builtin.ToolExecutionContext; + +import java.util.Map; + +/** + * 音乐生成工具 — Agent 可通过 @Tool 注解调用 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class MusicGenerateTool { + + private final MusicGenerationService musicGenerationService; + + @Tool(description = "生成音乐或歌曲。支持文字描述生成音乐、歌词谱曲、纯音乐等模式。支持 Google Lyria 和 MiniMax Music 等 Provider。") + public String music_generate( + @ToolParam(description = "音乐风格/场景描述，如：'轻快的钢琴爵士乐'、'史诗电影配乐'、'欢快的流行歌曲'") String prompt, + @ToolParam(description = "歌词文本（可选，不填则由 AI 生成或生成纯音乐）") String lyrics, + @ToolParam(description = "是否生成纯音乐（无人声），默认 false") Boolean instrumental) { + + String conversationId = ToolExecutionContext.conversationId(); + if (conversationId == null) { + return "无法获取会话 ID"; + } + + MusicGenerationRequest request = MusicGenerationRequest.builder() + .prompt(prompt) + .lyrics(lyrics) + .instrumental(instrumental != null ? instrumental : false) + .build(); + + Map result = musicGenerationService.generate(conversationId, request); + + if (Boolean.TRUE.equals(result.get("success"))) { + StringBuilder sb = new StringBuilder("音乐生成完成！\n"); + sb.append("播放链接: ").append(result.get("audioUrl")); + if (result.containsKey("lyrics") && result.get("lyrics") != null) { + sb.append("\n\n歌词:\n").append(result.get("lyrics")); + } + return sb.toString(); + } else { + return "音乐生成失败: " + result.get("error"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationProvider.java new file mode 100644 index 00000000..f1296a77 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationProvider.java @@ -0,0 +1,23 @@ +package vip.mate.tool.music; + +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.List; + +/** + * 音乐生成提供商接口 + */ +public interface MusicGenerationProvider { + String id(); + String label(); + boolean requiresCredential(); + int autoDetectOrder(); + boolean isAvailable(SystemSettingsDTO config); + List availableModels(); + String defaultModel(); + + /** + * 生成音乐（同步返回音频字节） + */ + MusicGenerationResult generate(MusicGenerationRequest request, SystemSettingsDTO config); +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationRequest.java new file mode 100644 index 00000000..032960d2 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationRequest.java @@ -0,0 +1,17 @@ +package vip.mate.tool.music; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class MusicGenerationRequest { + private String prompt; + private String lyrics; + @Builder.Default + private Boolean instrumental = false; + private Integer durationSeconds; + private String model; + @Builder.Default + private String format = "mp3"; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationResult.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationResult.java new file mode 100644 index 00000000..e39c9490 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationResult.java @@ -0,0 +1,27 @@ +package vip.mate.tool.music; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class MusicGenerationResult { + private boolean success; + private byte[] audioData; + private String contentType; + private String format; + private String lyrics; + private String errorMessage; + + public static MusicGenerationResult success(byte[] audioData, String contentType, String format) { + return MusicGenerationResult.builder().success(true).audioData(audioData).contentType(contentType).format(format).build(); + } + + public static MusicGenerationResult successWithLyrics(byte[] audioData, String contentType, String format, String lyrics) { + return MusicGenerationResult.builder().success(true).audioData(audioData).contentType(contentType).format(format).lyrics(lyrics).build(); + } + + public static MusicGenerationResult failure(String errorMessage) { + return MusicGenerationResult.builder().success(false).errorMessage(errorMessage).build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java new file mode 100644 index 00000000..1d36cdcb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java @@ -0,0 +1,85 @@ +package vip.mate.tool.music; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +/** + * 音乐生成服务 + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MusicGenerationService { + + private final SystemSettingService systemSettingService; + private final MusicProviderRegistry providerRegistry; + + private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads"); + + public Map generate(String conversationId, MusicGenerationRequest request) { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + + if (!Boolean.TRUE.equals(config.getMusicEnabled())) { + return Map.of("success", false, "error", "音乐生成功能未启用"); + } + + MusicGenerationResult result = generateWithFallback(request, config); + if (!result.isSuccess()) { + return Map.of("success", false, "error", result.getErrorMessage()); + } + + try { + String fileId = UUID.randomUUID().toString().replace("-", "").substring(0, 12); + Path dir = UPLOAD_ROOT.resolve(conversationId); + Files.createDirectories(dir); + String fileName = "music_" + fileId + "." + result.getFormat(); + Path filePath = dir.resolve(fileName); + Files.write(filePath, result.getAudioData()); + + String audioUrl = "/api/v1/chat/files/" + conversationId + "/" + fileName; + + Map response = new LinkedHashMap<>(); + response.put("success", true); + response.put("audioUrl", audioUrl); + response.put("contentType", result.getContentType()); + response.put("format", result.getFormat()); + if (result.getLyrics() != null) { + response.put("lyrics", result.getLyrics()); + } + return response; + } catch (IOException e) { + log.error("[Music] Failed to save audio: {}", e.getMessage()); + return Map.of("success", false, "error", "音频文件保存失败"); + } + } + + private MusicGenerationResult generateWithFallback(MusicGenerationRequest request, SystemSettingsDTO config) { + MusicGenerationProvider primary = providerRegistry.resolve(config); + if (primary == null) return MusicGenerationResult.failure("没有可用的音乐 Provider"); + + MusicGenerationResult result = primary.generate(request, config); + if (result.isSuccess()) return result; + + List errors = new ArrayList<>(); + errors.add(primary.id() + ": " + result.getErrorMessage()); + + if (Boolean.TRUE.equals(config.getMusicFallbackEnabled())) { + for (MusicGenerationProvider fb : providerRegistry.fallbackCandidates(config, primary.id())) { + result = fb.generate(request, config); + if (result.isSuccess()) return result; + errors.add(fb.id() + ": " + result.getErrorMessage()); + } + } + + return MusicGenerationResult.failure("所有音乐 Provider 均失败\n" + String.join("\n", errors)); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicProviderRegistry.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicProviderRegistry.java new file mode 100644 index 00000000..2e6a42d3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicProviderRegistry.java @@ -0,0 +1,45 @@ +package vip.mate.tool.music; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +@Component +public class MusicProviderRegistry { + + private final List sortedProviders; + private final Map providerMap; + + public MusicProviderRegistry(List providers) { + this.sortedProviders = providers.stream() + .sorted(Comparator.comparingInt(MusicGenerationProvider::autoDetectOrder)) + .toList(); + this.providerMap = providers.stream() + .collect(Collectors.toMap(MusicGenerationProvider::id, Function.identity())); + log.info("注册音乐生成 Provider {} 个: {}", sortedProviders.size(), + sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList()); + } + + public MusicGenerationProvider resolve(SystemSettingsDTO config) { + String configuredId = config.getMusicProvider(); + if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) { + MusicGenerationProvider p = providerMap.get(configuredId); + if (p != null && p.isAvailable(config)) return p; + } + for (MusicGenerationProvider p : sortedProviders) { + if (p.isAvailable(config)) return p; + } + return null; + } + + public List fallbackCandidates(SystemSettingsDTO config, String excludeId) { + return sortedProviders.stream().filter(p -> !p.id().equals(excludeId)).filter(p -> p.isAvailable(config)).toList(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/provider/GoogleLyriaProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/music/provider/GoogleLyriaProvider.java new file mode 100644 index 00000000..0e77feee --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/provider/GoogleLyriaProvider.java @@ -0,0 +1,133 @@ +package vip.mate.tool.music.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.music.MusicGenerationProvider; +import vip.mate.tool.music.MusicGenerationRequest; +import vip.mate.tool.music.MusicGenerationResult; + +import java.util.Base64; +import java.util.List; + +/** + * Google Lyria 音乐生成 Provider + *

+ * 使用 Gemini API 的音频生成能力。复用 Google LLM API Key。 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class GoogleLyriaProvider implements MusicGenerationProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://generativelanguage.googleapis.com"; + private static final String DEFAULT_MODEL = "lyria-3-clip-preview"; + + @Override public String id() { return "google-lyria"; } + @Override public String label() { return "Google Lyria"; } + @Override public boolean requiresCredential() { return true; } + @Override public int autoDetectOrder() { return 100; } + @Override public String defaultModel() { return DEFAULT_MODEL; } + @Override public List availableModels() { return List.of("lyria-3-clip-preview"); } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { return modelProviderService.isProviderConfigured("google"); } + catch (Exception e) { return false; } + } + + @Override + public MusicGenerationResult generate(MusicGenerationRequest request, SystemSettingsDTO config) { + try { + String apiKey = modelProviderService.getProviderConfig("google").getApiKey(); + if (apiKey == null) return MusicGenerationResult.failure("Google API Key 未配置"); + + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + + // 构建 prompt + StringBuilder prompt = new StringBuilder(request.getPrompt()); + if (Boolean.TRUE.equals(request.getInstrumental())) { + prompt.append("\nInstrumental only, no vocals."); + } + if (request.getLyrics() != null && !request.getLyrics().isBlank()) { + prompt.append("\nLyrics:\n").append(request.getLyrics()); + } + + ObjectNode body = objectMapper.createObjectNode(); + ArrayNode contents = body.putArray("contents"); + ObjectNode content = contents.addObject(); + ArrayNode parts = content.putArray("parts"); + parts.addObject().put("text", prompt.toString()); + + ObjectNode genConfig = body.putObject("generationConfig"); + ArrayNode modalities = genConfig.putArray("responseModalities"); + modalities.add("AUDIO"); + modalities.add("TEXT"); + + String url = BASE_URL + "/v1beta/models/" + model + ":generateContent?key=" + apiKey; + + HttpResponse response = HttpRequest.post(url) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(120_000) // 音乐生成较慢 + .execute(); + + if (response.getStatus() != 200) { + log.warn("[Google Lyria] Failed: HTTP {} - {}", response.getStatus(), response.body()); + return MusicGenerationResult.failure("Google Lyria 失败: HTTP " + response.getStatus()); + } + + JsonNode result = objectMapper.readTree(response.body()); + return extractAudioFromResponse(result); + + } catch (Exception e) { + log.error("[Google Lyria] Error: {}", e.getMessage(), e); + return MusicGenerationResult.failure("Google Lyria 异常: " + e.getMessage()); + } + } + + private MusicGenerationResult extractAudioFromResponse(JsonNode result) { + String lyrics = null; + byte[] audioData = null; + String mimeType = "audio/mpeg"; + + JsonNode candidates = result.path("candidates"); + if (candidates.isArray()) { + for (JsonNode candidate : candidates) { + JsonNode parts = candidate.path("content").path("parts"); + if (parts.isArray()) { + for (JsonNode part : parts) { + if (part.has("text")) { + lyrics = part.get("text").asText(); + } + JsonNode inlineData = part.has("inlineData") ? part.get("inlineData") : part.path("inline_data"); + if (inlineData.has("data")) { + mimeType = inlineData.has("mimeType") ? inlineData.get("mimeType").asText("audio/mpeg") + : inlineData.path("mime_type").asText("audio/mpeg"); + audioData = Base64.getDecoder().decode(inlineData.get("data").asText()); + } + } + } + } + } + + if (audioData == null || audioData.length == 0) { + return MusicGenerationResult.failure("Google Lyria 未返回音频数据"); + } + + String format = mimeType.contains("wav") ? "wav" : "mp3"; + log.info("[Google Lyria] Generated {} bytes audio", audioData.length); + return MusicGenerationResult.successWithLyrics(audioData, mimeType, format, lyrics); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/provider/MiniMaxMusicProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/music/provider/MiniMaxMusicProvider.java new file mode 100644 index 00000000..ae395193 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/provider/MiniMaxMusicProvider.java @@ -0,0 +1,140 @@ +package vip.mate.tool.music.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.music.MusicGenerationProvider; +import vip.mate.tool.music.MusicGenerationRequest; +import vip.mate.tool.music.MusicGenerationResult; + +import java.util.List; + +/** + * MiniMax 音乐生成 Provider — music-2.5+ + *

+ * 复用视频生成中的 MiniMax API Key。 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class MiniMaxMusicProvider implements MusicGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://api.minimax.io"; + private static final String DEFAULT_MODEL = "music-2.5+"; + + @Override public String id() { return "minimax"; } + @Override public String label() { return "MiniMax Music"; } + @Override public boolean requiresCredential() { return true; } + @Override public int autoDetectOrder() { return 200; } + @Override public String defaultModel() { return DEFAULT_MODEL; } + @Override public List availableModels() { return List.of("music-2.5+", "music-2.5"); } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return StringUtils.hasText(config.getMinimaxApiKey()); + } + + @Override + public MusicGenerationResult generate(MusicGenerationRequest request, SystemSettingsDTO config) { + try { + String apiKey = config.getMinimaxApiKey(); + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + + // 构建 prompt（可加时长提示） + String prompt = request.getPrompt(); + if (request.getDurationSeconds() != null) { + prompt += " Duration: " + request.getDurationSeconds() + " seconds."; + } + body.put("prompt", prompt); + + if (Boolean.TRUE.equals(request.getInstrumental())) { + body.put("is_instrumental", true); + } + + if (request.getLyrics() != null && !request.getLyrics().isBlank()) { + body.put("lyrics", request.getLyrics()); + } else if (!Boolean.TRUE.equals(request.getInstrumental())) { + body.put("lyrics_optimizer", true); + } + + body.put("output_format", "url"); + ObjectNode audioSetting = body.putObject("audio_setting"); + audioSetting.put("sample_rate", 44100); + audioSetting.put("bitrate", 256000); + audioSetting.put("format", "mp3"); + + HttpResponse response = HttpRequest.post(BASE_URL + "/v1/music_generation") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(120_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + int statusCode = result.path("base_resp").path("status_code").asInt(-1); + if (statusCode != 0) { + String errMsg = result.path("base_resp").path("status_msg").asText("未知错误"); + log.warn("[MiniMax Music] Failed: {}", errMsg); + return MusicGenerationResult.failure(errMsg); + } + + // 提取音频 URL 或内联数据 + String audioUrl = result.has("audio_url") ? result.get("audio_url").asText(null) + : result.path("data").path("audio_url").asText(null); + String lyrics = result.has("lyrics") ? result.get("lyrics").asText(null) + : result.path("data").path("lyrics").asText(null); + + if (StringUtils.hasText(audioUrl)) { + // 下载音频 + byte[] audioData = HttpRequest.get(audioUrl).timeout(30_000).execute().bodyBytes(); + log.info("[MiniMax Music] Generated {} bytes audio (model={})", audioData.length, model); + return MusicGenerationResult.successWithLyrics(audioData, "audio/mpeg", "mp3", lyrics); + } + + // 尝试 inline audio + String inlineAudio = result.has("audio") ? result.get("audio").asText(null) + : result.path("data").path("audio").asText(null); + if (StringUtils.hasText(inlineAudio)) { + byte[] audioData = decodeAudio(inlineAudio); + return MusicGenerationResult.successWithLyrics(audioData, "audio/mpeg", "mp3", lyrics); + } + + return MusicGenerationResult.failure("MiniMax 未返回音频数据"); + + } catch (Exception e) { + log.error("[MiniMax Music] Error: {}", e.getMessage(), e); + return MusicGenerationResult.failure("MiniMax Music 异常: " + e.getMessage()); + } + } + + private byte[] decodeAudio(String data) { + // MiniMax 可能返回 hex 或 base64 + if (data.matches("^[0-9a-fA-F]+$") && data.length() % 2 == 0) { + return hexToBytes(data); + } + return java.util.Base64.getDecoder().decode(data); + } + + private byte[] hexToBytes(String hex) { + int len = hex.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4) + + Character.digit(hex.charAt(i + 1), 16)); + } + return data; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/MiniMaxVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/MiniMaxVideoProvider.java new file mode 100644 index 00000000..e05195f7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/MiniMaxVideoProvider.java @@ -0,0 +1,176 @@ +package vip.mate.tool.video.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.video.*; + +import java.util.List; +import java.util.Set; + +/** + * MiniMax (Hailuo / 海螺) 视频生成 Provider + *

+ * API 文档: https://platform.minimaxi.com/document/video-generation + * 鉴权: Bearer Token + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class MiniMaxVideoProvider implements VideoGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://api.minimax.io"; + private static final String DEFAULT_MODEL = "MiniMax-Hailuo-2.3"; + + @Override + public String id() { + return "minimax"; + } + + @Override + public String label() { + return "MiniMax (Hailuo)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 350; + } + + @Override + public Set capabilities() { + return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO); + } + + @Override + public VideoProviderCapabilities detailedCapabilities() { + return VideoProviderCapabilities.builder() + .modes(capabilities()) + .aspectRatios(List.of("16:9", "9:16", "1:1")) + .supportedDurations(List.of(6, 10)) + .maxDurationSeconds(10) + .defaultModel(DEFAULT_MODEL) + .models(List.of("MiniMax-Hailuo-2.3", "MiniMax-Hailuo-2.3-Fast", "I2V-01-live")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return StringUtils.hasText(config.getMinimaxApiKey()); + } + + @Override + public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { + try { + String apiKey = config.getMinimaxApiKey(); + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("prompt", request.getPrompt()); + + if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { + body.put("first_frame_image", request.getImageUrl()); + } + if (request.getDurationSeconds() != null) { + body.put("duration", request.getDurationSeconds()); + } + + HttpResponse response = HttpRequest.post(BASE_URL + "/v1/video_generation") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + // MiniMax 返回 { task_id, base_resp: { status_code, status_msg } } + int statusCode = result.path("base_resp").path("status_code").asInt(-1); + if (statusCode == 0 && result.has("task_id")) { + String taskId = result.get("task_id").asText(); + log.info("[MiniMax] Submitted task: {} (model={})", taskId, model); + return VideoSubmitResult.success(taskId, id()); + } else { + String errMsg = result.path("base_resp").path("status_msg").asText("未知错误"); + log.warn("[MiniMax] Submit failed: {}", errMsg); + return VideoSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[MiniMax] Submit error: {}", e.getMessage(), e); + return VideoSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + try { + String apiKey = config.getMinimaxApiKey(); + + HttpResponse response = HttpRequest.get( + BASE_URL + "/v1/query/video_generation?task_id=" + providerTaskId) + .header("Authorization", "Bearer " + apiKey) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + String status = result.path("status").asText(); + + return switch (status) { + case "Success" -> { + // 优先取 video_url，备选 file_id + String videoUrl = result.has("video_url") ? result.get("video_url").asText(null) : null; + if (videoUrl == null && result.has("file_id")) { + videoUrl = resolveFileUrl(result.get("file_id").asText(), apiKey); + } + yield TaskPollResult.succeeded(videoUrl, null, result.toString()); + } + case "Fail" -> { + String errMsg = result.path("base_resp").path("status_msg").asText("任务失败"); + yield TaskPollResult.failed(errMsg); + } + case "Processing" -> TaskPollResult.running(null); + default -> TaskPollResult.pending(null); // Preparing + }; + } catch (Exception e) { + log.error("[MiniMax] Poll error for task {}: {}", providerTaskId, e.getMessage()); + return null; + } + } + + /** + * 通过 file_id 获取视频下载 URL + */ + private String resolveFileUrl(String fileId, String apiKey) { + try { + HttpResponse response = HttpRequest.get( + BASE_URL + "/v1/files/retrieve?file_id=" + fileId) + .header("Authorization", "Bearer " + apiKey) + .timeout(10_000) + .execute(); + JsonNode result = objectMapper.readTree(response.body()); + JsonNode file = result.path("file"); + return file.has("download_url") ? file.get("download_url").asText(null) : null; + } catch (Exception e) { + log.warn("[MiniMax] Failed to resolve file URL for {}: {}", fileId, e.getMessage()); + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/RunwayVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/RunwayVideoProvider.java new file mode 100644 index 00000000..d975dd39 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/RunwayVideoProvider.java @@ -0,0 +1,171 @@ +package vip.mate.tool.video.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.video.*; + +import java.util.List; +import java.util.Set; + +/** + * Runway 视频生成 Provider — gen4.5 / gen4_turbo / gen3a_turbo + *

+ * API 文档: https://docs.runwayml.com/ + * 鉴权: Bearer Token + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class RunwayVideoProvider implements VideoGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://api.dev.runwayml.com"; + private static final String API_VERSION = "2024-11-06"; + private static final String DEFAULT_MODEL = "gen4_turbo"; + + @Override + public String id() { + return "runway"; + } + + @Override + public String label() { + return "Runway"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 250; + } + + @Override + public Set capabilities() { + return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO); + } + + @Override + public VideoProviderCapabilities detailedCapabilities() { + return VideoProviderCapabilities.builder() + .modes(capabilities()) + .aspectRatios(List.of("16:9", "9:16")) + .supportedDurations(List.of(5, 10)) + .maxDurationSeconds(10) + .defaultModel(DEFAULT_MODEL) + .models(List.of("gen4.5", "gen4_turbo", "gen3a_turbo")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return StringUtils.hasText(config.getRunwayApiKey()); + } + + @Override + public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { + try { + String apiKey = config.getRunwayApiKey(); + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + + // 根据模式选择端点 + String endpoint = request.getMode() == VideoCapability.IMAGE_TO_VIDEO + ? "/v1/image_to_video" : "/v1/text_to_video"; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("promptText", request.getPrompt()); + + if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { + body.put("promptImage", request.getImageUrl()); + } + if (request.getAspectRatio() != null) { + // Runway 用 "1280:720" 格式 + String ratio = request.getAspectRatio().equals("16:9") ? "1280:720" + : request.getAspectRatio().equals("9:16") ? "720:1280" + : "1280:720"; + body.put("ratio", ratio); + } + if (request.getDurationSeconds() != null) { + body.put("duration", request.getDurationSeconds()); + } + + HttpResponse response = HttpRequest.post(BASE_URL + endpoint) + .header("Authorization", "Bearer " + apiKey) + .header("X-Runway-Version", API_VERSION) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("id")) { + String taskId = result.get("id").asText(); + log.info("[Runway] Submitted task: {} (model={})", taskId, model); + return VideoSubmitResult.success(taskId, id()); + } else { + String errMsg = result.has("error") ? result.get("error").asText() + : "HTTP " + response.getStatus(); + log.warn("[Runway] Submit failed: {}", errMsg); + return VideoSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[Runway] Submit error: {}", e.getMessage(), e); + return VideoSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + try { + String apiKey = config.getRunwayApiKey(); + + HttpResponse response = HttpRequest.get(BASE_URL + "/v1/tasks/" + providerTaskId) + .header("Authorization", "Bearer " + apiKey) + .header("X-Runway-Version", API_VERSION) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + String status = result.path("status").asText(); + + return switch (status) { + case "SUCCEEDED" -> { + // output 是视频 URL 数组 + JsonNode output = result.path("output"); + String videoUrl = null; + if (output.isArray() && !output.isEmpty()) { + videoUrl = output.get(0).asText(); + } + yield TaskPollResult.succeeded(videoUrl, null, result.toString()); + } + case "FAILED", "CANCELLED" -> { + String failure = result.has("failure") ? result.get("failure").asText() : "任务失败"; + yield TaskPollResult.failed(failure); + } + case "RUNNING" -> TaskPollResult.running(null); + case "THROTTLED" -> TaskPollResult.running(null); // 被限流，等同于运行中 + default -> TaskPollResult.pending(null); // PENDING + }; + } catch (Exception e) { + log.error("[Runway] Poll error for task {}: {}", providerTaskId, e.getMessage()); + return null; + } + } +} diff --git a/mateclaw-ui/src/composables/chat/useChat.ts b/mateclaw-ui/src/composables/chat/useChat.ts index 0b4066f5..30b319ca 100644 --- a/mateclaw-ui/src/composables/chat/useChat.ts +++ b/mateclaw-ui/src/composables/chat/useChat.ts @@ -631,35 +631,45 @@ export function useChat(options: UseChatOptions): UseChatReturn { stream.on('async_task_completed', (data) => { console.log('[useChat] Async task completed:', data) if (data.success && streamConversationId) { + let mediaPart: MessageContentPart | null = null if (data.videoUrl) { - // 视频生成完成 - addMessage({ - role: 'assistant', - content: '', - contentParts: [{ - type: 'video', - fileUrl: data.videoUrl, - fileName: `video_${data.taskId}.mp4`, - contentType: 'video/mp4', - }] as MessageContentPart[], - status: 'completed', - conversationId: streamConversationId, - }) + mediaPart = { + type: 'video', + fileUrl: data.videoUrl, + fileName: `video_${data.taskId}.mp4`, + contentType: 'video/mp4', + } as MessageContentPart } else if (data.imageUrl) { - // 图片生成完成 - addMessage({ - role: 'assistant', - content: '', - contentParts: [{ - type: 'image', - fileUrl: data.imageUrl, - fileName: `image_${data.taskId}.png`, - contentType: 'image/png', - }] as MessageContentPart[], - status: 'completed', - conversationId: streamConversationId, - }) + mediaPart = { + type: 'image', + fileUrl: data.imageUrl, + fileName: `image_${data.taskId}.png`, + contentType: 'image/png', + } as MessageContentPart } + + if (!mediaPart) return + + // 优先附加到当前 assistant 消息（避免图片跑到文字回复上方） + if (currentAssistantId.value) { + const msg = getMessage(currentAssistantId.value) + if (msg) { + const existingParts = (msg as any).contentParts || [] + updateMessage(currentAssistantId.value, { + contentParts: [...existingParts, mediaPart], + } as any) + return + } + } + + // 回退：Agent 已结束，新建独立消息 + addMessage({ + role: 'assistant', + content: '', + contentParts: [mediaPart], + status: 'completed', + conversationId: streamConversationId, + }) } }) diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 925d8ae9..dd66eb2d 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -208,6 +208,8 @@ export default { system: 'System', image: 'Image Generation', tts: 'Text-to-Speech', + stt: 'Speech Recognition', + music: 'Music Generation', video: 'Video Generation', about: 'About', }, @@ -337,6 +339,12 @@ export default { tavilyBaseUrl: 'Tavily Base URL', duckduckgoEnabled: 'DuckDuckGo (Keyless)', searxngBaseUrl: 'SearXNG Base URL', + sttEnabled: 'Enable Speech Recognition', + sttProvider: 'Preferred STT Provider', + sttFallbackEnabled: 'Provider Fallback', + musicEnabled: 'Enable Music Generation', + musicProvider: 'Preferred Music Provider', + musicFallbackEnabled: 'Provider Fallback', ttsEnabled: 'Enable Text-to-Speech', ttsProvider: 'Preferred TTS Provider', ttsFallbackEnabled: 'Provider Fallback', @@ -355,6 +363,8 @@ export default { falApiKey: 'fal.ai API Key', klingAccessKey: 'Kling Access Key', klingSecretKey: 'Kling Secret Key', + runwayApiKey: 'Runway API Key', + minimaxApiKey: 'MiniMax API Key', }, hints: { provider: 'Current implementation applies DashScope model options at runtime.', @@ -370,6 +380,16 @@ export default { tavilyBaseUrl: 'Usually no need to change unless using a custom proxy.', duckduckgoEnabled: 'Free keyless search fallback. No API key required. Enabled by default as a zero-config fallback.', searxngBaseUrl: 'Self-hosted SearXNG instance URL. Auto-configured when deploying via Docker.', + sttEnabled: 'Enable speech-to-text for voice messages. Both providers reuse existing API keys.', + sttProvider: 'Select preferred STT provider. Auto mode picks the first available one.', + sttFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.', + openaiSttInfo: 'Reuses OpenAI API Key from Model Management. Whisper model, supports multilingual auto-detection.', + dashscopeSttInfo: 'Reuses DashScope API Key from Model Management. Paraformer model, excellent Chinese recognition.', + musicEnabled: 'Enable to let Agent use the music generation tool. Google Lyria reuses Google Key.', + musicProvider: 'Select preferred music provider. Auto mode prioritizes Google Lyria.', + musicFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.', + googleLyriaInfo: 'Reuses Google API Key from Model Management. Lyria 3 model, supports lyrics and instrumental.', + minimaxMusicInfo: 'Reuses MiniMax API Key from video settings. music-2.5+ model, supports lyrics and instrumental.', ttsEnabled: 'Enable to use TTS via Read Aloud button or auto mode. Edge TTS is free, no API key needed.', ttsProvider: 'Select preferred TTS provider. Auto mode prioritizes free Edge TTS.', ttsFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.', @@ -385,6 +405,8 @@ export default { openaiImageStatus: 'Reuses the OpenAI API Key configured in Model Management. No extra setup needed.', zhipuImageApiKey: 'Get from bigmodel.cn. CogView-3-Flash model is free. Shared with video generation.', falImageApiKey: 'Get from fal.ai for Flux image generation models. Shared with video generation.', + googleImagenInfo: 'Reuses Google API Key from Model Management. Supports Gemini image generation and Imagen 4.0.', + minimaxImageInfo: 'Reuses MiniMax API Key from video settings. image-01 model, multiple aspect ratios, up to 9 images.', videoEnabled: 'Enable to let Agent use the video generation tool. Requires at least one configured video provider.', videoProvider: 'Select preferred video provider. Auto mode picks the first available one.', videoFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.', @@ -394,9 +416,19 @@ export default { falApiKey: 'Get from fal.ai. One key accesses Kling, Runway, Luma and more.', klingAccessKey: 'Get from Kuaishou Open Platform for Kling video generation API.', klingSecretKey: 'Paired with Access Key for JWT authentication.', + runwayApiKey: 'Get from runwayml.com. Supports gen4.5, gen4_turbo flagship models.', + minimaxApiKey: 'Get from minimaxi.com. Hailuo video with free quota, excellent for Chinese scenes.', }, searchTitle: 'Search Service', searchDesc: 'Configure the built-in search tool provider and API credentials', + sttTitle: 'Speech Recognition', + sttDesc: 'Configure STT speech-to-text with OpenAI Whisper and DashScope Paraformer', + sttProviderOptions: { auto: 'Auto Select' }, + sttProviderTags: { reuseLlmKey: 'Reuses LLM API Key' }, + musicTitle: 'Music Generation', + musicDesc: 'Configure AI music generation with Google Lyria and MiniMax Music', + musicProviderOptions: { auto: 'Auto Select' }, + musicProviderTags: { reuseLlmKey: 'Reuses LLM API Key', sharedWithVideo: 'Shared with Video' }, ttsTitle: 'Text-to-Speech', ttsDesc: 'Configure TTS with Edge TTS (free), OpenAI TTS, and DashScope CosyVoice', ttsProviderOptions: { diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 61c218ec..b66448c3 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -208,6 +208,8 @@ export default { system: '系统设置', image: '图片生成', tts: '语音合成', + stt: '语音识别', + music: '音乐生成', video: '视频生成', about: '关于', }, @@ -337,6 +339,14 @@ export default { tavilyBaseUrl: 'Tavily 接口地址', duckduckgoEnabled: 'DuckDuckGo（免 Key）', searxngBaseUrl: 'SearXNG 地址', + // STT 语音识别 + sttEnabled: '启用语音识别', + sttProvider: '首选 STT Provider', + sttFallbackEnabled: 'Provider 回退', + // 音乐生成 + musicEnabled: '启用音乐生成', + musicProvider: '首选音乐 Provider', + musicFallbackEnabled: 'Provider 回退', // TTS 语音合成 ttsEnabled: '启用语音合成', ttsProvider: '首选 TTS Provider', @@ -358,6 +368,8 @@ export default { falApiKey: 'fal.ai API Key', klingAccessKey: '可灵 Access Key', klingSecretKey: '可灵 Secret Key', + runwayApiKey: 'Runway API Key', + minimaxApiKey: 'MiniMax API Key', }, hints: { provider: '当前版本会把 DashScope 模型参数真实应用到 Agent 调用链路。', @@ -373,6 +385,18 @@ export default { tavilyBaseUrl: '通常无需修改，除非使用自定义代理地址。', duckduckgoEnabled: '免费搜索兜底，无需 API Key。默认开启，作为零配置下的搜索降级方案。', searxngBaseUrl: '自部署 SearXNG 实例地址。Docker 部署时自动配置。', + // STT 语音识别 + sttEnabled: '开启后支持语音消息转文字。OpenAI Whisper 和 DashScope Paraformer 均复用已有 Key。', + sttProvider: '选择首选 STT Provider，auto 模式自动选择可用的 Provider。', + sttFallbackEnabled: '首选 Provider 失败时自动尝试其他已配置的 Provider。', + openaiSttInfo: '复用模型管理中的 OpenAI API Key。使用 Whisper 模型，支持多语言自动识别。', + dashscopeSttInfo: '复用模型管理中的 DashScope API Key。使用 Paraformer 模型，中文识别效果优秀。', + // 音乐生成 + musicEnabled: '开启后 Agent 可通过 music_generate 工具生成音乐。Google Lyria 复用 Google Key。', + musicProvider: '选择首选音乐 Provider，auto 模式优先使用 Google Lyria。', + musicFallbackEnabled: '首选 Provider 失败时自动尝试其他已配置的 Provider。', + googleLyriaInfo: '复用模型管理中的 Google API Key。Lyria 3 模型，支持歌词谱曲和纯音乐生成。', + minimaxMusicInfo: '复用视频生成中的 MiniMax API Key。music-2.5+ 模型，支持歌词和纯音乐。', // TTS 语音合成 ttsEnabled: '开启后可通过消息朗读按钮或自动模式使用语音合成。Edge TTS 免费无需 Key。', ttsProvider: '选择首选 TTS Provider，auto 模式优先使用免费的 Edge TTS。', @@ -390,6 +414,8 @@ export default { openaiImageStatus: '复用模型管理中配置的 OpenAI API Key，无需额外配置。', zhipuImageApiKey: '从 bigmodel.cn 获取。CogView-3-Flash 模型免费。与视频生成共用同一 Key。', falImageApiKey: '从 fal.ai 获取，支持 Flux 系列图片生成模型。与视频生成共用同一 Key。', + googleImagenInfo: '复用模型管理中的 Google API Key。支持 Gemini 图片生成和 Imagen 4.0 模型。', + minimaxImageInfo: '复用视频生成中的 MiniMax API Key。image-01 模型，支持多种画面比例，最多 9 张。', // 视频生成 videoEnabled: '开启后 Agent 可使用视频生成工具。需至少配置一个视频 Provider 的 API Key。', videoProvider: '选择首选视频生成 Provider，auto 模式自动选择第一个可用的。', @@ -400,9 +426,19 @@ export default { falApiKey: '从 fal.ai 获取，一个 Key 可访问 Kling、Runway、Luma 等多个模型。', klingAccessKey: '从快手开放平台获取，用于调用可灵视频生成 API。', klingSecretKey: '与 Access Key 配对使用，用于 JWT 签名鉴权。', + runwayApiKey: '从 runwayml.com 获取。支持 gen4.5、gen4_turbo 等旗舰模型。', + minimaxApiKey: '从 minimaxi.com 获取。海螺 Hailuo 视频，有免费额度，中文场景优秀。', }, searchTitle: '搜索服务', searchDesc: '配置内置搜索工具的提供商与 API 凭证', + sttTitle: '语音识别', + sttDesc: '配置 STT 语音转文字，支持 OpenAI Whisper 和 DashScope Paraformer', + sttProviderOptions: { auto: '自动选择' }, + sttProviderTags: { reuseLlmKey: '复用 LLM API Key' }, + musicTitle: '音乐生成', + musicDesc: '配置 AI 音乐生成，支持 Google Lyria 和 MiniMax Music', + musicProviderOptions: { auto: '自动选择' }, + musicProviderTags: { reuseLlmKey: '复用 LLM API Key', sharedWithVideo: '与视频共用' }, ttsTitle: '语音合成', ttsDesc: '配置 TTS 语音合成，支持 Edge TTS（免费）、OpenAI TTS、DashScope CosyVoice', ttsProviderOptions: { diff --git a/mateclaw-ui/src/router/index.ts b/mateclaw-ui/src/router/index.ts index ef28077c..1cf33ac6 100644 --- a/mateclaw-ui/src/router/index.ts +++ b/mateclaw-ui/src/router/index.ts @@ -97,6 +97,18 @@ const router = createRouter({ component: () => import('@/views/Settings/Tts/index.vue'), meta: { title: 'Settings - TTS' }, }, + { + path: 'stt', + name: 'SettingsStt', + component: () => import('@/views/Settings/Stt/index.vue'), + meta: { title: 'Settings - STT' }, + }, + { + path: 'music', + name: 'SettingsMusic', + component: () => import('@/views/Settings/Music/index.vue'), + meta: { title: 'Settings - Music' }, + }, { path: 'video', name: 'SettingsVideo', diff --git a/mateclaw-ui/src/views/Settings/Image/index.vue b/mateclaw-ui/src/views/Settings/Image/index.vue index f98d58a1..71ba8587 100644 --- a/mateclaw-ui/src/views/Settings/Image/index.vue +++ b/mateclaw-ui/src/views/Settings/Image/index.vue @@ -33,6 +33,8 @@ + + @@ -142,6 +144,36 @@ + + +

+
+ Google Imagen + {{ t('settings.imageProviderTags.reuseLlmKey') }} +
+
+
+
+
{{ t('settings.hints.googleImagenInfo') }}
+
+
+
+
+ + +
+
+ MiniMax Image + {{ t('settings.imageProviderTags.sharedWithVideo') }} +
+
+
+
+
{{ t('settings.hints.minimaxImageInfo') }}
+
+
+
+
diff --git a/mateclaw-ui/src/views/Settings/Layout.vue b/mateclaw-ui/src/views/Settings/Layout.vue index aba79b06..03632bd0 100644 --- a/mateclaw-ui/src/views/Settings/Layout.vue +++ b/mateclaw-ui/src/views/Settings/Layout.vue @@ -53,6 +53,18 @@ const sections = computed(() => [ label: t('settings.sections.tts'), icon: '', }, + { + id: 'stt', + path: '/settings/stt', + label: t('settings.sections.stt'), + icon: '', + }, + { + id: 'music', + path: '/settings/music', + label: t('settings.sections.music'), + icon: '', + }, { id: 'video', path: '/settings/video', diff --git a/mateclaw-ui/src/views/Settings/Music/index.vue b/mateclaw-ui/src/views/Settings/Music/index.vue new file mode 100644 index 00000000..6e405010 --- /dev/null +++ b/mateclaw-ui/src/views/Settings/Music/index.vue @@ -0,0 +1,133 @@ + + + + + diff --git a/mateclaw-ui/src/views/Settings/Stt/index.vue b/mateclaw-ui/src/views/Settings/Stt/index.vue new file mode 100644 index 00000000..7032d677 --- /dev/null +++ b/mateclaw-ui/src/views/Settings/Stt/index.vue @@ -0,0 +1,133 @@ + + + + + diff --git a/mateclaw-ui/src/views/Settings/Video/index.vue b/mateclaw-ui/src/views/Settings/Video/index.vue index d34fd682..ea2dbe28 100644 --- a/mateclaw-ui/src/views/Settings/Video/index.vue +++ b/mateclaw-ui/src/views/Settings/Video/index.vue @@ -33,6 +33,8 @@ + +
@@ -175,6 +177,56 @@ + + +
+
+ Runway + gen4.5 / gen4_turbo +
+
+
+
+
{{ t('settings.fields.runwayApiKey') }}
+
{{ t('settings.hints.runwayApiKey') }}
+
+
+ +
+
+
+
+ + +
+
+ MiniMax (Hailuo) + {{ t('settings.videoProviderTags.freeQuota') }} +
+
+
+
+
{{ t('settings.fields.minimaxApiKey') }}
+
{{ t('settings.hints.minimaxApiKey') }}
+
+
+ +
+
+
+
@@ -199,6 +251,8 @@ const zhipuApiKeyInput = ref('') const falApiKeyInput = ref('') const klingAccessKeyInput = ref('') const klingSecretKeyInput = ref('') +const runwayApiKeyInput = ref('') +const minimaxApiKeyInput = ref('') const settings = reactive({ videoEnabled: false, @@ -209,6 +263,8 @@ const settings = reactive({ falApiKeyMasked: '', klingAccessKeyMasked: '', klingSecretKeyMasked: '', + runwayApiKeyMasked: '', + minimaxApiKeyMasked: '', }) onMounted(async () => { @@ -226,11 +282,15 @@ async function loadSettings() { settings.falApiKeyMasked = data.falApiKeyMasked ?? '' settings.klingAccessKeyMasked = data.klingAccessKeyMasked ?? '' settings.klingSecretKeyMasked = data.klingSecretKeyMasked ?? '' + settings.runwayApiKeyMasked = data.runwayApiKeyMasked ?? '' + settings.minimaxApiKeyMasked = data.minimaxApiKeyMasked ?? '' // 清空密钥输入 zhipuApiKeyInput.value = '' falApiKeyInput.value = '' klingAccessKeyInput.value = '' klingSecretKeyInput.value = '' + runwayApiKeyInput.value = '' + minimaxApiKeyInput.value = '' } async function onSaveSettings() { @@ -244,6 +304,8 @@ async function onSaveSettings() { if (falApiKeyInput.value) payload.falApiKey = falApiKeyInput.value if (klingAccessKeyInput.value) payload.klingAccessKey = klingAccessKeyInput.value if (klingSecretKeyInput.value) payload.klingSecretKey = klingSecretKeyInput.value + if (runwayApiKeyInput.value) payload.runwayApiKey = runwayApiKeyInput.value + if (minimaxApiKeyInput.value) payload.minimaxApiKey = minimaxApiKeyInput.value await settingsApi.update(payload) await loadSettings()