feat: add TTS, STT, music generation, video providers (Runway/MiniMax), image providers (Google Imagen/MiniMax), and tool registry fix

This commit is contained in:
matevip 2026-04-08 08:49:09 +08:00
parent 43acd2bf94
commit 7007ea844d
32 changed files with 2151 additions and 34 deletions

View File

@ -0,0 +1,34 @@
package vip.mate.stt;
import lombok.RequiredArgsConstructor;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.Map;
/**
* STT 语音识别 REST 端点
*/
@RestController
@RequestMapping("/api/v1/stt")
@RequiredArgsConstructor
public class SttController {
private final SttService sttService;
@PostMapping(value = "/transcribe", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<Map<String, Object>> transcribe(
@RequestPart("file") MultipartFile file,
@RequestParam(required = false) String language) throws Exception {
Map<String, Object> result = sttService.transcribe(
file.getBytes(),
file.getOriginalFilename(),
file.getContentType(),
language
);
return ResponseEntity.ok(result);
}
}

View File

@ -0,0 +1,19 @@
package vip.mate.stt;
import vip.mate.system.model.SystemSettingsDTO;
/**
* STT 语音识别提供商接口
*/
public interface SttProvider {
String id();
String label();
boolean requiresCredential();
int autoDetectOrder();
boolean isAvailable(SystemSettingsDTO config);
/**
* 转写音频
*/
SttResult transcribe(SttRequest request, SystemSettingsDTO config);
}

View File

@ -0,0 +1,51 @@
package vip.mate.stt;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.system.model.SystemSettingsDTO;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* STT 提供商注册表
*/
@Slf4j
@Component
public class SttProviderRegistry {
private final List<SttProvider> sortedProviders;
private final Map<String, SttProvider> providerMap;
public SttProviderRegistry(List<SttProvider> providers) {
this.sortedProviders = providers.stream()
.sorted(Comparator.comparingInt(SttProvider::autoDetectOrder))
.toList();
this.providerMap = providers.stream()
.collect(Collectors.toMap(SttProvider::id, Function.identity()));
log.info("注册 STT 提供商 {} 个: {}", sortedProviders.size(),
sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList());
}
public SttProvider resolve(SystemSettingsDTO config) {
String configuredId = config.getSttProvider();
if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) {
SttProvider p = providerMap.get(configuredId);
if (p != null && p.isAvailable(config)) return p;
}
for (SttProvider p : sortedProviders) {
if (p.isAvailable(config)) return p;
}
return null;
}
public List<SttProvider> fallbackCandidates(SystemSettingsDTO config, String excludeId) {
return sortedProviders.stream()
.filter(p -> !p.id().equals(excludeId))
.filter(p -> p.isAvailable(config))
.toList();
}
}

View File

@ -0,0 +1,22 @@
package vip.mate.stt;
import lombok.Builder;
import lombok.Data;
/**
* STT 语音识别统一请求
*/
@Data
@Builder
public class SttRequest {
/** 音频二进制数据 */
private byte[] audioData;
/** 原始文件名 */
private String fileName;
/** MIME 类型audio/ogg, audio/mpeg 等) */
private String contentType;
/** 目标语言(可选,如 "zh", "en" */
private String language;
/** 模型(可选) */
private String model;
}

View File

@ -0,0 +1,28 @@
package vip.mate.stt;
import lombok.Builder;
import lombok.Data;
/**
* STT 语音识别结果
*/
@Data
@Builder
public class SttResult {
private boolean success;
private String text;
private String language;
private String errorMessage;
public static SttResult success(String text) {
return SttResult.builder().success(true).text(text).build();
}
public static SttResult success(String text, String language) {
return SttResult.builder().success(true).text(text).language(language).build();
}
public static SttResult failure(String errorMessage) {
return SttResult.builder().success(false).errorMessage(errorMessage).build();
}
}

View File

@ -0,0 +1,70 @@
package vip.mate.stt;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.system.service.SystemSettingService;
import java.util.*;
/**
* STT 语音识别服务
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class SttService {
private final SystemSettingService systemSettingService;
private final SttProviderRegistry providerRegistry;
public Map<String, Object> transcribe(byte[] audioData, String fileName, String contentType, String language) {
SystemSettingsDTO config = systemSettingService.getAllSettings();
if (!Boolean.TRUE.equals(config.getSttEnabled())) {
return Map.of("success", false, "error", "STT 功能未启用,请在系统设置中开启");
}
SttRequest request = SttRequest.builder()
.audioData(audioData)
.fileName(fileName)
.contentType(contentType)
.language(language)
.build();
// Provider 选择 + fallback
SttResult result = transcribeWithFallback(request, config);
if (result.isSuccess()) {
return Map.of("success", true, "text", result.getText(),
"language", result.getLanguage() != null ? result.getLanguage() : "");
} else {
return Map.of("success", false, "error", result.getErrorMessage());
}
}
private SttResult transcribeWithFallback(SttRequest request, SystemSettingsDTO config) {
SttProvider primary = providerRegistry.resolve(config);
if (primary == null) {
return SttResult.failure("没有可用的 STT Provider请检查配置");
}
SttResult result = primary.transcribe(request, config);
if (result.isSuccess()) return result;
List<String> errors = new ArrayList<>();
errors.add(primary.id() + ": " + result.getErrorMessage());
if (Boolean.TRUE.equals(config.getSttFallbackEnabled())) {
for (SttProvider fb : providerRegistry.fallbackCandidates(config, primary.id())) {
log.info("[STT] Trying fallback provider: {}", fb.id());
result = fb.transcribe(request, config);
if (result.isSuccess()) return result;
errors.add(fb.id() + ": " + result.getErrorMessage());
}
}
return SttResult.failure("所有 STT Provider 均失败\n" + String.join("\n", errors));
}
}

View File

@ -0,0 +1,73 @@
package vip.mate.stt.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.llm.service.ModelProviderService;
import vip.mate.stt.SttProvider;
import vip.mate.stt.SttRequest;
import vip.mate.stt.SttResult;
import vip.mate.system.model.SystemSettingsDTO;
/**
* DashScope STT Provider ParaformerOpenAI 兼容接口
* <p>
* 复用模型管理中的 DashScope API Key中文识别效果优秀
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class DashScopeSttProvider implements SttProvider {
private final ModelProviderService modelProviderService;
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
private static final String DEFAULT_MODEL = "paraformer-v2";
@Override public String id() { return "dashscope"; }
@Override public String label() { return "DashScope (Paraformer)"; }
@Override public boolean requiresCredential() { return true; }
@Override public int autoDetectOrder() { return 150; }
@Override
public boolean isAvailable(SystemSettingsDTO config) {
try { return modelProviderService.isProviderConfigured("dashscope"); }
catch (Exception e) { return false; }
}
@Override
public SttResult transcribe(SttRequest request, SystemSettingsDTO config) {
try {
String apiKey = modelProviderService.getProviderConfig("dashscope").getApiKey();
if (apiKey == null) return SttResult.failure("DashScope API Key 未配置");
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
String fileName = request.getFileName() != null ? request.getFileName() : "audio.ogg";
HttpResponse response = HttpRequest.post(BASE_URL + "/audio/transcriptions")
.header("Authorization", "Bearer " + apiKey)
.form("model", model)
.form("file", request.getAudioData(), request.getContentType(), fileName)
.timeout(60_000)
.execute();
if (response.getStatus() == 200) {
JsonNode result = objectMapper.readTree(response.body());
String text = result.path("text").asText("");
log.info("[DashScope STT] Transcribed {} chars (model={})", text.length(), model);
return SttResult.success(text);
} else {
log.warn("[DashScope STT] Failed: HTTP {} - {}", response.getStatus(), response.body());
return SttResult.failure("DashScope STT 失败: HTTP " + response.getStatus());
}
} catch (Exception e) {
log.error("[DashScope STT] Error: {}", e.getMessage(), e);
return SttResult.failure("DashScope STT 异常: " + e.getMessage());
}
}
}

View File

@ -0,0 +1,74 @@
package vip.mate.stt.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.llm.service.ModelProviderService;
import vip.mate.stt.SttProvider;
import vip.mate.stt.SttRequest;
import vip.mate.stt.SttResult;
import vip.mate.system.model.SystemSettingsDTO;
/**
* OpenAI STT Provider Whisper / gpt-4o-mini-transcribe
* <p>
* 复用模型管理中的 OpenAI API Key
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class OpenAiSttProvider implements SttProvider {
private final ModelProviderService modelProviderService;
private final ObjectMapper objectMapper;
private static final String DEFAULT_MODEL = "whisper-1";
@Override public String id() { return "openai"; }
@Override public String label() { return "OpenAI Whisper"; }
@Override public boolean requiresCredential() { return true; }
@Override public int autoDetectOrder() { return 100; }
@Override
public boolean isAvailable(SystemSettingsDTO config) {
try { return modelProviderService.isProviderConfigured("openai"); }
catch (Exception e) { return false; }
}
@Override
public SttResult transcribe(SttRequest request, SystemSettingsDTO config) {
try {
String apiKey = modelProviderService.getProviderConfig("openai").getApiKey();
String baseUrl = modelProviderService.getProviderConfig("openai").getBaseUrl();
if (apiKey == null) return SttResult.failure("OpenAI API Key 未配置");
String url = (baseUrl != null ? baseUrl : "https://api.openai.com") + "/v1/audio/transcriptions";
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
String fileName = request.getFileName() != null ? request.getFileName() : "audio.ogg";
HttpResponse response = HttpRequest.post(url)
.header("Authorization", "Bearer " + apiKey)
.form("model", model)
.form("file", request.getAudioData(), request.getContentType(), fileName)
.timeout(60_000)
.execute();
if (response.getStatus() == 200) {
JsonNode result = objectMapper.readTree(response.body());
String text = result.path("text").asText("");
log.info("[OpenAI STT] Transcribed {} chars (model={})", text.length(), model);
return SttResult.success(text);
} else {
log.warn("[OpenAI STT] Failed: HTTP {} - {}", response.getStatus(), response.body());
return SttResult.failure("OpenAI STT 失败: HTTP " + response.getStatus());
}
} catch (Exception e) {
log.error("[OpenAI STT] Error: {}", e.getMessage(), e);
return SttResult.failure("OpenAI STT 异常: " + e.getMessage());
}
}
}

View File

@ -61,6 +61,16 @@ public class SystemSettingsDTO {
private String klingAccessKeyMasked;
private String klingSecretKeyMasked;
// --- Runway ---
@JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
private String runwayApiKey;
private String runwayApiKeyMasked;
// --- MiniMax (Hailuo) ---
@JsonProperty(access = JsonProperty.Access.WRITE_ONLY)
private String minimaxApiKey;
private String minimaxApiKeyMasked;
// ===== 图片生成配置 =====
/** 是否启用图片生成能力 */
private Boolean imageEnabled;
@ -82,4 +92,16 @@ public class SystemSettingsDTO {
private String ttsDefaultVoice;
/** 默认语速 0.5-2.0 */
private Double ttsSpeed;
// ===== STT 语音识别配置 =====
private Boolean sttEnabled;
/** 首选 STT provider: auto / openai / dashscope */
private String sttProvider;
private Boolean sttFallbackEnabled;
// ===== 音乐生成配置 =====
private Boolean musicEnabled;
/** 首选音乐 provider: auto / google-lyria / minimax */
private String musicProvider;
private Boolean musicFallbackEnabled;
}

View File

@ -44,11 +44,23 @@ public class SystemSettingService {
private static final String TTS_AUTO_MODE_KEY = "ttsAutoMode";
private static final String TTS_DEFAULT_VOICE_KEY = "ttsDefaultVoice";
private static final String TTS_SPEED_KEY = "ttsSpeed";
// STT 配置 keys
private static final String STT_ENABLED_KEY = "sttEnabled";
private static final String STT_PROVIDER_KEY = "sttProvider";
private static final String STT_FALLBACK_ENABLED_KEY = "sttFallbackEnabled";
// 音乐生成配置 keys
private static final String MUSIC_ENABLED_KEY = "musicEnabled";
private static final String MUSIC_PROVIDER_KEY = "musicProvider";
private static final String MUSIC_FALLBACK_ENABLED_KEY = "musicFallbackEnabled";
private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey";
private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl";
private static final String FAL_API_KEY_KEY = "falApiKey";
private static final String KLING_ACCESS_KEY_KEY = "klingAccessKey";
private static final String KLING_SECRET_KEY_KEY = "klingSecretKey";
private static final String RUNWAY_API_KEY_KEY = "runwayApiKey";
private static final String MINIMAX_API_KEY_KEY = "minimaxApiKey";
private final SystemSettingMapper systemSettingMapper;
@ -81,6 +93,8 @@ public class SystemSettingService {
dto.setFalApiKeyMasked(maskApiKey(getValue(FAL_API_KEY_KEY, "")));
dto.setKlingAccessKeyMasked(maskApiKey(getValue(KLING_ACCESS_KEY_KEY, "")));
dto.setKlingSecretKeyMasked(maskApiKey(getValue(KLING_SECRET_KEY_KEY, "")));
dto.setRunwayApiKeyMasked(maskApiKey(getValue(RUNWAY_API_KEY_KEY, "")));
dto.setMinimaxApiKeyMasked(maskApiKey(getValue(MINIMAX_API_KEY_KEY, "")));
// 图片生成配置
dto.setImageEnabled(Boolean.parseBoolean(getValue(IMAGE_ENABLED_KEY, "false")));
@ -95,6 +109,16 @@ public class SystemSettingService {
dto.setTtsDefaultVoice(getValue(TTS_DEFAULT_VOICE_KEY, ""));
String speedStr = getValue(TTS_SPEED_KEY, "1.0");
try { dto.setTtsSpeed(Double.parseDouble(speedStr)); } catch (NumberFormatException e) { dto.setTtsSpeed(1.0); }
// STT 配置
dto.setSttEnabled(Boolean.parseBoolean(getValue(STT_ENABLED_KEY, "false")));
dto.setSttProvider(getValue(STT_PROVIDER_KEY, "auto"));
dto.setSttFallbackEnabled(Boolean.parseBoolean(getValue(STT_FALLBACK_ENABLED_KEY, "true")));
// 音乐生成配置
dto.setMusicEnabled(Boolean.parseBoolean(getValue(MUSIC_ENABLED_KEY, "false")));
dto.setMusicProvider(getValue(MUSIC_PROVIDER_KEY, "auto"));
dto.setMusicFallbackEnabled(Boolean.parseBoolean(getValue(MUSIC_FALLBACK_ENABLED_KEY, "true")));
return dto;
}
@ -111,6 +135,8 @@ public class SystemSettingService {
dto.setFalApiKey(getValue(FAL_API_KEY_KEY, ""));
dto.setKlingAccessKey(getValue(KLING_ACCESS_KEY_KEY, ""));
dto.setKlingSecretKey(getValue(KLING_SECRET_KEY_KEY, ""));
dto.setRunwayApiKey(getValue(RUNWAY_API_KEY_KEY, ""));
dto.setMinimaxApiKey(getValue(MINIMAX_API_KEY_KEY, ""));
return dto;
}
@ -193,6 +219,12 @@ public class SystemSettingService {
if (dto.getKlingSecretKey() != null && !dto.getKlingSecretKey().isBlank()) {
saveValue(KLING_SECRET_KEY_KEY, dto.getKlingSecretKey(), "快手可灵 Secret Key");
}
if (dto.getRunwayApiKey() != null && !dto.getRunwayApiKey().isBlank()) {
saveValue(RUNWAY_API_KEY_KEY, dto.getRunwayApiKey(), "Runway API Key");
}
if (dto.getMinimaxApiKey() != null && !dto.getMinimaxApiKey().isBlank()) {
saveValue(MINIMAX_API_KEY_KEY, dto.getMinimaxApiKey(), "MiniMax API Key");
}
// 图片生成配置
if (dto.getImageEnabled() != null) {
@ -224,6 +256,28 @@ public class SystemSettingService {
if (dto.getTtsSpeed() != null) {
saveValue(TTS_SPEED_KEY, String.valueOf(dto.getTtsSpeed()), "TTS 默认语速");
}
// STT 配置
if (dto.getSttEnabled() != null) {
saveValue(STT_ENABLED_KEY, String.valueOf(dto.getSttEnabled()), "是否启用 STT 语音识别");
}
if (dto.getSttProvider() != null) {
saveValue(STT_PROVIDER_KEY, dto.getSttProvider(), "STT 首选 Provider");
}
if (dto.getSttFallbackEnabled() != null) {
saveValue(STT_FALLBACK_ENABLED_KEY, String.valueOf(dto.getSttFallbackEnabled()), "STT Provider 级 Fallback");
}
// 音乐生成配置
if (dto.getMusicEnabled() != null) {
saveValue(MUSIC_ENABLED_KEY, String.valueOf(dto.getMusicEnabled()), "是否启用音乐生成");
}
if (dto.getMusicProvider() != null) {
saveValue(MUSIC_PROVIDER_KEY, dto.getMusicProvider(), "音乐生成首选 Provider");
}
if (dto.getMusicFallbackEnabled() != null) {
saveValue(MUSIC_FALLBACK_ENABLED_KEY, String.valueOf(dto.getMusicFallbackEnabled()), "音乐 Provider 级 Fallback");
}
return getSettings();
}

View File

@ -38,10 +38,12 @@ public class ToolRegistry {
* 通过数据库 enabled 标志过滤确保 UI 开关真正生效
*/
public List<Object> getEnabledTools() {
// 1. 从数据库获取已启用的 beanName 集合
Set<String> enabledBeanNames = toolMapper.selectList(
// 1. 从数据库获取明确禁用的 beanName 黑名单
// 逻辑只有 DB 中存在记录且 enabled=false 的才跳过
// DB 中没有记录的 bean 默认启用向后兼容 + 新工具自动可用
Set<String> disabledBeanNames = toolMapper.selectList(
new LambdaQueryWrapper<ToolEntity>()
.eq(ToolEntity::getEnabled, true)
.eq(ToolEntity::getEnabled, false)
.isNotNull(ToolEntity::getBeanName)
).stream()
.map(ToolEntity::getBeanName)
@ -59,13 +61,12 @@ public class ToolRegistry {
.anyMatch(m -> m.isAnnotationPresent(Tool.class));
if (hasToolMethod) {
// 3. 如果 DB 中有该 beanName 的记录则按 DB enabled 状态决定是否加入
// 如果 DB 中没有记录未注册默认加入保持向后兼容
if (enabledBeanNames.isEmpty() || enabledBeanNames.contains(beanName)) {
// 3. 只有 DB 中明确 enabled=false 的才跳过其余全部启用
if (disabledBeanNames.contains(beanName)) {
log.debug("Skipped disabled tool bean: {} (beanName={})", bean.getClass().getSimpleName(), beanName);
} else {
tools.add(bean);
log.debug("Registered tool bean: {} (beanName={})", bean.getClass().getSimpleName(), beanName);
} else {
log.debug("Skipped disabled tool bean: {} (beanName={})", bean.getClass().getSimpleName(), beanName);
}
}
}

View File

@ -0,0 +1,187 @@
package vip.mate.tool.image.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.llm.service.ModelProviderService;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.tool.image.*;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Set;
/**
* Google Imagen 图片生成 Provider 使用 Gemini API 的图片生成能力
* <p>
* 同步模式直接返回 Base64 图片数据
* 复用已有的 Google/Gemini LLM provider API Key
* <p>
* API: POST https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent
*
* @author MateClaw Team
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class GoogleImagenProvider implements ImageGenerationProvider {
private final ModelProviderService modelProviderService;
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://generativelanguage.googleapis.com";
private static final String DEFAULT_MODEL = "gemini-2.0-flash-preview-image-generation";
@Override
public String id() {
return "google-imagen";
}
@Override
public String label() {
return "Google Imagen";
}
@Override
public boolean requiresCredential() {
return true;
}
@Override
public int autoDetectOrder() {
return 250;
}
@Override
public Set<ImageCapability> capabilities() {
return Set.of(ImageCapability.TEXT_TO_IMAGE);
}
@Override
public ImageProviderCapabilities detailedCapabilities() {
return ImageProviderCapabilities.builder()
.modes(capabilities())
.supportedSizes(List.of("1024x1024", "1024x1536", "1536x1024", "1024x1792", "1792x1024"))
.aspectRatios(List.of("1:1", "3:4", "4:3", "9:16", "16:9"))
.maxCount(4)
.defaultModel(DEFAULT_MODEL)
.models(List.of("gemini-2.0-flash-preview-image-generation", "imagen-4.0-generate-preview", "imagen-4.0-ultra-generate-preview"))
.build();
}
@Override
public boolean isAvailable(SystemSettingsDTO config) {
try {
return modelProviderService.isProviderConfigured("google");
} catch (Exception e) {
return false;
}
}
@Override
public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) {
try {
String apiKey = getApiKey();
if (apiKey == null) {
return ImageSubmitResult.failure(id(), "Google API Key 未配置");
}
String model = request.getModel() != null && !request.getModel().isBlank()
? request.getModel() : DEFAULT_MODEL;
// 构建请求体
ObjectNode body = objectMapper.createObjectNode();
// contents
ArrayNode contents = body.putArray("contents");
ObjectNode content = contents.addObject();
content.put("role", "user");
ArrayNode parts = content.putArray("parts");
parts.addObject().put("text", request.getPrompt());
// generationConfig
ObjectNode genConfig = body.putObject("generationConfig");
ArrayNode modalities = genConfig.putArray("responseModalities");
modalities.add("TEXT");
modalities.add("IMAGE");
if (request.getAspectRatio() != null) {
ObjectNode imageConfig = genConfig.putObject("imageConfig");
imageConfig.put("aspectRatio", request.getAspectRatio());
}
String url = BASE_URL + "/v1beta/models/" + model + ":generateContent?key=" + apiKey;
HttpResponse response = HttpRequest.post(url)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(60_000)
.execute();
if (response.getStatus() != 200) {
String errBody = response.body();
log.warn("[Google Imagen] Failed: HTTP {} - {}", response.getStatus(), errBody);
return ImageSubmitResult.failure(id(), "Google Imagen 失败: HTTP " + response.getStatus());
}
JsonNode result = objectMapper.readTree(response.body());
List<String> imageUrls = extractImagesFromResponse(result);
if (imageUrls.isEmpty()) {
return ImageSubmitResult.failure(id(), "Google Imagen 未返回图片");
}
log.info("[Google Imagen] Generated {} images (model={})", imageUrls.size(), model);
return ImageSubmitResult.syncSuccess(id(), imageUrls);
} catch (Exception e) {
log.error("[Google Imagen] Error: {}", e.getMessage(), e);
return ImageSubmitResult.failure(id(), "Google Imagen 异常: " + e.getMessage());
}
}
/**
* Gemini 响应中提取 Base64 图片转换为 data URI
*/
private List<String> extractImagesFromResponse(JsonNode result) {
List<String> images = new ArrayList<>();
JsonNode candidates = result.path("candidates");
if (candidates.isArray()) {
for (JsonNode candidate : candidates) {
JsonNode parts = candidate.path("content").path("parts");
if (parts.isArray()) {
for (JsonNode part : parts) {
// 尝试 inlineData inline_data
JsonNode inlineData = part.has("inlineData") ? part.get("inlineData")
: part.path("inline_data");
if (inlineData.has("data")) {
String mimeType = inlineData.has("mimeType")
? inlineData.get("mimeType").asText("image/png")
: inlineData.path("mime_type").asText("image/png");
String base64Data = inlineData.get("data").asText();
// 返回 data URI 格式
images.add("data:" + mimeType + ";base64," + base64Data);
}
}
}
}
}
return images;
}
private String getApiKey() {
try {
return modelProviderService.getProviderConfig("google").getApiKey();
} catch (Exception e) {
return null;
}
}
}

View File

@ -0,0 +1,151 @@
package vip.mate.tool.image.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.tool.image.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* MiniMax 图片生成 Provider image-01 模型
* <p>
* 同步模式返回 Base64 图片
* 复用视频生成中的 MiniMax API Key
* <p>
* API: POST https://api.minimax.io/v1/image_generation
*
* @author MateClaw Team
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class MiniMaxImageProvider implements ImageGenerationProvider {
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://api.minimax.io";
private static final String DEFAULT_MODEL = "image-01";
@Override
public String id() {
return "minimax";
}
@Override
public String label() {
return "MiniMax Image";
}
@Override
public boolean requiresCredential() {
return true;
}
@Override
public int autoDetectOrder() {
return 350;
}
@Override
public Set<ImageCapability> capabilities() {
return Set.of(ImageCapability.TEXT_TO_IMAGE);
}
@Override
public ImageProviderCapabilities detailedCapabilities() {
return ImageProviderCapabilities.builder()
.modes(capabilities())
.supportedSizes(List.of("1024x1024"))
.aspectRatios(List.of("1:1", "16:9", "4:3", "3:2", "2:3", "3:4", "9:16"))
.maxCount(9)
.defaultModel(DEFAULT_MODEL)
.models(List.of("image-01"))
.build();
}
@Override
public boolean isAvailable(SystemSettingsDTO config) {
// 复用视频生成的 MiniMax API Key
return StringUtils.hasText(config.getMinimaxApiKey());
}
@Override
public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) {
try {
String apiKey = config.getMinimaxApiKey();
String model = request.getModel() != null && !request.getModel().isBlank()
? request.getModel() : DEFAULT_MODEL;
ObjectNode body = objectMapper.createObjectNode();
body.put("model", model);
body.put("prompt", request.getPrompt());
body.put("response_format", "url");
body.put("n", request.getCount() != null ? request.getCount() : 1);
if (request.getAspectRatio() != null) {
body.put("aspect_ratio", request.getAspectRatio());
}
HttpResponse response = HttpRequest.post(BASE_URL + "/v1/image_generation")
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(60_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
int statusCode = result.path("base_resp").path("status_code").asInt(-1);
if (statusCode != 0) {
String errMsg = result.path("base_resp").path("status_msg").asText("未知错误");
log.warn("[MiniMax Image] Failed: {}", errMsg);
return ImageSubmitResult.failure(id(), errMsg);
}
List<String> imageUrls = new ArrayList<>();
// 尝试从 data.image_base64 提取
JsonNode imageBase64Array = result.path("data").path("image_base64");
if (imageBase64Array.isArray()) {
for (JsonNode b64 : imageBase64Array) {
String base64Str = b64.asText();
if (StringUtils.hasText(base64Str)) {
imageUrls.add("data:image/png;base64," + base64Str);
}
}
}
// 尝试从 data.image_urls 提取response_format=url
JsonNode imageUrlsNode = result.path("data").path("image_urls");
if (imageUrlsNode.isArray()) {
for (JsonNode urlNode : imageUrlsNode) {
String url = urlNode.asText(null);
if (StringUtils.hasText(url)) {
imageUrls.add(url);
}
}
}
if (imageUrls.isEmpty()) {
return ImageSubmitResult.failure(id(), "MiniMax 未返回图片数据");
}
log.info("[MiniMax Image] Generated {} images (model={})", imageUrls.size(), model);
return ImageSubmitResult.syncSuccess(id(), imageUrls);
} catch (Exception e) {
log.error("[MiniMax Image] Error: {}", e.getMessage(), e);
return ImageSubmitResult.failure(id(), "MiniMax Image 异常: " + e.getMessage());
}
}
}

View File

@ -0,0 +1,52 @@
package vip.mate.tool.music;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.tool.annotation.Tool;
import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.stereotype.Component;
import vip.mate.tool.builtin.ToolExecutionContext;
import java.util.Map;
/**
* 音乐生成工具 Agent 可通过 @Tool 注解调用
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class MusicGenerateTool {
private final MusicGenerationService musicGenerationService;
@Tool(description = "生成音乐或歌曲。支持文字描述生成音乐、歌词谱曲、纯音乐等模式。支持 Google Lyria 和 MiniMax Music 等 Provider。")
public String music_generate(
@ToolParam(description = "音乐风格/场景描述,如:'轻快的钢琴爵士乐'、'史诗电影配乐'、'欢快的流行歌曲'") String prompt,
@ToolParam(description = "歌词文本(可选,不填则由 AI 生成或生成纯音乐)") String lyrics,
@ToolParam(description = "是否生成纯音乐(无人声),默认 false") Boolean instrumental) {
String conversationId = ToolExecutionContext.conversationId();
if (conversationId == null) {
return "无法获取会话 ID";
}
MusicGenerationRequest request = MusicGenerationRequest.builder()
.prompt(prompt)
.lyrics(lyrics)
.instrumental(instrumental != null ? instrumental : false)
.build();
Map<String, Object> result = musicGenerationService.generate(conversationId, request);
if (Boolean.TRUE.equals(result.get("success"))) {
StringBuilder sb = new StringBuilder("音乐生成完成!\n");
sb.append("播放链接: ").append(result.get("audioUrl"));
if (result.containsKey("lyrics") && result.get("lyrics") != null) {
sb.append("\n\n歌词:\n").append(result.get("lyrics"));
}
return sb.toString();
} else {
return "音乐生成失败: " + result.get("error");
}
}
}

View File

@ -0,0 +1,23 @@
package vip.mate.tool.music;
import vip.mate.system.model.SystemSettingsDTO;
import java.util.List;
/**
* 音乐生成提供商接口
*/
public interface MusicGenerationProvider {
String id();
String label();
boolean requiresCredential();
int autoDetectOrder();
boolean isAvailable(SystemSettingsDTO config);
List<String> availableModels();
String defaultModel();
/**
* 生成音乐同步返回音频字节
*/
MusicGenerationResult generate(MusicGenerationRequest request, SystemSettingsDTO config);
}

View File

@ -0,0 +1,17 @@
package vip.mate.tool.music;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class MusicGenerationRequest {
private String prompt;
private String lyrics;
@Builder.Default
private Boolean instrumental = false;
private Integer durationSeconds;
private String model;
@Builder.Default
private String format = "mp3";
}

View File

@ -0,0 +1,27 @@
package vip.mate.tool.music;
import lombok.Builder;
import lombok.Data;
@Data
@Builder
public class MusicGenerationResult {
private boolean success;
private byte[] audioData;
private String contentType;
private String format;
private String lyrics;
private String errorMessage;
public static MusicGenerationResult success(byte[] audioData, String contentType, String format) {
return MusicGenerationResult.builder().success(true).audioData(audioData).contentType(contentType).format(format).build();
}
public static MusicGenerationResult successWithLyrics(byte[] audioData, String contentType, String format, String lyrics) {
return MusicGenerationResult.builder().success(true).audioData(audioData).contentType(contentType).format(format).lyrics(lyrics).build();
}
public static MusicGenerationResult failure(String errorMessage) {
return MusicGenerationResult.builder().success(false).errorMessage(errorMessage).build();
}
}

View File

@ -0,0 +1,85 @@
package vip.mate.tool.music;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.system.service.SystemSettingService;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
/**
* 音乐生成服务
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class MusicGenerationService {
private final SystemSettingService systemSettingService;
private final MusicProviderRegistry providerRegistry;
private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads");
public Map<String, Object> generate(String conversationId, MusicGenerationRequest request) {
SystemSettingsDTO config = systemSettingService.getAllSettings();
if (!Boolean.TRUE.equals(config.getMusicEnabled())) {
return Map.of("success", false, "error", "音乐生成功能未启用");
}
MusicGenerationResult result = generateWithFallback(request, config);
if (!result.isSuccess()) {
return Map.of("success", false, "error", result.getErrorMessage());
}
try {
String fileId = UUID.randomUUID().toString().replace("-", "").substring(0, 12);
Path dir = UPLOAD_ROOT.resolve(conversationId);
Files.createDirectories(dir);
String fileName = "music_" + fileId + "." + result.getFormat();
Path filePath = dir.resolve(fileName);
Files.write(filePath, result.getAudioData());
String audioUrl = "/api/v1/chat/files/" + conversationId + "/" + fileName;
Map<String, Object> response = new LinkedHashMap<>();
response.put("success", true);
response.put("audioUrl", audioUrl);
response.put("contentType", result.getContentType());
response.put("format", result.getFormat());
if (result.getLyrics() != null) {
response.put("lyrics", result.getLyrics());
}
return response;
} catch (IOException e) {
log.error("[Music] Failed to save audio: {}", e.getMessage());
return Map.of("success", false, "error", "音频文件保存失败");
}
}
private MusicGenerationResult generateWithFallback(MusicGenerationRequest request, SystemSettingsDTO config) {
MusicGenerationProvider primary = providerRegistry.resolve(config);
if (primary == null) return MusicGenerationResult.failure("没有可用的音乐 Provider");
MusicGenerationResult result = primary.generate(request, config);
if (result.isSuccess()) return result;
List<String> errors = new ArrayList<>();
errors.add(primary.id() + ": " + result.getErrorMessage());
if (Boolean.TRUE.equals(config.getMusicFallbackEnabled())) {
for (MusicGenerationProvider fb : providerRegistry.fallbackCandidates(config, primary.id())) {
result = fb.generate(request, config);
if (result.isSuccess()) return result;
errors.add(fb.id() + ": " + result.getErrorMessage());
}
}
return MusicGenerationResult.failure("所有音乐 Provider 均失败\n" + String.join("\n", errors));
}
}

View File

@ -0,0 +1,45 @@
package vip.mate.tool.music;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.system.model.SystemSettingsDTO;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@Component
public class MusicProviderRegistry {
private final List<MusicGenerationProvider> sortedProviders;
private final Map<String, MusicGenerationProvider> providerMap;
public MusicProviderRegistry(List<MusicGenerationProvider> providers) {
this.sortedProviders = providers.stream()
.sorted(Comparator.comparingInt(MusicGenerationProvider::autoDetectOrder))
.toList();
this.providerMap = providers.stream()
.collect(Collectors.toMap(MusicGenerationProvider::id, Function.identity()));
log.info("注册音乐生成 Provider {} 个: {}", sortedProviders.size(),
sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList());
}
public MusicGenerationProvider resolve(SystemSettingsDTO config) {
String configuredId = config.getMusicProvider();
if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) {
MusicGenerationProvider p = providerMap.get(configuredId);
if (p != null && p.isAvailable(config)) return p;
}
for (MusicGenerationProvider p : sortedProviders) {
if (p.isAvailable(config)) return p;
}
return null;
}
public List<MusicGenerationProvider> fallbackCandidates(SystemSettingsDTO config, String excludeId) {
return sortedProviders.stream().filter(p -> !p.id().equals(excludeId)).filter(p -> p.isAvailable(config)).toList();
}
}

View File

@ -0,0 +1,133 @@
package vip.mate.tool.music.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.llm.service.ModelProviderService;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.tool.music.MusicGenerationProvider;
import vip.mate.tool.music.MusicGenerationRequest;
import vip.mate.tool.music.MusicGenerationResult;
import java.util.Base64;
import java.util.List;
/**
* Google Lyria 音乐生成 Provider
* <p>
* 使用 Gemini API 的音频生成能力复用 Google LLM API Key
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class GoogleLyriaProvider implements MusicGenerationProvider {
private final ModelProviderService modelProviderService;
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://generativelanguage.googleapis.com";
private static final String DEFAULT_MODEL = "lyria-3-clip-preview";
@Override public String id() { return "google-lyria"; }
@Override public String label() { return "Google Lyria"; }
@Override public boolean requiresCredential() { return true; }
@Override public int autoDetectOrder() { return 100; }
@Override public String defaultModel() { return DEFAULT_MODEL; }
@Override public List<String> availableModels() { return List.of("lyria-3-clip-preview"); }
@Override
public boolean isAvailable(SystemSettingsDTO config) {
try { return modelProviderService.isProviderConfigured("google"); }
catch (Exception e) { return false; }
}
@Override
public MusicGenerationResult generate(MusicGenerationRequest request, SystemSettingsDTO config) {
try {
String apiKey = modelProviderService.getProviderConfig("google").getApiKey();
if (apiKey == null) return MusicGenerationResult.failure("Google API Key 未配置");
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
// 构建 prompt
StringBuilder prompt = new StringBuilder(request.getPrompt());
if (Boolean.TRUE.equals(request.getInstrumental())) {
prompt.append("\nInstrumental only, no vocals.");
}
if (request.getLyrics() != null && !request.getLyrics().isBlank()) {
prompt.append("\nLyrics:\n").append(request.getLyrics());
}
ObjectNode body = objectMapper.createObjectNode();
ArrayNode contents = body.putArray("contents");
ObjectNode content = contents.addObject();
ArrayNode parts = content.putArray("parts");
parts.addObject().put("text", prompt.toString());
ObjectNode genConfig = body.putObject("generationConfig");
ArrayNode modalities = genConfig.putArray("responseModalities");
modalities.add("AUDIO");
modalities.add("TEXT");
String url = BASE_URL + "/v1beta/models/" + model + ":generateContent?key=" + apiKey;
HttpResponse response = HttpRequest.post(url)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(120_000) // 音乐生成较慢
.execute();
if (response.getStatus() != 200) {
log.warn("[Google Lyria] Failed: HTTP {} - {}", response.getStatus(), response.body());
return MusicGenerationResult.failure("Google Lyria 失败: HTTP " + response.getStatus());
}
JsonNode result = objectMapper.readTree(response.body());
return extractAudioFromResponse(result);
} catch (Exception e) {
log.error("[Google Lyria] Error: {}", e.getMessage(), e);
return MusicGenerationResult.failure("Google Lyria 异常: " + e.getMessage());
}
}
private MusicGenerationResult extractAudioFromResponse(JsonNode result) {
String lyrics = null;
byte[] audioData = null;
String mimeType = "audio/mpeg";
JsonNode candidates = result.path("candidates");
if (candidates.isArray()) {
for (JsonNode candidate : candidates) {
JsonNode parts = candidate.path("content").path("parts");
if (parts.isArray()) {
for (JsonNode part : parts) {
if (part.has("text")) {
lyrics = part.get("text").asText();
}
JsonNode inlineData = part.has("inlineData") ? part.get("inlineData") : part.path("inline_data");
if (inlineData.has("data")) {
mimeType = inlineData.has("mimeType") ? inlineData.get("mimeType").asText("audio/mpeg")
: inlineData.path("mime_type").asText("audio/mpeg");
audioData = Base64.getDecoder().decode(inlineData.get("data").asText());
}
}
}
}
}
if (audioData == null || audioData.length == 0) {
return MusicGenerationResult.failure("Google Lyria 未返回音频数据");
}
String format = mimeType.contains("wav") ? "wav" : "mp3";
log.info("[Google Lyria] Generated {} bytes audio", audioData.length);
return MusicGenerationResult.successWithLyrics(audioData, mimeType, format, lyrics);
}
}

View File

@ -0,0 +1,140 @@
package vip.mate.tool.music.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.tool.music.MusicGenerationProvider;
import vip.mate.tool.music.MusicGenerationRequest;
import vip.mate.tool.music.MusicGenerationResult;
import java.util.List;
/**
* MiniMax 音乐生成 Provider music-2.5+
* <p>
* 复用视频生成中的 MiniMax API Key
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class MiniMaxMusicProvider implements MusicGenerationProvider {
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://api.minimax.io";
private static final String DEFAULT_MODEL = "music-2.5+";
@Override public String id() { return "minimax"; }
@Override public String label() { return "MiniMax Music"; }
@Override public boolean requiresCredential() { return true; }
@Override public int autoDetectOrder() { return 200; }
@Override public String defaultModel() { return DEFAULT_MODEL; }
@Override public List<String> availableModels() { return List.of("music-2.5+", "music-2.5"); }
@Override
public boolean isAvailable(SystemSettingsDTO config) {
return StringUtils.hasText(config.getMinimaxApiKey());
}
@Override
public MusicGenerationResult generate(MusicGenerationRequest request, SystemSettingsDTO config) {
try {
String apiKey = config.getMinimaxApiKey();
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
ObjectNode body = objectMapper.createObjectNode();
body.put("model", model);
// 构建 prompt可加时长提示
String prompt = request.getPrompt();
if (request.getDurationSeconds() != null) {
prompt += " Duration: " + request.getDurationSeconds() + " seconds.";
}
body.put("prompt", prompt);
if (Boolean.TRUE.equals(request.getInstrumental())) {
body.put("is_instrumental", true);
}
if (request.getLyrics() != null && !request.getLyrics().isBlank()) {
body.put("lyrics", request.getLyrics());
} else if (!Boolean.TRUE.equals(request.getInstrumental())) {
body.put("lyrics_optimizer", true);
}
body.put("output_format", "url");
ObjectNode audioSetting = body.putObject("audio_setting");
audioSetting.put("sample_rate", 44100);
audioSetting.put("bitrate", 256000);
audioSetting.put("format", "mp3");
HttpResponse response = HttpRequest.post(BASE_URL + "/v1/music_generation")
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(120_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
int statusCode = result.path("base_resp").path("status_code").asInt(-1);
if (statusCode != 0) {
String errMsg = result.path("base_resp").path("status_msg").asText("未知错误");
log.warn("[MiniMax Music] Failed: {}", errMsg);
return MusicGenerationResult.failure(errMsg);
}
// 提取音频 URL 或内联数据
String audioUrl = result.has("audio_url") ? result.get("audio_url").asText(null)
: result.path("data").path("audio_url").asText(null);
String lyrics = result.has("lyrics") ? result.get("lyrics").asText(null)
: result.path("data").path("lyrics").asText(null);
if (StringUtils.hasText(audioUrl)) {
// 下载音频
byte[] audioData = HttpRequest.get(audioUrl).timeout(30_000).execute().bodyBytes();
log.info("[MiniMax Music] Generated {} bytes audio (model={})", audioData.length, model);
return MusicGenerationResult.successWithLyrics(audioData, "audio/mpeg", "mp3", lyrics);
}
// 尝试 inline audio
String inlineAudio = result.has("audio") ? result.get("audio").asText(null)
: result.path("data").path("audio").asText(null);
if (StringUtils.hasText(inlineAudio)) {
byte[] audioData = decodeAudio(inlineAudio);
return MusicGenerationResult.successWithLyrics(audioData, "audio/mpeg", "mp3", lyrics);
}
return MusicGenerationResult.failure("MiniMax 未返回音频数据");
} catch (Exception e) {
log.error("[MiniMax Music] Error: {}", e.getMessage(), e);
return MusicGenerationResult.failure("MiniMax Music 异常: " + e.getMessage());
}
}
private byte[] decodeAudio(String data) {
// MiniMax 可能返回 hex base64
if (data.matches("^[0-9a-fA-F]+$") && data.length() % 2 == 0) {
return hexToBytes(data);
}
return java.util.Base64.getDecoder().decode(data);
}
private byte[] hexToBytes(String hex) {
int len = hex.length();
byte[] data = new byte[len / 2];
for (int i = 0; i < len; i += 2) {
data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
+ Character.digit(hex.charAt(i + 1), 16));
}
return data;
}
}

View File

@ -0,0 +1,176 @@
package vip.mate.tool.video.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.task.AsyncTaskService.TaskPollResult;
import vip.mate.tool.video.*;
import java.util.List;
import java.util.Set;
/**
* MiniMax (Hailuo / 海螺) 视频生成 Provider
* <p>
* API 文档: https://platform.minimaxi.com/document/video-generation
* 鉴权: Bearer Token
*
* @author MateClaw Team
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class MiniMaxVideoProvider implements VideoGenerationProvider {
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://api.minimax.io";
private static final String DEFAULT_MODEL = "MiniMax-Hailuo-2.3";
@Override
public String id() {
return "minimax";
}
@Override
public String label() {
return "MiniMax (Hailuo)";
}
@Override
public boolean requiresCredential() {
return true;
}
@Override
public int autoDetectOrder() {
return 350;
}
@Override
public Set<VideoCapability> capabilities() {
return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO);
}
@Override
public VideoProviderCapabilities detailedCapabilities() {
return VideoProviderCapabilities.builder()
.modes(capabilities())
.aspectRatios(List.of("16:9", "9:16", "1:1"))
.supportedDurations(List.of(6, 10))
.maxDurationSeconds(10)
.defaultModel(DEFAULT_MODEL)
.models(List.of("MiniMax-Hailuo-2.3", "MiniMax-Hailuo-2.3-Fast", "I2V-01-live"))
.build();
}
@Override
public boolean isAvailable(SystemSettingsDTO config) {
return StringUtils.hasText(config.getMinimaxApiKey());
}
@Override
public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) {
try {
String apiKey = config.getMinimaxApiKey();
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
ObjectNode body = objectMapper.createObjectNode();
body.put("model", model);
body.put("prompt", request.getPrompt());
if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) {
body.put("first_frame_image", request.getImageUrl());
}
if (request.getDurationSeconds() != null) {
body.put("duration", request.getDurationSeconds());
}
HttpResponse response = HttpRequest.post(BASE_URL + "/v1/video_generation")
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(30_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
// MiniMax 返回 { task_id, base_resp: { status_code, status_msg } }
int statusCode = result.path("base_resp").path("status_code").asInt(-1);
if (statusCode == 0 && result.has("task_id")) {
String taskId = result.get("task_id").asText();
log.info("[MiniMax] Submitted task: {} (model={})", taskId, model);
return VideoSubmitResult.success(taskId, id());
} else {
String errMsg = result.path("base_resp").path("status_msg").asText("未知错误");
log.warn("[MiniMax] Submit failed: {}", errMsg);
return VideoSubmitResult.failure(id(), errMsg);
}
} catch (Exception e) {
log.error("[MiniMax] Submit error: {}", e.getMessage(), e);
return VideoSubmitResult.failure(id(), e.getMessage());
}
}
@Override
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
try {
String apiKey = config.getMinimaxApiKey();
HttpResponse response = HttpRequest.get(
BASE_URL + "/v1/query/video_generation?task_id=" + providerTaskId)
.header("Authorization", "Bearer " + apiKey)
.timeout(15_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
String status = result.path("status").asText();
return switch (status) {
case "Success" -> {
// 优先取 video_url备选 file_id
String videoUrl = result.has("video_url") ? result.get("video_url").asText(null) : null;
if (videoUrl == null && result.has("file_id")) {
videoUrl = resolveFileUrl(result.get("file_id").asText(), apiKey);
}
yield TaskPollResult.succeeded(videoUrl, null, result.toString());
}
case "Fail" -> {
String errMsg = result.path("base_resp").path("status_msg").asText("任务失败");
yield TaskPollResult.failed(errMsg);
}
case "Processing" -> TaskPollResult.running(null);
default -> TaskPollResult.pending(null); // Preparing
};
} catch (Exception e) {
log.error("[MiniMax] Poll error for task {}: {}", providerTaskId, e.getMessage());
return null;
}
}
/**
* 通过 file_id 获取视频下载 URL
*/
private String resolveFileUrl(String fileId, String apiKey) {
try {
HttpResponse response = HttpRequest.get(
BASE_URL + "/v1/files/retrieve?file_id=" + fileId)
.header("Authorization", "Bearer " + apiKey)
.timeout(10_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
JsonNode file = result.path("file");
return file.has("download_url") ? file.get("download_url").asText(null) : null;
} catch (Exception e) {
log.warn("[MiniMax] Failed to resolve file URL for {}: {}", fileId, e.getMessage());
return null;
}
}
}

View File

@ -0,0 +1,171 @@
package vip.mate.tool.video.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.task.AsyncTaskService.TaskPollResult;
import vip.mate.tool.video.*;
import java.util.List;
import java.util.Set;
/**
* Runway 视频生成 Provider gen4.5 / gen4_turbo / gen3a_turbo
* <p>
* API 文档: https://docs.runwayml.com/
* 鉴权: Bearer Token
*
* @author MateClaw Team
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class RunwayVideoProvider implements VideoGenerationProvider {
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://api.dev.runwayml.com";
private static final String API_VERSION = "2024-11-06";
private static final String DEFAULT_MODEL = "gen4_turbo";
@Override
public String id() {
return "runway";
}
@Override
public String label() {
return "Runway";
}
@Override
public boolean requiresCredential() {
return true;
}
@Override
public int autoDetectOrder() {
return 250;
}
@Override
public Set<VideoCapability> capabilities() {
return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO);
}
@Override
public VideoProviderCapabilities detailedCapabilities() {
return VideoProviderCapabilities.builder()
.modes(capabilities())
.aspectRatios(List.of("16:9", "9:16"))
.supportedDurations(List.of(5, 10))
.maxDurationSeconds(10)
.defaultModel(DEFAULT_MODEL)
.models(List.of("gen4.5", "gen4_turbo", "gen3a_turbo"))
.build();
}
@Override
public boolean isAvailable(SystemSettingsDTO config) {
return StringUtils.hasText(config.getRunwayApiKey());
}
@Override
public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) {
try {
String apiKey = config.getRunwayApiKey();
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
// 根据模式选择端点
String endpoint = request.getMode() == VideoCapability.IMAGE_TO_VIDEO
? "/v1/image_to_video" : "/v1/text_to_video";
ObjectNode body = objectMapper.createObjectNode();
body.put("model", model);
body.put("promptText", request.getPrompt());
if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) {
body.put("promptImage", request.getImageUrl());
}
if (request.getAspectRatio() != null) {
// Runway "1280:720" 格式
String ratio = request.getAspectRatio().equals("16:9") ? "1280:720"
: request.getAspectRatio().equals("9:16") ? "720:1280"
: "1280:720";
body.put("ratio", ratio);
}
if (request.getDurationSeconds() != null) {
body.put("duration", request.getDurationSeconds());
}
HttpResponse response = HttpRequest.post(BASE_URL + endpoint)
.header("Authorization", "Bearer " + apiKey)
.header("X-Runway-Version", API_VERSION)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(30_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
if (response.getStatus() == 200 && result.has("id")) {
String taskId = result.get("id").asText();
log.info("[Runway] Submitted task: {} (model={})", taskId, model);
return VideoSubmitResult.success(taskId, id());
} else {
String errMsg = result.has("error") ? result.get("error").asText()
: "HTTP " + response.getStatus();
log.warn("[Runway] Submit failed: {}", errMsg);
return VideoSubmitResult.failure(id(), errMsg);
}
} catch (Exception e) {
log.error("[Runway] Submit error: {}", e.getMessage(), e);
return VideoSubmitResult.failure(id(), e.getMessage());
}
}
@Override
public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) {
try {
String apiKey = config.getRunwayApiKey();
HttpResponse response = HttpRequest.get(BASE_URL + "/v1/tasks/" + providerTaskId)
.header("Authorization", "Bearer " + apiKey)
.header("X-Runway-Version", API_VERSION)
.timeout(15_000)
.execute();
JsonNode result = objectMapper.readTree(response.body());
String status = result.path("status").asText();
return switch (status) {
case "SUCCEEDED" -> {
// output 是视频 URL 数组
JsonNode output = result.path("output");
String videoUrl = null;
if (output.isArray() && !output.isEmpty()) {
videoUrl = output.get(0).asText();
}
yield TaskPollResult.succeeded(videoUrl, null, result.toString());
}
case "FAILED", "CANCELLED" -> {
String failure = result.has("failure") ? result.get("failure").asText() : "任务失败";
yield TaskPollResult.failed(failure);
}
case "RUNNING" -> TaskPollResult.running(null);
case "THROTTLED" -> TaskPollResult.running(null); // 被限流等同于运行中
default -> TaskPollResult.pending(null); // PENDING
};
} catch (Exception e) {
log.error("[Runway] Poll error for task {}: {}", providerTaskId, e.getMessage());
return null;
}
}
}

View File

@ -631,35 +631,45 @@ export function useChat(options: UseChatOptions): UseChatReturn {
stream.on('async_task_completed', (data) => {
console.log('[useChat] Async task completed:', data)
if (data.success && streamConversationId) {
let mediaPart: MessageContentPart | null = null
if (data.videoUrl) {
// 视频生成完成
addMessage({
role: 'assistant',
content: '',
contentParts: [{
type: 'video',
fileUrl: data.videoUrl,
fileName: `video_${data.taskId}.mp4`,
contentType: 'video/mp4',
}] as MessageContentPart[],
status: 'completed',
conversationId: streamConversationId,
})
mediaPart = {
type: 'video',
fileUrl: data.videoUrl,
fileName: `video_${data.taskId}.mp4`,
contentType: 'video/mp4',
} as MessageContentPart
} else if (data.imageUrl) {
// 图片生成完成
addMessage({
role: 'assistant',
content: '',
contentParts: [{
type: 'image',
fileUrl: data.imageUrl,
fileName: `image_${data.taskId}.png`,
contentType: 'image/png',
}] as MessageContentPart[],
status: 'completed',
conversationId: streamConversationId,
})
mediaPart = {
type: 'image',
fileUrl: data.imageUrl,
fileName: `image_${data.taskId}.png`,
contentType: 'image/png',
} as MessageContentPart
}
if (!mediaPart) return
// 优先附加到当前 assistant 消息(避免图片跑到文字回复上方)
if (currentAssistantId.value) {
const msg = getMessage(currentAssistantId.value)
if (msg) {
const existingParts = (msg as any).contentParts || []
updateMessage(currentAssistantId.value, {
contentParts: [...existingParts, mediaPart],
} as any)
return
}
}
// 回退Agent 已结束,新建独立消息
addMessage({
role: 'assistant',
content: '',
contentParts: [mediaPart],
status: 'completed',
conversationId: streamConversationId,
})
}
})

View File

@ -208,6 +208,8 @@ export default {
system: 'System',
image: 'Image Generation',
tts: 'Text-to-Speech',
stt: 'Speech Recognition',
music: 'Music Generation',
video: 'Video Generation',
about: 'About',
},
@ -337,6 +339,12 @@ export default {
tavilyBaseUrl: 'Tavily Base URL',
duckduckgoEnabled: 'DuckDuckGo (Keyless)',
searxngBaseUrl: 'SearXNG Base URL',
sttEnabled: 'Enable Speech Recognition',
sttProvider: 'Preferred STT Provider',
sttFallbackEnabled: 'Provider Fallback',
musicEnabled: 'Enable Music Generation',
musicProvider: 'Preferred Music Provider',
musicFallbackEnabled: 'Provider Fallback',
ttsEnabled: 'Enable Text-to-Speech',
ttsProvider: 'Preferred TTS Provider',
ttsFallbackEnabled: 'Provider Fallback',
@ -355,6 +363,8 @@ export default {
falApiKey: 'fal.ai API Key',
klingAccessKey: 'Kling Access Key',
klingSecretKey: 'Kling Secret Key',
runwayApiKey: 'Runway API Key',
minimaxApiKey: 'MiniMax API Key',
},
hints: {
provider: 'Current implementation applies DashScope model options at runtime.',
@ -370,6 +380,16 @@ export default {
tavilyBaseUrl: 'Usually no need to change unless using a custom proxy.',
duckduckgoEnabled: 'Free keyless search fallback. No API key required. Enabled by default as a zero-config fallback.',
searxngBaseUrl: 'Self-hosted SearXNG instance URL. Auto-configured when deploying via Docker.',
sttEnabled: 'Enable speech-to-text for voice messages. Both providers reuse existing API keys.',
sttProvider: 'Select preferred STT provider. Auto mode picks the first available one.',
sttFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.',
openaiSttInfo: 'Reuses OpenAI API Key from Model Management. Whisper model, supports multilingual auto-detection.',
dashscopeSttInfo: 'Reuses DashScope API Key from Model Management. Paraformer model, excellent Chinese recognition.',
musicEnabled: 'Enable to let Agent use the music generation tool. Google Lyria reuses Google Key.',
musicProvider: 'Select preferred music provider. Auto mode prioritizes Google Lyria.',
musicFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.',
googleLyriaInfo: 'Reuses Google API Key from Model Management. Lyria 3 model, supports lyrics and instrumental.',
minimaxMusicInfo: 'Reuses MiniMax API Key from video settings. music-2.5+ model, supports lyrics and instrumental.',
ttsEnabled: 'Enable to use TTS via Read Aloud button or auto mode. Edge TTS is free, no API key needed.',
ttsProvider: 'Select preferred TTS provider. Auto mode prioritizes free Edge TTS.',
ttsFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.',
@ -385,6 +405,8 @@ export default {
openaiImageStatus: 'Reuses the OpenAI API Key configured in Model Management. No extra setup needed.',
zhipuImageApiKey: 'Get from bigmodel.cn. CogView-3-Flash model is free. Shared with video generation.',
falImageApiKey: 'Get from fal.ai for Flux image generation models. Shared with video generation.',
googleImagenInfo: 'Reuses Google API Key from Model Management. Supports Gemini image generation and Imagen 4.0.',
minimaxImageInfo: 'Reuses MiniMax API Key from video settings. image-01 model, multiple aspect ratios, up to 9 images.',
videoEnabled: 'Enable to let Agent use the video generation tool. Requires at least one configured video provider.',
videoProvider: 'Select preferred video provider. Auto mode picks the first available one.',
videoFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.',
@ -394,9 +416,19 @@ export default {
falApiKey: 'Get from fal.ai. One key accesses Kling, Runway, Luma and more.',
klingAccessKey: 'Get from Kuaishou Open Platform for Kling video generation API.',
klingSecretKey: 'Paired with Access Key for JWT authentication.',
runwayApiKey: 'Get from runwayml.com. Supports gen4.5, gen4_turbo flagship models.',
minimaxApiKey: 'Get from minimaxi.com. Hailuo video with free quota, excellent for Chinese scenes.',
},
searchTitle: 'Search Service',
searchDesc: 'Configure the built-in search tool provider and API credentials',
sttTitle: 'Speech Recognition',
sttDesc: 'Configure STT speech-to-text with OpenAI Whisper and DashScope Paraformer',
sttProviderOptions: { auto: 'Auto Select' },
sttProviderTags: { reuseLlmKey: 'Reuses LLM API Key' },
musicTitle: 'Music Generation',
musicDesc: 'Configure AI music generation with Google Lyria and MiniMax Music',
musicProviderOptions: { auto: 'Auto Select' },
musicProviderTags: { reuseLlmKey: 'Reuses LLM API Key', sharedWithVideo: 'Shared with Video' },
ttsTitle: 'Text-to-Speech',
ttsDesc: 'Configure TTS with Edge TTS (free), OpenAI TTS, and DashScope CosyVoice',
ttsProviderOptions: {

View File

@ -208,6 +208,8 @@ export default {
system: '系统设置',
image: '图片生成',
tts: '语音合成',
stt: '语音识别',
music: '音乐生成',
video: '视频生成',
about: '关于',
},
@ -337,6 +339,14 @@ export default {
tavilyBaseUrl: 'Tavily 接口地址',
duckduckgoEnabled: 'DuckDuckGo免 Key',
searxngBaseUrl: 'SearXNG 地址',
// STT 语音识别
sttEnabled: '启用语音识别',
sttProvider: '首选 STT Provider',
sttFallbackEnabled: 'Provider 回退',
// 音乐生成
musicEnabled: '启用音乐生成',
musicProvider: '首选音乐 Provider',
musicFallbackEnabled: 'Provider 回退',
// TTS 语音合成
ttsEnabled: '启用语音合成',
ttsProvider: '首选 TTS Provider',
@ -358,6 +368,8 @@ export default {
falApiKey: 'fal.ai API Key',
klingAccessKey: '可灵 Access Key',
klingSecretKey: '可灵 Secret Key',
runwayApiKey: 'Runway API Key',
minimaxApiKey: 'MiniMax API Key',
},
hints: {
provider: '当前版本会把 DashScope 模型参数真实应用到 Agent 调用链路。',
@ -373,6 +385,18 @@ export default {
tavilyBaseUrl: '通常无需修改,除非使用自定义代理地址。',
duckduckgoEnabled: '免费搜索兜底,无需 API Key。默认开启作为零配置下的搜索降级方案。',
searxngBaseUrl: '自部署 SearXNG 实例地址。Docker 部署时自动配置。',
// STT 语音识别
sttEnabled: '开启后支持语音消息转文字。OpenAI Whisper 和 DashScope Paraformer 均复用已有 Key。',
sttProvider: '选择首选 STT Providerauto 模式自动选择可用的 Provider。',
sttFallbackEnabled: '首选 Provider 失败时自动尝试其他已配置的 Provider。',
openaiSttInfo: '复用模型管理中的 OpenAI API Key。使用 Whisper 模型,支持多语言自动识别。',
dashscopeSttInfo: '复用模型管理中的 DashScope API Key。使用 Paraformer 模型,中文识别效果优秀。',
// 音乐生成
musicEnabled: '开启后 Agent 可通过 music_generate 工具生成音乐。Google Lyria 复用 Google Key。',
musicProvider: '选择首选音乐 Providerauto 模式优先使用 Google Lyria。',
musicFallbackEnabled: '首选 Provider 失败时自动尝试其他已配置的 Provider。',
googleLyriaInfo: '复用模型管理中的 Google API Key。Lyria 3 模型,支持歌词谱曲和纯音乐生成。',
minimaxMusicInfo: '复用视频生成中的 MiniMax API Key。music-2.5+ 模型,支持歌词和纯音乐。',
// TTS 语音合成
ttsEnabled: '开启后可通过消息朗读按钮或自动模式使用语音合成。Edge TTS 免费无需 Key。',
ttsProvider: '选择首选 TTS Providerauto 模式优先使用免费的 Edge TTS。',
@ -390,6 +414,8 @@ export default {
openaiImageStatus: '复用模型管理中配置的 OpenAI API Key无需额外配置。',
zhipuImageApiKey: '从 bigmodel.cn 获取。CogView-3-Flash 模型免费。与视频生成共用同一 Key。',
falImageApiKey: '从 fal.ai 获取,支持 Flux 系列图片生成模型。与视频生成共用同一 Key。',
googleImagenInfo: '复用模型管理中的 Google API Key。支持 Gemini 图片生成和 Imagen 4.0 模型。',
minimaxImageInfo: '复用视频生成中的 MiniMax API Key。image-01 模型,支持多种画面比例,最多 9 张。',
// 视频生成
videoEnabled: '开启后 Agent 可使用视频生成工具。需至少配置一个视频 Provider 的 API Key。',
videoProvider: '选择首选视频生成 Providerauto 模式自动选择第一个可用的。',
@ -400,9 +426,19 @@ export default {
falApiKey: '从 fal.ai 获取,一个 Key 可访问 Kling、Runway、Luma 等多个模型。',
klingAccessKey: '从快手开放平台获取,用于调用可灵视频生成 API。',
klingSecretKey: '与 Access Key 配对使用,用于 JWT 签名鉴权。',
runwayApiKey: '从 runwayml.com 获取。支持 gen4.5、gen4_turbo 等旗舰模型。',
minimaxApiKey: '从 minimaxi.com 获取。海螺 Hailuo 视频,有免费额度,中文场景优秀。',
},
searchTitle: '搜索服务',
searchDesc: '配置内置搜索工具的提供商与 API 凭证',
sttTitle: '语音识别',
sttDesc: '配置 STT 语音转文字,支持 OpenAI Whisper 和 DashScope Paraformer',
sttProviderOptions: { auto: '自动选择' },
sttProviderTags: { reuseLlmKey: '复用 LLM API Key' },
musicTitle: '音乐生成',
musicDesc: '配置 AI 音乐生成,支持 Google Lyria 和 MiniMax Music',
musicProviderOptions: { auto: '自动选择' },
musicProviderTags: { reuseLlmKey: '复用 LLM API Key', sharedWithVideo: '与视频共用' },
ttsTitle: '语音合成',
ttsDesc: '配置 TTS 语音合成,支持 Edge TTS免费、OpenAI TTS、DashScope CosyVoice',
ttsProviderOptions: {

View File

@ -97,6 +97,18 @@ const router = createRouter({
component: () => import('@/views/Settings/Tts/index.vue'),
meta: { title: 'Settings - TTS' },
},
{
path: 'stt',
name: 'SettingsStt',
component: () => import('@/views/Settings/Stt/index.vue'),
meta: { title: 'Settings - STT' },
},
{
path: 'music',
name: 'SettingsMusic',
component: () => import('@/views/Settings/Music/index.vue'),
meta: { title: 'Settings - Music' },
},
{
path: 'video',
name: 'SettingsVideo',

View File

@ -33,6 +33,8 @@
<option value="zhipu-cogview">智谱 CogView</option>
<option value="openai">OpenAI (DALL-E)</option>
<option value="fal">fal.ai (Flux)</option>
<option value="google-imagen">Google Imagen</option>
<option value="minimax">MiniMax Image</option>
</select>
</div>
</div>
@ -142,6 +144,36 @@
</div>
</div>
</div>
<!-- Google Imagen -->
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">Google Imagen</span>
<span class="provider-tag">{{ t('settings.imageProviderTags.reuseLlmKey') }}</span>
</div>
<div class="settings-card">
<div class="setting-item">
<div class="setting-info">
<div class="setting-hint">{{ t('settings.hints.googleImagenInfo') }}</div>
</div>
</div>
</div>
</div>
<!-- MiniMax Image -->
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">MiniMax Image</span>
<span class="provider-tag">{{ t('settings.imageProviderTags.sharedWithVideo') }}</span>
</div>
<div class="settings-card">
<div class="setting-item">
<div class="setting-info">
<div class="setting-hint">{{ t('settings.hints.minimaxImageInfo') }}</div>
</div>
</div>
</div>
</div>
</template>
<div class="save-bar">

View File

@ -53,6 +53,18 @@ const sections = computed(() => [
label: t('settings.sections.tts'),
icon: '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><polygon points="11 5 6 9 2 9 2 15 6 15 11 19 11 5"/><path d="M19.07 4.93a10 10 0 0 1 0 14.14"/><path d="M15.54 8.46a5 5 0 0 1 0 7.07"/></svg>',
},
{
id: 'stt',
path: '/settings/stt',
label: t('settings.sections.stt'),
icon: '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"/><path d="M19 10v2a7 7 0 0 1-14 0v-2"/><line x1="12" y1="19" x2="12" y2="23"/><line x1="8" y1="23" x2="16" y2="23"/></svg>',
},
{
id: 'music',
path: '/settings/music',
label: t('settings.sections.music'),
icon: '<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path d="M9 18V5l12-2v13"/><circle cx="6" cy="18" r="3"/><circle cx="18" cy="16" r="3"/></svg>',
},
{
id: 'video',
path: '/settings/video',

View File

@ -0,0 +1,133 @@
<template>
<div class="settings-section">
<div class="section-header">
<h2 class="section-title">{{ t('settings.musicTitle') }}</h2>
<p class="section-desc">{{ t('settings.musicDesc') }}</p>
</div>
<div class="settings-card">
<div class="setting-item">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.musicEnabled') }}</div>
<div class="setting-hint">{{ t('settings.hints.musicEnabled') }}</div>
</div>
<div class="setting-control">
<label class="toggle-switch">
<input v-model="settings.musicEnabled" type="checkbox" />
<span class="toggle-slider"></span>
</label>
</div>
</div>
<div class="setting-item">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.musicProvider') }}</div>
<div class="setting-hint">{{ t('settings.hints.musicProvider') }}</div>
</div>
<div class="setting-control">
<select v-model="settings.musicProvider" class="form-input" :disabled="!settings.musicEnabled">
<option value="auto">{{ t('settings.musicProviderOptions.auto') }}</option>
<option value="google-lyria">Google Lyria</option>
<option value="minimax">MiniMax Music</option>
</select>
</div>
</div>
<div class="setting-item">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.musicFallbackEnabled') }}</div>
<div class="setting-hint">{{ t('settings.hints.musicFallbackEnabled') }}</div>
</div>
<div class="setting-control">
<label class="toggle-switch">
<input v-model="settings.musicFallbackEnabled" type="checkbox" :disabled="!settings.musicEnabled" />
<span class="toggle-slider"></span>
</label>
</div>
</div>
</div>
<template v-if="settings.musicEnabled">
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">Google Lyria</span>
<span class="provider-tag">{{ t('settings.musicProviderTags.reuseLlmKey') }}</span>
</div>
<div class="settings-card"><div class="setting-item"><div class="setting-info"><div class="setting-hint">{{ t('settings.hints.googleLyriaInfo') }}</div></div></div></div>
</div>
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">MiniMax Music</span>
<span class="provider-tag">{{ t('settings.musicProviderTags.sharedWithVideo') }}</span>
</div>
<div class="settings-card"><div class="setting-item"><div class="setting-info"><div class="setting-hint">{{ t('settings.hints.minimaxMusicInfo') }}</div></div></div></div>
</div>
</template>
<div class="save-bar">
<button class="btn-secondary" @click="loadSettings">{{ t('common.reset') }}</button>
<button class="btn-primary" @click="onSaveSettings">{{ t('settings.actions.saveSystem') }}</button>
</div>
<div v-if="savedTip" class="save-tip">{{ savedTip }}</div>
</div>
</template>
<script setup lang="ts">
import { onMounted, reactive, ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { settingsApi } from '@/api'
const { t } = useI18n()
const savedTip = ref('')
const settings = reactive({ musicEnabled: false, musicProvider: 'auto', musicFallbackEnabled: true })
onMounted(() => loadSettings())
async function loadSettings() {
const res: any = await settingsApi.get()
const d = res.data || {}
settings.musicEnabled = d.musicEnabled ?? false
settings.musicProvider = d.musicProvider ?? 'auto'
settings.musicFallbackEnabled = d.musicFallbackEnabled ?? true
}
async function onSaveSettings() {
await settingsApi.update({ musicEnabled: settings.musicEnabled, musicProvider: settings.musicProvider, musicFallbackEnabled: settings.musicFallbackEnabled })
await loadSettings()
savedTip.value = t('settings.messages.saveSuccess')
setTimeout(() => { savedTip.value = '' }, 2500)
}
</script>
<style scoped>
.settings-section { width: 100%; }
.section-header { display: flex; flex-direction: column; gap: 6px; margin-bottom: 20px; }
.section-title { margin: 0; font-size: 22px; font-weight: 700; color: var(--mc-text-primary); }
.section-desc { margin: 0; font-size: 14px; color: var(--mc-text-secondary); }
.settings-card { background: var(--mc-bg-elevated); border: 1px solid var(--mc-border); border-radius: 16px; padding: 18px; box-shadow: 0 8px 24px rgba(124,63,30,0.04); width: 100%; }
.setting-item { display: flex; justify-content: space-between; gap: 20px; padding: 16px 0; border-bottom: 1px solid var(--mc-border-light); }
.setting-item:last-child { border-bottom: none; }
.setting-info { flex: 1; }
.setting-label { font-size: 15px; font-weight: 600; color: var(--mc-text-primary); margin-bottom: 4px; }
.setting-hint { font-size: 13px; color: var(--mc-text-secondary); }
.setting-control { width: 220px; display: flex; align-items: center; justify-content: flex-end; }
.form-input { width: 100%; border: 1px solid var(--mc-border); border-radius: 10px; padding: 10px 12px; font-size: 14px; background: var(--mc-bg-sunken); color: var(--mc-text-primary); }
.form-input:disabled { opacity: 0.5; cursor: not-allowed; }
.toggle-switch { position: relative; display: inline-flex; width: 44px; height: 24px; }
.toggle-switch input { opacity: 0; width: 0; height: 0; }
.toggle-slider { position: absolute; inset: 0; cursor: pointer; background: var(--mc-border); border-radius: 999px; transition: 0.2s; }
.toggle-slider::before { content: ''; position: absolute; width: 18px; height: 18px; left: 3px; top: 3px; background: var(--mc-bg-elevated); border-radius: 50%; transition: 0.2s; }
.toggle-switch input:checked + .toggle-slider { background: var(--mc-primary); }
.toggle-switch input:checked + .toggle-slider::before { transform: translateX(20px); }
.toggle-switch input:disabled + .toggle-slider { opacity: 0.5; cursor: not-allowed; }
.provider-section { margin-top: 24px; }
.provider-header { display: flex; align-items: center; gap: 10px; margin-bottom: 12px; }
.provider-name { font-size: 16px; font-weight: 600; color: var(--mc-text-primary); }
.provider-tag { font-size: 12px; padding: 2px 8px; border-radius: 6px; background: var(--mc-bg-sunken); color: var(--mc-text-secondary); }
.save-bar { display: flex; justify-content: flex-end; gap: 10px; margin-top: 20px; }
.btn-primary, .btn-secondary { border: none; border-radius: 10px; padding: 9px 14px; font-size: 14px; cursor: pointer; transition: all 0.15s; }
.btn-primary { background: var(--mc-primary); color: white; }
.btn-primary:hover { background: var(--mc-primary-hover); }
.btn-secondary { background: var(--mc-bg-elevated); color: var(--mc-text-primary); border: 1px solid var(--mc-border); }
.save-tip { position: fixed; right: 24px; bottom: 24px; background: var(--mc-text-primary); color: var(--mc-text-inverse); padding: 10px 14px; border-radius: 10px; box-shadow: 0 10px 30px rgba(124,63,30,0.22); }
</style>

View File

@ -0,0 +1,133 @@
<template>
<div class="settings-section">
<div class="section-header">
<h2 class="section-title">{{ t('settings.sttTitle') }}</h2>
<p class="section-desc">{{ t('settings.sttDesc') }}</p>
</div>
<div class="settings-card">
<div class="setting-item">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.sttEnabled') }}</div>
<div class="setting-hint">{{ t('settings.hints.sttEnabled') }}</div>
</div>
<div class="setting-control">
<label class="toggle-switch">
<input v-model="settings.sttEnabled" type="checkbox" />
<span class="toggle-slider"></span>
</label>
</div>
</div>
<div class="setting-item">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.sttProvider') }}</div>
<div class="setting-hint">{{ t('settings.hints.sttProvider') }}</div>
</div>
<div class="setting-control">
<select v-model="settings.sttProvider" class="form-input" :disabled="!settings.sttEnabled">
<option value="auto">{{ t('settings.sttProviderOptions.auto') }}</option>
<option value="openai">OpenAI Whisper</option>
<option value="dashscope">DashScope (Paraformer)</option>
</select>
</div>
</div>
<div class="setting-item">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.sttFallbackEnabled') }}</div>
<div class="setting-hint">{{ t('settings.hints.sttFallbackEnabled') }}</div>
</div>
<div class="setting-control">
<label class="toggle-switch">
<input v-model="settings.sttFallbackEnabled" type="checkbox" :disabled="!settings.sttEnabled" />
<span class="toggle-slider"></span>
</label>
</div>
</div>
</div>
<template v-if="settings.sttEnabled">
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">OpenAI Whisper</span>
<span class="provider-tag">{{ t('settings.sttProviderTags.reuseLlmKey') }}</span>
</div>
<div class="settings-card"><div class="setting-item"><div class="setting-info"><div class="setting-hint">{{ t('settings.hints.openaiSttInfo') }}</div></div></div></div>
</div>
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">DashScope (Paraformer)</span>
<span class="provider-tag">{{ t('settings.sttProviderTags.reuseLlmKey') }}</span>
</div>
<div class="settings-card"><div class="setting-item"><div class="setting-info"><div class="setting-hint">{{ t('settings.hints.dashscopeSttInfo') }}</div></div></div></div>
</div>
</template>
<div class="save-bar">
<button class="btn-secondary" @click="loadSettings">{{ t('common.reset') }}</button>
<button class="btn-primary" @click="onSaveSettings">{{ t('settings.actions.saveSystem') }}</button>
</div>
<div v-if="savedTip" class="save-tip">{{ savedTip }}</div>
</div>
</template>
<script setup lang="ts">
import { onMounted, reactive, ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { settingsApi } from '@/api'
const { t } = useI18n()
const savedTip = ref('')
const settings = reactive({ sttEnabled: false, sttProvider: 'auto', sttFallbackEnabled: true })
onMounted(() => loadSettings())
async function loadSettings() {
const res: any = await settingsApi.get()
const d = res.data || {}
settings.sttEnabled = d.sttEnabled ?? false
settings.sttProvider = d.sttProvider ?? 'auto'
settings.sttFallbackEnabled = d.sttFallbackEnabled ?? true
}
async function onSaveSettings() {
await settingsApi.update({ sttEnabled: settings.sttEnabled, sttProvider: settings.sttProvider, sttFallbackEnabled: settings.sttFallbackEnabled })
await loadSettings()
savedTip.value = t('settings.messages.saveSuccess')
setTimeout(() => { savedTip.value = '' }, 2500)
}
</script>
<style scoped>
.settings-section { width: 100%; }
.section-header { display: flex; flex-direction: column; gap: 6px; margin-bottom: 20px; }
.section-title { margin: 0; font-size: 22px; font-weight: 700; color: var(--mc-text-primary); }
.section-desc { margin: 0; font-size: 14px; color: var(--mc-text-secondary); }
.settings-card { background: var(--mc-bg-elevated); border: 1px solid var(--mc-border); border-radius: 16px; padding: 18px; box-shadow: 0 8px 24px rgba(124,63,30,0.04); width: 100%; }
.setting-item { display: flex; justify-content: space-between; gap: 20px; padding: 16px 0; border-bottom: 1px solid var(--mc-border-light); }
.setting-item:last-child { border-bottom: none; }
.setting-info { flex: 1; }
.setting-label { font-size: 15px; font-weight: 600; color: var(--mc-text-primary); margin-bottom: 4px; }
.setting-hint { font-size: 13px; color: var(--mc-text-secondary); }
.setting-control { width: 220px; display: flex; align-items: center; justify-content: flex-end; }
.form-input { width: 100%; border: 1px solid var(--mc-border); border-radius: 10px; padding: 10px 12px; font-size: 14px; background: var(--mc-bg-sunken); color: var(--mc-text-primary); }
.form-input:disabled { opacity: 0.5; cursor: not-allowed; }
.toggle-switch { position: relative; display: inline-flex; width: 44px; height: 24px; }
.toggle-switch input { opacity: 0; width: 0; height: 0; }
.toggle-slider { position: absolute; inset: 0; cursor: pointer; background: var(--mc-border); border-radius: 999px; transition: 0.2s; }
.toggle-slider::before { content: ''; position: absolute; width: 18px; height: 18px; left: 3px; top: 3px; background: var(--mc-bg-elevated); border-radius: 50%; transition: 0.2s; }
.toggle-switch input:checked + .toggle-slider { background: var(--mc-primary); }
.toggle-switch input:checked + .toggle-slider::before { transform: translateX(20px); }
.toggle-switch input:disabled + .toggle-slider { opacity: 0.5; cursor: not-allowed; }
.provider-section { margin-top: 24px; }
.provider-header { display: flex; align-items: center; gap: 10px; margin-bottom: 12px; }
.provider-name { font-size: 16px; font-weight: 600; color: var(--mc-text-primary); }
.provider-tag { font-size: 12px; padding: 2px 8px; border-radius: 6px; background: var(--mc-bg-sunken); color: var(--mc-text-secondary); }
.save-bar { display: flex; justify-content: flex-end; gap: 10px; margin-top: 20px; }
.btn-primary, .btn-secondary { border: none; border-radius: 10px; padding: 9px 14px; font-size: 14px; cursor: pointer; transition: all 0.15s; }
.btn-primary { background: var(--mc-primary); color: white; }
.btn-primary:hover { background: var(--mc-primary-hover); }
.btn-secondary { background: var(--mc-bg-elevated); color: var(--mc-text-primary); border: 1px solid var(--mc-border); }
.save-tip { position: fixed; right: 24px; bottom: 24px; background: var(--mc-text-primary); color: var(--mc-text-inverse); padding: 10px 14px; border-radius: 10px; box-shadow: 0 10px 30px rgba(124,63,30,0.22); }
</style>

View File

@ -33,6 +33,8 @@
<option value="zhipu-cogvideo">智谱 CogVideoX</option>
<option value="fal">fal.ai (Kling/Runway/Luma)</option>
<option value="kling">快手可灵 Kling</option>
<option value="runway">Runway</option>
<option value="minimax">MiniMax (Hailuo)</option>
</select>
</div>
</div>
@ -175,6 +177,56 @@
</div>
</div>
</div>
<!-- Runway -->
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">Runway</span>
<span class="provider-tag">gen4.5 / gen4_turbo</span>
</div>
<div class="settings-card">
<div class="setting-item setting-item-vertical">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.runwayApiKey') }}</div>
<div class="setting-hint">{{ t('settings.hints.runwayApiKey') }}</div>
</div>
<div class="setting-control-full">
<input
v-model="runwayApiKeyInput"
type="password"
class="form-input"
:placeholder="settings.runwayApiKeyMasked || t('settings.model.apiKeyInput')"
autocomplete="off"
/>
</div>
</div>
</div>
</div>
<!-- MiniMax (Hailuo) -->
<div class="provider-section">
<div class="provider-header">
<span class="provider-name">MiniMax (Hailuo)</span>
<span class="provider-tag tag-free">{{ t('settings.videoProviderTags.freeQuota') }}</span>
</div>
<div class="settings-card">
<div class="setting-item setting-item-vertical">
<div class="setting-info">
<div class="setting-label">{{ t('settings.fields.minimaxApiKey') }}</div>
<div class="setting-hint">{{ t('settings.hints.minimaxApiKey') }}</div>
</div>
<div class="setting-control-full">
<input
v-model="minimaxApiKeyInput"
type="password"
class="form-input"
:placeholder="settings.minimaxApiKeyMasked || t('settings.model.apiKeyInput')"
autocomplete="off"
/>
</div>
</div>
</div>
</div>
</template>
<div class="save-bar">
@ -199,6 +251,8 @@ const zhipuApiKeyInput = ref('')
const falApiKeyInput = ref('')
const klingAccessKeyInput = ref('')
const klingSecretKeyInput = ref('')
const runwayApiKeyInput = ref('')
const minimaxApiKeyInput = ref('')
const settings = reactive({
videoEnabled: false,
@ -209,6 +263,8 @@ const settings = reactive({
falApiKeyMasked: '',
klingAccessKeyMasked: '',
klingSecretKeyMasked: '',
runwayApiKeyMasked: '',
minimaxApiKeyMasked: '',
})
onMounted(async () => {
@ -226,11 +282,15 @@ async function loadSettings() {
settings.falApiKeyMasked = data.falApiKeyMasked ?? ''
settings.klingAccessKeyMasked = data.klingAccessKeyMasked ?? ''
settings.klingSecretKeyMasked = data.klingSecretKeyMasked ?? ''
settings.runwayApiKeyMasked = data.runwayApiKeyMasked ?? ''
settings.minimaxApiKeyMasked = data.minimaxApiKeyMasked ?? ''
//
zhipuApiKeyInput.value = ''
falApiKeyInput.value = ''
klingAccessKeyInput.value = ''
klingSecretKeyInput.value = ''
runwayApiKeyInput.value = ''
minimaxApiKeyInput.value = ''
}
async function onSaveSettings() {
@ -244,6 +304,8 @@ async function onSaveSettings() {
if (falApiKeyInput.value) payload.falApiKey = falApiKeyInput.value
if (klingAccessKeyInput.value) payload.klingAccessKey = klingAccessKeyInput.value
if (klingSecretKeyInput.value) payload.klingSecretKey = klingSecretKeyInput.value
if (runwayApiKeyInput.value) payload.runwayApiKey = runwayApiKeyInput.value
if (minimaxApiKeyInput.value) payload.minimaxApiKey = minimaxApiKeyInput.value
await settingsApi.update(payload)
await loadSettings()