From fe3a7f7b016e1fe0d5d3493c5fbe78a6b97d8f1e Mon Sep 17 00:00:00 2001 From: matevip Date: Tue, 7 Apr 2026 19:14:27 +0800 Subject: [PATCH] feat(video): add video generation capability with 4 providers and async task infrastructure --- .../graph/executor/ToolExecutionExecutor.java | 16 +- .../mate/system/model/SystemSettingsDTO.java | 27 ++ .../system/service/SystemSettingService.java | 62 ++++ .../java/vip/mate/task/AsyncTaskService.java | 340 ++++++++++++++++++ .../vip/mate/task/model/AsyncTaskEntity.java | 64 ++++ .../vip/mate/task/model/AsyncTaskInfo.java | 29 ++ .../mate/task/repository/AsyncTaskMapper.java | 14 + .../tool/builtin/ToolExecutionContext.java | 35 ++ .../mate/tool/builtin/VideoGenerateTool.java | 185 ++++++++++ .../vip/mate/tool/video/VideoCapability.java | 18 + .../mate/tool/video/VideoFileDownloader.java | 60 ++++ .../tool/video/VideoGenerationProvider.java | 61 ++++ .../tool/video/VideoGenerationRequest.java | 41 +++ .../tool/video/VideoGenerationResult.java | 47 +++ .../tool/video/VideoGenerationService.java | 231 ++++++++++++ .../tool/video/VideoProviderCapabilities.java | 80 +++++ .../tool/video/VideoProviderRegistry.java | 90 +++++ .../mate/tool/video/VideoSubmitResult.java | 42 +++ .../tool/video/provider/CogVideoProvider.java | 180 ++++++++++ .../provider/DashScopeVideoProvider.java | 222 ++++++++++++ .../tool/video/provider/FalVideoProvider.java | 194 ++++++++++ .../video/provider/KlingVideoProvider.java | 191 ++++++++++ .../src/main/resources/db/data-en.sql | 5 + .../src/main/resources/db/data-zh.sql | 5 + .../src/main/resources/db/schema.sql | 24 ++ mateclaw-ui/src/composables/chat/useChat.ts | 20 ++ mateclaw-ui/src/composables/chat/useStream.ts | 3 + mateclaw-ui/src/i18n/locales/en-US.ts | 29 ++ mateclaw-ui/src/i18n/locales/zh-CN.ts | 31 ++ mateclaw-ui/src/router/index.ts | 6 + mateclaw-ui/src/types/index.ts | 13 + mateclaw-ui/src/views/Settings/Layout.vue | 6 + .../src/views/Settings/Video/index.vue | 307 ++++++++++++++++ 33 files changed, 2677 insertions(+), 1 deletion(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskEntity.java create mode 100644 mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskInfo.java create mode 100644 mateclaw-server/src/main/java/vip/mate/task/repository/AsyncTaskMapper.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/builtin/ToolExecutionContext.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/builtin/VideoGenerateTool.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoCapability.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoFileDownloader.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationRequest.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderCapabilities.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderRegistry.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/VideoSubmitResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/provider/CogVideoProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/provider/FalVideoProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/video/provider/KlingVideoProvider.java create mode 100644 mateclaw-ui/src/views/Settings/Video/index.vue diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java index 94a0b08d..1ed69946 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java @@ -5,6 +5,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.ToolResponseMessage; import org.springframework.ai.tool.ToolCallback; +import vip.mate.tool.builtin.ToolExecutionContext; import vip.mate.agent.AgentToolSet; import vip.mate.agent.GraphEventPublisher; import vip.mate.approval.ApprovalWorkflowService; @@ -143,9 +144,13 @@ public class ToolExecutionExecutor { return execute(toolCalls, conversationId, agentId, isReplay, ""); } + /** 当前执行的 requesterId，传递给 ToolExecutionContext */ + private volatile String currentRequesterId; + public ToolExecutionResult execute(List toolCalls, String conversationId, String agentId, boolean isReplay, String requesterId) { + this.currentRequesterId = requesterId; List allResponses = new ArrayList<>(); List events = Collections.synchronizedList(new ArrayList<>()); @@ -386,7 +391,16 @@ public class ToolExecutionExecutor { log.info("[ToolExecutor] Executing tool: {} with args: {}", toolName, pc.arguments != null && pc.arguments.length() > 200 ? pc.arguments.substring(0, 200) + "..." : pc.arguments); - String result = pc.callback.call(pc.arguments); + + // 注入工具执行上下文（供 VideoGenerateTool 等获取 conversationId / username） + ToolExecutionContext.set(pc.conversationId, currentRequesterId); + String result; + try { + result = pc.callback.call(pc.arguments); + } finally { + ToolExecutionContext.clear(); + } + int rawLen = result != null ? result.length() : 0; result = truncateToolResult(result, MAX_TOOL_RESULT_CHARS); log.info("[ToolExecutor] Tool {} returned {} chars{}", toolName, rawLen, diff --git a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java index e4f8a594..d1636dcb 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java @@ -33,4 +33,31 @@ public class SystemSettingsDTO { // 用于前端回显脱敏后的 API Key private String serperApiKeyMasked; private String tavilyApiKeyMasked; + + // ===== 视频生成配置 ===== + /** 是否启用视频生成能力 */ + private Boolean videoEnabled; + /** 首选视频 provider: auto / dashscope / zhipu-cogvideo / fal / kling */ + private String videoProvider; + /** 是否启用 provider 级 fallback */ + private Boolean videoFallbackEnabled; + + // --- 智谱 CogVideo --- + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private String zhipuApiKey; + private String zhipuBaseUrl; + private String zhipuApiKeyMasked; + + // --- fal.ai --- + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private String falApiKey; + private String falApiKeyMasked; + + // --- 快手可灵 Kling --- + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private String klingAccessKey; + @JsonProperty(access = JsonProperty.Access.WRITE_ONLY) + private String klingSecretKey; + private String klingAccessKeyMasked; + private String klingSecretKeyMasked; } diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index d2ae1602..b797cf98 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -27,6 +27,16 @@ public class SystemSettingService { private static final String DUCKDUCKGO_ENABLED_KEY = "duckduckgoEnabled"; private static final String SEARXNG_BASE_URL_KEY = "searxngBaseUrl"; + // 视频生成配置 keys + private static final String VIDEO_ENABLED_KEY = "videoEnabled"; + private static final String VIDEO_PROVIDER_KEY = "videoProvider"; + private static final String VIDEO_FALLBACK_ENABLED_KEY = "videoFallbackEnabled"; + private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey"; + private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl"; + private static final String FAL_API_KEY_KEY = "falApiKey"; + private static final String KLING_ACCESS_KEY_KEY = "klingAccessKey"; + private static final String KLING_SECRET_KEY_KEY = "klingSecretKey"; + private final SystemSettingMapper systemSettingMapper; public SystemSettingsDTO getSettings() { @@ -48,6 +58,32 @@ public class SystemSettingService { // API Key 脱敏回显 dto.setSerperApiKeyMasked(maskApiKey(getValue(SERPER_API_KEY_KEY, ""))); dto.setTavilyApiKeyMasked(maskApiKey(getValue(TAVILY_API_KEY_KEY, ""))); + + // 视频生成配置 + dto.setVideoEnabled(Boolean.parseBoolean(getValue(VIDEO_ENABLED_KEY, "false"))); + dto.setVideoProvider(getValue(VIDEO_PROVIDER_KEY, "auto")); + dto.setVideoFallbackEnabled(Boolean.parseBoolean(getValue(VIDEO_FALLBACK_ENABLED_KEY, "true"))); + dto.setZhipuBaseUrl(getValue(ZHIPU_BASE_URL_KEY, "")); + dto.setZhipuApiKeyMasked(maskApiKey(getValue(ZHIPU_API_KEY_KEY, ""))); + dto.setFalApiKeyMasked(maskApiKey(getValue(FAL_API_KEY_KEY, ""))); + dto.setKlingAccessKeyMasked(maskApiKey(getValue(KLING_ACCESS_KEY_KEY, ""))); + dto.setKlingSecretKeyMasked(maskApiKey(getValue(KLING_SECRET_KEY_KEY, ""))); + return dto; + } + + /** + * 获取全部配置（内部使用，包含明文 API Key）— 供 VideoGenerationService 等后端服务使用 + */ + public SystemSettingsDTO getAllSettings() { + SystemSettingsDTO dto = getSettings(); + // 补充搜索明文 Key + dto.setSerperApiKey(getValue(SERPER_API_KEY_KEY, "")); + dto.setTavilyApiKey(getValue(TAVILY_API_KEY_KEY, "")); + // 补充视频明文 Key + dto.setZhipuApiKey(getValue(ZHIPU_API_KEY_KEY, "")); + dto.setFalApiKey(getValue(FAL_API_KEY_KEY, "")); + dto.setKlingAccessKey(getValue(KLING_ACCESS_KEY_KEY, "")); + dto.setKlingSecretKey(getValue(KLING_SECRET_KEY_KEY, "")); return dto; } @@ -104,6 +140,32 @@ public class SystemSettingService { if (dto.getSearxngBaseUrl() != null) { saveValue(SEARXNG_BASE_URL_KEY, dto.getSearxngBaseUrl(), "SearXNG 实例地址"); } + + // 视频生成配置 + if (dto.getVideoEnabled() != null) { + saveValue(VIDEO_ENABLED_KEY, String.valueOf(dto.getVideoEnabled()), "是否启用视频生成"); + } + if (dto.getVideoProvider() != null) { + saveValue(VIDEO_PROVIDER_KEY, dto.getVideoProvider(), "视频生成首选 Provider"); + } + if (dto.getVideoFallbackEnabled() != null) { + saveValue(VIDEO_FALLBACK_ENABLED_KEY, String.valueOf(dto.getVideoFallbackEnabled()), "视频 Provider 级 Fallback"); + } + if (dto.getZhipuApiKey() != null && !dto.getZhipuApiKey().isBlank()) { + saveValue(ZHIPU_API_KEY_KEY, dto.getZhipuApiKey(), "智谱 CogVideo API Key"); + } + if (dto.getZhipuBaseUrl() != null) { + saveValue(ZHIPU_BASE_URL_KEY, dto.getZhipuBaseUrl(), "智谱 API Base URL"); + } + if (dto.getFalApiKey() != null && !dto.getFalApiKey().isBlank()) { + saveValue(FAL_API_KEY_KEY, dto.getFalApiKey(), "fal.ai API Key"); + } + if (dto.getKlingAccessKey() != null && !dto.getKlingAccessKey().isBlank()) { + saveValue(KLING_ACCESS_KEY_KEY, dto.getKlingAccessKey(), "快手可灵 Access Key"); + } + if (dto.getKlingSecretKey() != null && !dto.getKlingSecretKey().isBlank()) { + saveValue(KLING_SECRET_KEY_KEY, dto.getKlingSecretKey(), "快手可灵 Secret Key"); + } return getSettings(); } diff --git a/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java b/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java new file mode 100644 index 00000000..c8ef8462 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java @@ -0,0 +1,340 @@ +package vip.mate.task; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.stereotype.Service; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.task.model.AsyncTaskEntity; +import vip.mate.task.model.AsyncTaskInfo; +import vip.mate.task.repository.AsyncTaskMapper; + +import jakarta.annotation.PreDestroy; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.*; +import java.util.function.BiConsumer; +import java.util.function.Function; + +/** + * 通用异步任务服务 — 管理长耗时任务的生命周期（提交、轮询、完成回写） + *

+ * 可复用于视频生成、图片生成、音频生成等异步场景。 + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class AsyncTaskService implements ApplicationRunner { + + private final AsyncTaskMapper asyncTaskMapper; + private final ChatStreamTracker streamTracker; + + /** 轮询线程池（小池，2 线程足够） */ + private final ScheduledExecutorService pollExecutor = + Executors.newScheduledThreadPool(2, r -> { + Thread t = new Thread(r, "async-task-poll"); + t.setDaemon(true); + return t; + }); + + /** 活跃轮询任务，key = taskId */ + private final ConcurrentHashMap> activePolls = new ConcurrentHashMap<>(); + + /** 每用户最多并行任务数 */ + private static final int MAX_ACTIVE_TASKS_PER_USER = 3; + + /** 默认轮询间隔（秒） */ + private static final int POLL_INTERVAL_SECONDS = 8; + + /** 最大轮询时长（分钟），超时自动标记失败 */ + private static final int MAX_POLL_DURATION_MINUTES = 15; + + /** 连续轮询失败次数上限，超出自动标记任务失败 */ + private static final int MAX_POLL_ERROR_COUNT = 5; + + /** 轮询连续错误计数 */ + private final ConcurrentHashMap pollErrorCounts = new ConcurrentHashMap<>(); + + // ==================== 任务创建 ==================== + + /** + * 创建一个异步任务记录 + * + * @return 创建的任务实体 + */ + public AsyncTaskEntity createTask(String taskType, String conversationId, + Long messageId, String providerName, + String providerTaskId, String requestJson, + String createdBy) { + // 并发限制检查 + long activeCount = asyncTaskMapper.selectCount( + new LambdaQueryWrapper() + .eq(AsyncTaskEntity::getCreatedBy, createdBy) + .in(AsyncTaskEntity::getStatus, List.of("pending", "running")) + ); + if (activeCount >= MAX_ACTIVE_TASKS_PER_USER) { + throw new IllegalStateException("已达到最大并行任务数（" + MAX_ACTIVE_TASKS_PER_USER + "），请等待现有任务完成"); + } + + AsyncTaskEntity entity = new AsyncTaskEntity(); + entity.setTaskId(UUID.randomUUID().toString().replace("-", "").substring(0, 16)); + entity.setTaskType(taskType); + entity.setStatus("pending"); + entity.setConversationId(conversationId); + entity.setMessageId(messageId); + entity.setProviderName(providerName); + entity.setProviderTaskId(providerTaskId); + entity.setRequestJson(requestJson); + entity.setProgress(0); + entity.setCreatedBy(createdBy); + entity.setCreateTime(LocalDateTime.now()); + entity.setUpdateTime(LocalDateTime.now()); + asyncTaskMapper.insert(entity); + + log.info("[AsyncTask] Created task {} (type={}, provider={}, providerTaskId={})", + entity.getTaskId(), taskType, providerName, providerTaskId); + return entity; + } + + // ==================== 轮询管理 ==================== + + /** + * 启动对某个任务的定期轮询 + * + * @param taskId 内部任务 ID + * @param statusChecker 轮询函数：providerTaskId → 状态 + * @param onComplete 完成回调：(task, status) → void + */ + public void startPolling(String taskId, + Function statusChecker, + BiConsumer onComplete) { + AsyncTaskEntity task = findByTaskId(taskId); + if (task == null) { + log.warn("[AsyncTask] Cannot start polling: task {} not found", taskId); + return; + } + + // 更新状态为 running + updateStatus(taskId, "running", null, null, null); + + LocalDateTime deadline = LocalDateTime.now().plusMinutes(MAX_POLL_DURATION_MINUTES); + + ScheduledFuture future = pollExecutor.scheduleWithFixedDelay(() -> { + try { + // 超时检查 + if (LocalDateTime.now().isAfter(deadline)) { + log.warn("[AsyncTask] Task {} timed out after {} minutes", taskId, MAX_POLL_DURATION_MINUTES); + updateStatus(taskId, "failed", null, null, "任务超时（超过 " + MAX_POLL_DURATION_MINUTES + " 分钟）"); + cancelPolling(taskId); + broadcastTaskEvent(task, "async_task_completed", false, null, "任务超时"); + return; + } + + TaskPollResult result = statusChecker.apply(task.getProviderTaskId()); + if (result == null) { + return; + } + + // 轮询成功，重置错误计数 + pollErrorCounts.remove(taskId); + + // 更新进度 + if (result.progress() != null) { + updateStatus(taskId, "running", result.progress(), null, null); + broadcastProgress(task, result.progress()); + } + + // 终态处理 + if (result.isTerminal()) { + cancelPolling(taskId); + if (result.succeeded()) { + updateStatus(taskId, "succeeded", 100, result.resultJson(), null); + } else { + updateStatus(taskId, "failed", null, null, result.errorMessage()); + } + // 刷新任务实体 + AsyncTaskEntity freshTask = findByTaskId(taskId); + onComplete.accept(freshTask, result); + } + } catch (Exception e) { + int errorCount = pollErrorCounts.merge(taskId, 1, Integer::sum); + log.error("[AsyncTask] Polling error for task {} ({}/{}): {}", + taskId, errorCount, MAX_POLL_ERROR_COUNT, e.getMessage(), e); + if (errorCount >= MAX_POLL_ERROR_COUNT) { + log.error("[AsyncTask] Task {} exceeded max poll errors, marking as failed", taskId); + updateStatus(taskId, "failed", null, null, + "轮询连续失败 " + errorCount + " 次: " + e.getMessage()); + cancelPolling(taskId); + pollErrorCounts.remove(taskId); + broadcastTaskEvent(task, "async_task_completed", false, null, + "轮询异常，任务已标记失败"); + } + } + }, 3, POLL_INTERVAL_SECONDS, TimeUnit.SECONDS); + + activePolls.put(taskId, future); + log.info("[AsyncTask] Started polling for task {} (interval={}s, timeout={}min)", + taskId, POLL_INTERVAL_SECONDS, MAX_POLL_DURATION_MINUTES); + } + + private void cancelPolling(String taskId) { + ScheduledFuture future = activePolls.remove(taskId); + if (future != null) { + future.cancel(false); + } + } + + // ==================== 状态更新 ==================== + + public void updateStatus(String taskId, String status, Integer progress, + String resultJson, String errorMessage) { + LambdaUpdateWrapper wrapper = new LambdaUpdateWrapper() + .eq(AsyncTaskEntity::getTaskId, taskId) + .set(AsyncTaskEntity::getStatus, status) + .set(AsyncTaskEntity::getUpdateTime, LocalDateTime.now()); + if (progress != null) { + wrapper.set(AsyncTaskEntity::getProgress, progress); + } + if (resultJson != null) { + wrapper.set(AsyncTaskEntity::getResultJson, resultJson); + } + if (errorMessage != null) { + wrapper.set(AsyncTaskEntity::getErrorMessage, errorMessage); + } + asyncTaskMapper.update(null, wrapper); + } + + // ==================== 查询 ==================== + + public AsyncTaskInfo getTaskInfo(String taskId) { + AsyncTaskEntity entity = findByTaskId(taskId); + if (entity == null) { + return null; + } + return toInfo(entity); + } + + public List listActiveTasks(String conversationId) { + List entities = asyncTaskMapper.selectList( + new LambdaQueryWrapper() + .eq(AsyncTaskEntity::getConversationId, conversationId) + .in(AsyncTaskEntity::getStatus, List.of("pending", "running")) + .orderByDesc(AsyncTaskEntity::getCreateTime) + ); + return entities.stream().map(this::toInfo).toList(); + } + + private AsyncTaskEntity findByTaskId(String taskId) { + return asyncTaskMapper.selectOne( + new LambdaQueryWrapper() + .eq(AsyncTaskEntity::getTaskId, taskId) + ); + } + + private AsyncTaskInfo toInfo(AsyncTaskEntity entity) { + return AsyncTaskInfo.builder() + .taskId(entity.getTaskId()) + .taskType(entity.getTaskType()) + .status(entity.getStatus()) + .progress(entity.getProgress()) + .providerName(entity.getProviderName()) + .errorMessage(entity.getErrorMessage()) + .createTime(entity.getCreateTime()) + .build(); + } + + // ==================== SSE 广播 ==================== + + private void broadcastProgress(AsyncTaskEntity task, int progress) { + Map data = Map.of( + "taskId", task.getTaskId(), + "taskType", task.getTaskType(), + "progress", progress, + "providerName", Objects.toString(task.getProviderName(), "") + ); + streamTracker.broadcastObject(task.getConversationId(), "async_task_progress", data); + } + + public void broadcastTaskEvent(AsyncTaskEntity task, String eventName, + boolean success, String videoUrl, String errorMessage) { + Map data = new HashMap<>(); + data.put("taskId", task.getTaskId()); + data.put("taskType", task.getTaskType()); + data.put("success", success); + if (videoUrl != null) data.put("videoUrl", videoUrl); + if (errorMessage != null) data.put("errorMessage", errorMessage); + streamTracker.broadcastObject(task.getConversationId(), eventName, data); + } + + // ==================== 启动恢复 ==================== + + @Override + public void run(ApplicationArguments args) { + List pendingTasks = asyncTaskMapper.selectList( + new LambdaQueryWrapper() + .in(AsyncTaskEntity::getStatus, List.of("pending", "running")) + ); + if (!pendingTasks.isEmpty()) { + log.info("[AsyncTask] Found {} unfinished tasks on startup, marking as failed", pendingTasks.size()); + for (AsyncTaskEntity task : pendingTasks) { + updateStatus(task.getTaskId(), "failed", null, null, + "服务重启导致任务中断，请重新提交"); + } + } + } + + // ==================== 生命周期 ==================== + + @PreDestroy + public void shutdown() { + int count = activePolls.size(); + activePolls.values().forEach(f -> f.cancel(false)); + activePolls.clear(); + pollErrorCounts.clear(); + pollExecutor.shutdownNow(); + log.info("[AsyncTask] Shutdown complete, cancelled {} active polls", count); + } + + // ==================== 轮询结果 ==================== + + /** + * Provider 轮询返回的结果 + */ + public record TaskPollResult( + String state, // pending / running / succeeded / failed + Integer progress, // 0-100, nullable + String videoUrl, // 成功时的视频 URL + String coverImageUrl,// 可选封面图 + String resultJson, // 完成时的完整结果 JSON + String errorMessage // 失败时的错误信息 + ) { + public boolean isTerminal() { + return "succeeded".equals(state) || "failed".equals(state); + } + + public boolean succeeded() { + return "succeeded".equals(state); + } + + public static TaskPollResult pending(Integer progress) { + return new TaskPollResult("pending", progress, null, null, null, null); + } + + public static TaskPollResult running(Integer progress) { + return new TaskPollResult("running", progress, null, null, null, null); + } + + public static TaskPollResult succeeded(String videoUrl, String coverImageUrl, String resultJson) { + return new TaskPollResult("succeeded", 100, videoUrl, coverImageUrl, resultJson, null); + } + + public static TaskPollResult failed(String errorMessage) { + return new TaskPollResult("failed", null, null, null, null, errorMessage); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskEntity.java b/mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskEntity.java new file mode 100644 index 00000000..b1f80dbf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskEntity.java @@ -0,0 +1,64 @@ +package vip.mate.task.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 异步任务实体 — 通用长耗时任务持久化（视频生成、图片生成等） + * + * @author MateClaw Team + */ +@Data +@TableName("mate_async_task") +public class AsyncTaskEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + /** 对外公开的任务 ID（UUID 格式） */ + private String taskId; + + /** 任务类型：video_generation / image_generation 等 */ + private String taskType; + + /** 任务状态：pending / running / succeeded / failed */ + private String status; + + /** 关联会话 ID（用于完成后回写） */ + private String conversationId; + + /** 关联消息 ID */ + private Long messageId; + + /** 处理该任务的 provider 名称 */ + private String providerName; + + /** provider 返回的外部任务 ID */ + private String providerTaskId; + + /** 序列化的请求参数（JSON），用于重试 */ + @TableField(updateStrategy = FieldStrategy.ALWAYS) + private String requestJson; + + /** 序列化的结果（JSON），完成时填充 */ + @TableField(updateStrategy = FieldStrategy.ALWAYS) + private String resultJson; + + /** 错误信息 */ + @TableField(updateStrategy = FieldStrategy.ALWAYS) + private String errorMessage; + + /** 进度（0-100） */ + private Integer progress; + + /** 创建人 */ + private String createdBy; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; +} diff --git a/mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskInfo.java b/mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskInfo.java new file mode 100644 index 00000000..126b777b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/task/model/AsyncTaskInfo.java @@ -0,0 +1,29 @@ +package vip.mate.task.model; + +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * 异步任务对外 DTO + * + * @author MateClaw Team + */ +@Data +@Builder +public class AsyncTaskInfo { + + private String taskId; + private String taskType; + private String status; + private Integer progress; + private String providerName; + private String resultVideoUrl; + private String errorMessage; + private LocalDateTime createTime; + + public boolean isTerminal() { + return "succeeded".equals(status) || "failed".equals(status); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/task/repository/AsyncTaskMapper.java b/mateclaw-server/src/main/java/vip/mate/task/repository/AsyncTaskMapper.java new file mode 100644 index 00000000..ddb806d7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/task/repository/AsyncTaskMapper.java @@ -0,0 +1,14 @@ +package vip.mate.task.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.task.model.AsyncTaskEntity; + +/** + * 异步任务 Mapper + * + * @author MateClaw Team + */ +@Mapper +public interface AsyncTaskMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ToolExecutionContext.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ToolExecutionContext.java new file mode 100644 index 00000000..7a4f5528 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ToolExecutionContext.java @@ -0,0 +1,35 @@ +package vip.mate.tool.builtin; + +/** + * 工具执行上下文 — 通过 ThreadLocal 向 @Tool 方法传递执行环境信息 + *

+ * 在 ToolExecutionExecutor.executeSingleTool() 中 set，在 finally 中 clear。 + * 视频生成等需要知道 conversationId 的工具从此处获取。 + * + * @author MateClaw Team + */ +public final class ToolExecutionContext { + + private static final ThreadLocal CONVERSATION_ID = new ThreadLocal<>(); + private static final ThreadLocal USERNAME = new ThreadLocal<>(); + + private ToolExecutionContext() {} + + public static void set(String conversationId, String username) { + CONVERSATION_ID.set(conversationId); + USERNAME.set(username); + } + + public static String conversationId() { + return CONVERSATION_ID.get(); + } + + public static String username() { + return USERNAME.get(); + } + + public static void clear() { + CONVERSATION_ID.remove(); + USERNAME.remove(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/VideoGenerateTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/VideoGenerateTool.java new file mode 100644 index 00000000..e26e9381 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/VideoGenerateTool.java @@ -0,0 +1,185 @@ +package vip.mate.tool.builtin; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.task.AsyncTaskService; +import vip.mate.task.model.AsyncTaskInfo; +import vip.mate.tool.video.*; + +import java.util.List; +import java.util.StringJoiner; + +/** + * 视频生成工具 — Agent 可调用的 @Tool，提交异步视频生成任务 + *

+ * 借鉴 OpenClaw 的 video-generate-tool.ts 设计，支持 action=generate/list/status， + * 以及 session 级重复提交防护。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class VideoGenerateTool { + + private final VideoGenerationService videoGenerationService; + private final VideoProviderRegistry providerRegistry; + private final SystemSettingService systemSettingService; + private final AsyncTaskService asyncTaskService; + + @Tool(description = "视频生成工具，支持以下 action：\n" + + "- generate（默认）：生成视频。提供 prompt 描述视频内容，可选 aspectRatio/duration/imageUrl/model\n" + + "- list：列出所有可用的视频 Provider 及其支持的模型和能力\n" + + "- status：查看当前会话中正在进行的视频生成任务状态\n" + + "视频生成是异步过程（1-5 分钟），完成后自动显示在对话中。") + public String video_generate( + @ToolParam(description = "操作类型: generate（生成视频）、list（列出可用 Provider）、status（查看任务状态），默认 generate", required = false) String action, + @ToolParam(description = "视频内容描述，尽量详细（generate 时必填）", required = false) String prompt, + @ToolParam(description = "画面比例: 16:9 / 9:16 / 1:1，默认 16:9", required = false) String aspectRatio, + @ToolParam(description = "视频时长（秒），如 5 或 10，默认 5", required = false) Integer duration, + @ToolParam(description = "参考图片 URL（图生视频模式）", required = false) String imageUrl, + @ToolParam(description = "指定模型名称（可选）", required = false) String model, + @ToolParam(description = "查询指定任务 ID 的状态（status 模式时使用）", required = false) String taskId + ) { + // 路由 action + String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase(); + + return switch (normalizedAction) { + case "list" -> handleListAction(); + case "status" -> handleStatusAction(taskId); + default -> handleGenerateAction(prompt, aspectRatio, duration, imageUrl, model); + }; + } + + // ==================== action=list ==================== + + private String handleListAction() { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + List providers = providerRegistry.allSorted(); + + if (providers.isEmpty()) { + return "当前没有注册的视频生成 Provider。"; + } + + StringJoiner sb = new StringJoiner("\n\n"); + sb.add("## 可用的视频生成 Provider\n"); + + for (VideoGenerationProvider p : providers) { + boolean available = p.isAvailable(config); + VideoProviderCapabilities caps = p.detailedCapabilities(); + + StringJoiner entry = new StringJoiner("\n"); + entry.add("### " + p.label() + " (" + p.id() + ") " + (available ? "[已配置]" : "[未配置]")); + + if (caps != null) { + entry.add("- 模式: " + caps.getModes()); + if (caps.getModels() != null && !caps.getModels().isEmpty()) { + entry.add("- 模型: " + String.join(", ", caps.getModels())); + } + entry.add("- 画面比例: " + String.join(", ", caps.getAspectRatios())); + entry.add("- 支持时长: " + caps.getSupportedDurations() + " 秒"); + } + sb.add(entry.toString()); + } + return sb.toString(); + } + + // ==================== action=status ==================== + + private String handleStatusAction(String taskId) { + String conversationId = ToolExecutionContext.conversationId(); + + // 指定 taskId 查询 + if (taskId != null && !taskId.isBlank()) { + AsyncTaskInfo info = videoGenerationService.checkTaskStatus(taskId); + if (info == null) { + return "未找到任务 ID: " + taskId; + } + return formatTaskStatus(info); + } + + // 查询当前会话的所有活跃任务 + if (conversationId == null) { + return "无法获取当前会话信息"; + } + List activeTasks = asyncTaskService.listActiveTasks(conversationId); + if (activeTasks.isEmpty()) { + return "当前会话没有进行中的视频生成任务。"; + } + + StringJoiner sb = new StringJoiner("\n"); + sb.add("当前会话有 " + activeTasks.size() + " 个进行中的任务："); + for (AsyncTaskInfo task : activeTasks) { + sb.add("- 任务 " + task.getTaskId() + ": " + formatTaskStatus(task)); + } + return sb.toString(); + } + + // ==================== action=generate ==================== + + private String handleGenerateAction(String prompt, String aspectRatio, Integer duration, + String imageUrl, String model) { + String conversationId = ToolExecutionContext.conversationId(); + String username = ToolExecutionContext.username(); + + if (conversationId == null || conversationId.isBlank()) { + return "错误：无法获取当前会话信息，请重试"; + } + + if (prompt == null || prompt.isBlank()) { + return "错误：prompt 为必填参数，请描述你想要生成的视频内容"; + } + + // Session 级重复提交防护（借鉴 OpenClaw 的 duplicateGuard） + List activeTasks = asyncTaskService.listActiveTasks(conversationId); + long videoTasks = activeTasks.stream() + .filter(t -> "video_generation".equals(t.getTaskType())) + .count(); + if (videoTasks > 0) { + AsyncTaskInfo existing = activeTasks.stream() + .filter(t -> "video_generation".equals(t.getTaskType())) + .findFirst().orElse(null); + return "当前会话已有一个视频生成任务正在进行中（任务 ID: " + + (existing != null ? existing.getTaskId() : "unknown") + + "）。请等待完成后再提交新任务，或使用 action=status 查看进度。"; + } + + VideoGenerationRequest request = VideoGenerationRequest.builder() + .prompt(prompt) + .aspectRatio(aspectRatio) + .durationSeconds(duration) + .imageUrl(imageUrl) + .model(model) + .build(); + + VideoGenerationResult result = videoGenerationService.submitGeneration( + request, conversationId, username != null ? username : "system"); + + if (result.isSubmitted()) { + return result.getMessage(); + } else { + return "视频生成失败：" + result.getMessage(); + } + } + + // ==================== 辅助方法 ==================== + + private String formatTaskStatus(AsyncTaskInfo info) { + return switch (info.getStatus()) { + case "pending" -> "排队中，请稍候..."; + case "running" -> { + String progressStr = info.getProgress() != null && info.getProgress() > 0 + ? "（进度: " + info.getProgress() + "%）" : ""; + yield "生成中" + progressStr + "（" + info.getProviderName() + "）"; + } + case "succeeded" -> "已完成，视频已显示在对话中"; + case "failed" -> "失败：" + (info.getErrorMessage() != null ? info.getErrorMessage() : "未知错误"); + default -> "状态: " + info.getStatus(); + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoCapability.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoCapability.java new file mode 100644 index 00000000..96e49094 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoCapability.java @@ -0,0 +1,18 @@ +package vip.mate.tool.video; + +/** + * 视频生成能力枚举 + * + * @author MateClaw Team + */ +public enum VideoCapability { + + /** 文字生成视频 */ + GENERATE, + + /** 图片生成视频 */ + IMAGE_TO_VIDEO, + + /** 视频生成视频（风格转换等） */ + VIDEO_TO_VIDEO +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoFileDownloader.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoFileDownloader.java new file mode 100644 index 00000000..61bfded6 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoFileDownloader.java @@ -0,0 +1,60 @@ +package vip.mate.tool.video; + +import cn.hutool.http.HttpUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * 视频文件下载器 — 从 provider CDN 下载视频到本地存储 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class VideoFileDownloader { + + private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads"); + + /** + * 下载视频到本地 + * + * @param videoUrl provider 返回的视频 URL + * @param conversationId 会话 ID + * @param taskId 任务 ID + * @return 本地文件路径 + */ + public Path download(String videoUrl, String conversationId, String taskId) throws IOException { + Path dir = UPLOAD_ROOT.resolve(conversationId); + Files.createDirectories(dir); + + String extension = guessExtension(videoUrl); + String fileName = "video_" + taskId + extension; + Path targetFile = dir.resolve(fileName); + + log.info("[VideoDownloader] Downloading video from {} to {}", videoUrl, targetFile); + long size = HttpUtil.downloadFile(videoUrl, targetFile.toFile()); + log.info("[VideoDownloader] Downloaded {} bytes to {}", size, targetFile); + + return targetFile; + } + + /** + * 构造文件的 API 访问 URL + */ + public String toServingUrl(String conversationId, Path localPath) { + return "/api/v1/chat/files/" + conversationId + "/" + localPath.getFileName().toString(); + } + + private String guessExtension(String url) { + String lower = url.toLowerCase(); + if (lower.contains(".mp4")) return ".mp4"; + if (lower.contains(".webm")) return ".webm"; + if (lower.contains(".mov")) return ".mov"; + return ".mp4"; // 默认 mp4 + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationProvider.java new file mode 100644 index 00000000..f022f003 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationProvider.java @@ -0,0 +1,61 @@ +package vip.mate.tool.video; + +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; + +import java.util.Set; + +/** + * 视频生成提供商接口 — 所有视频 provider 统一实现此接口 + *

+ * 设计参考 {@link vip.mate.tool.search.SearchProvider}。 + * + * @author MateClaw Team + */ +public interface VideoGenerationProvider { + + /** 提供商唯一 ID，如 "dashscope"、"zhipu-cogvideo"、"fal"、"kling" */ + String id(); + + /** 显示名称 */ + String label(); + + /** 是否需要 API Key / Credential */ + boolean requiresCredential(); + + /** + * 自动探测排序优先级（升序）。 + * 有 credential 的 provider 优先于不需要 credential 的。 + */ + int autoDetectOrder(); + + /** 该 provider 支持的能力集 */ + Set capabilities(); + + /** 细粒度能力声明（支持的 aspectRatio、duration、模型列表等） */ + VideoProviderCapabilities detailedCapabilities(); + + /** + * 判断该 provider 在当前配置下是否可用 + * （API Key 已配置等） + */ + boolean isAvailable(SystemSettingsDTO config); + + /** + * 提交视频生成任务（异步，非阻塞） + * + * @param request 统一请求参数 + * @param config 系统配置 + * @return 提交结果（含 provider 任务 ID） + */ + VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config); + + /** + * 轮询任务状态 + * + * @param providerTaskId provider 返回的任务 ID + * @param config 系统配置 + * @return 轮询结果 + */ + TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config); +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationRequest.java new file mode 100644 index 00000000..03a7c5a0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationRequest.java @@ -0,0 +1,41 @@ +package vip.mate.tool.video; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +/** + * 视频生成统一请求 + * + * @author MateClaw Team + */ +@Data +@Builder +public class VideoGenerationRequest { + + /** 视频内容描述 */ + private String prompt; + + /** 生成模式（由 runtime 自动推断） */ + private VideoCapability mode; + + /** 指定模型名称（可选，provider 有默认值） */ + private String model; + + /** 画面比例：16:9 / 9:16 / 1:1 */ + @Builder.Default + private String aspectRatio = "16:9"; + + /** 视频时长（秒） */ + private Integer durationSeconds; + + /** 参考图片 URL（IMAGE_TO_VIDEO 模式） */ + private String imageUrl; + + /** 参考视频 URL（VIDEO_TO_VIDEO 模式） */ + private String videoUrl; + + /** provider 特有的额外参数 */ + private Map extraParams; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationResult.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationResult.java new file mode 100644 index 00000000..d5e9f108 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationResult.java @@ -0,0 +1,47 @@ +package vip.mate.tool.video; + +import lombok.Builder; +import lombok.Data; + +/** + * 视频生成服务提交结果（面向 Tool 层） + * + * @author MateClaw Team + */ +@Data +@Builder +public class VideoGenerationResult { + + /** 内部任务 ID（供 Agent 查询状态用） */ + private String taskId; + + /** 处理该任务的 provider 名称 */ + private String providerName; + + /** 状态描述 */ + private String status; + + /** 面向 Agent 的说明文本 */ + private String message; + + /** 是否成功提交 */ + private boolean submitted; + + public static VideoGenerationResult success(String taskId, String providerName) { + return VideoGenerationResult.builder() + .taskId(taskId) + .providerName(providerName) + .status("submitted") + .submitted(true) + .message("视频生成任务已提交（任务 ID: " + taskId + "）。预计 1-5 分钟完成，完成后会自动显示在对话中。") + .build(); + } + + public static VideoGenerationResult failure(String message) { + return VideoGenerationResult.builder() + .submitted(false) + .status("failed") + .message(message) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java new file mode 100644 index 00000000..4a282ee2 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java @@ -0,0 +1,231 @@ +package vip.mate.tool.video; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.task.AsyncTaskService; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.task.model.AsyncTaskEntity; +import vip.mate.task.model.AsyncTaskInfo; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.nio.file.Path; +import java.util.List; + +/** + * 视频生成服务 — 统一入口，处理 provider 选择、参数归一化、fallback、异步提交 + *

+ * 对应 OpenClaw 的 video-generation/runtime.ts + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class VideoGenerationService { + + private final SystemSettingService systemSettingService; + private final VideoProviderRegistry providerRegistry; + private final AsyncTaskService asyncTaskService; + private final ConversationService conversationService; + private final VideoFileDownloader fileDownloader; + private final ObjectMapper objectMapper; + + private static final String TASK_TYPE = "video_generation"; + + /** + * 提交视频生成任务（异步） + */ + public VideoGenerationResult submitGeneration(VideoGenerationRequest request, + String conversationId, + String createdBy) { + SystemSettingsDTO config = systemSettingService.getAllSettings(); + + // 1. 检查视频功能是否启用 + if (!Boolean.TRUE.equals(config.getVideoEnabled())) { + return VideoGenerationResult.failure("视频生成功能未启用，请在系统设置中开启"); + } + + // 2. 模式推断 + if (request.getMode() == null) { + request.setMode(inferMode(request)); + } + + // 3. 参数归一化 + normalizeRequest(request); + + // 4. Provider 选择 + VideoProviderRegistry.ResolvedProvider resolved = + providerRegistry.resolve(config, request.getMode()); + if (resolved == null) { + return VideoGenerationResult.failure( + "没有可用的视频生成 Provider，请在系统设置中配置（支持 DashScope、智谱、fal.ai、可灵）"); + } + + // 5. 提交（含 fallback） + return submitWithFallback(request, config, resolved, conversationId, createdBy); + } + + /** + * 查询任务状态 + */ + public AsyncTaskInfo checkTaskStatus(String taskId) { + return asyncTaskService.getTaskInfo(taskId); + } + + // ==================== 内部逻辑 ==================== + + private VideoGenerationResult submitWithFallback(VideoGenerationRequest request, + SystemSettingsDTO config, + VideoProviderRegistry.ResolvedProvider primary, + String conversationId, + String createdBy) { + // 尝试 primary（先按 provider 能力归一化参数） + normalizeForProvider(request, primary.provider()); + VideoSubmitResult submitResult = primary.provider().submit(request, config); + if (submitResult.isAccepted()) { + return createAsyncTask(submitResult, request, conversationId, createdBy, config); + } + + // Fallback（收集所有尝试的错误信息） + List attemptErrors = new java.util.ArrayList<>(); + attemptErrors.add(primary.provider().id() + ": " + submitResult.getErrorMessage()); + + if (Boolean.TRUE.equals(config.getVideoFallbackEnabled())) { + List fallbacks = + providerRegistry.fallbackCandidates(config, request.getMode(), primary.provider().id()); + for (VideoGenerationProvider fb : fallbacks) { + log.info("[VideoGen] Trying fallback provider: {}", fb.id()); + normalizeForProvider(request, fb); + submitResult = fb.submit(request, config); + if (submitResult.isAccepted()) { + return createAsyncTask(submitResult, request, conversationId, createdBy, config); + } + attemptErrors.add(fb.id() + ": " + submitResult.getErrorMessage()); + log.warn("[VideoGen] Fallback provider {} failed: {}", fb.id(), submitResult.getErrorMessage()); + } + } + + return VideoGenerationResult.failure( + "所有 Provider 均提交失败\n" + String.join("\n", attemptErrors)); + } + + private VideoGenerationResult createAsyncTask(VideoSubmitResult submitResult, + VideoGenerationRequest request, + String conversationId, + String createdBy, + SystemSettingsDTO config) { + try { + String requestJson = objectMapper.writeValueAsString(request); + AsyncTaskEntity task = asyncTaskService.createTask( + TASK_TYPE, conversationId, null, + submitResult.getProviderName(), + submitResult.getProviderTaskId(), + requestJson, createdBy); + + // 获取 provider 引用用于轮询 + VideoGenerationProvider provider = providerRegistry.getById(submitResult.getProviderName()); + if (provider == null) { + return VideoGenerationResult.failure("Provider 不存在: " + submitResult.getProviderName()); + } + + // 启动轮询（每次轮询重新获取配置，避免 API Key 轮换后使用过期凭证） + asyncTaskService.startPolling( + task.getTaskId(), + providerTaskId -> provider.checkStatus(providerTaskId, systemSettingService.getAllSettings()), + (completedTask, pollResult) -> handleCompletion(completedTask, pollResult) + ); + + return VideoGenerationResult.success(task.getTaskId(), submitResult.getProviderName()); + } catch (Exception e) { + log.error("[VideoGen] Failed to create async task: {}", e.getMessage(), e); + return VideoGenerationResult.failure("创建任务失败: " + e.getMessage()); + } + } + + /** + * 任务完成时的回写逻辑：下载视频 → 保存消息 → 广播 SSE + */ + private void handleCompletion(AsyncTaskEntity task, TaskPollResult result) { + if (result.succeeded()) { + try { + String videoUrl = result.videoUrl(); + if (videoUrl == null) { + log.warn("[VideoGen] Task {} succeeded but no video URL", task.getTaskId()); + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + false, null, "视频生成成功但未返回视频 URL"); + return; + } + + // 下载视频到本地 + Path localPath = fileDownloader.download(videoUrl, task.getConversationId(), task.getTaskId()); + String servingUrl = fileDownloader.toServingUrl(task.getConversationId(), localPath); + + // 保存 assistant 消息（含 video content part） + MessageContentPart videoPart = MessageContentPart.video(null, localPath.getFileName().toString()); + videoPart.setFileUrl(servingUrl); + videoPart.setContentType("video/mp4"); + + conversationService.saveMessage( + task.getConversationId(), "assistant", + "视频已生成完毕", + List.of(videoPart), "completed"); + + // SSE 广播 + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + true, servingUrl, null); + + log.info("[VideoGen] Task {} completed, video saved: {}", task.getTaskId(), servingUrl); + } catch (Exception e) { + log.error("[VideoGen] Completion handling failed for task {}: {}", + task.getTaskId(), e.getMessage(), e); + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + false, null, "视频下载或保存失败: " + e.getMessage()); + } + } else { + // 失败 + asyncTaskService.broadcastTaskEvent(task, "async_task_completed", + false, null, result.errorMessage()); + log.warn("[VideoGen] Task {} failed: {}", task.getTaskId(), result.errorMessage()); + } + } + + private VideoCapability inferMode(VideoGenerationRequest request) { + if (request.getVideoUrl() != null && !request.getVideoUrl().isBlank()) { + return VideoCapability.VIDEO_TO_VIDEO; + } + if (request.getImageUrl() != null && !request.getImageUrl().isBlank()) { + return VideoCapability.IMAGE_TO_VIDEO; + } + return VideoCapability.GENERATE; + } + + private void normalizeRequest(VideoGenerationRequest request) { + if (request.getAspectRatio() == null || request.getAspectRatio().isBlank()) { + request.setAspectRatio("16:9"); + } + if (request.getDurationSeconds() == null) { + request.setDurationSeconds(5); + } + } + + /** + * 根据 provider 能力做参数就近归一化（借鉴 OpenClaw 的 resolveVideoGenerationOverrides） + */ + private void normalizeForProvider(VideoGenerationRequest request, VideoGenerationProvider provider) { + VideoProviderCapabilities caps = provider.detailedCapabilities(); + if (caps == null) return; + + // aspectRatio 就近匹配 + request.setAspectRatio(caps.normalizeAspectRatio(request.getAspectRatio())); + + // duration 就近匹配 + if (request.getDurationSeconds() != null) { + request.setDurationSeconds(caps.normalizeDuration(request.getDurationSeconds())); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderCapabilities.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderCapabilities.java new file mode 100644 index 00000000..a9a909ae --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderCapabilities.java @@ -0,0 +1,80 @@ +package vip.mate.tool.video; + +import lombok.Builder; +import lombok.Data; + +import java.util.List; +import java.util.Set; + +/** + * 视频生成 Provider 细粒度能力声明 + *

+ * 参考 OpenClaw 的 VideoGenerationProviderCapabilities 设计， + * 每个 Provider 显式声明支持的参数范围，Runtime 据此做就近归一化。 + * + * @author MateClaw Team + */ +@Data +@Builder +public class VideoProviderCapabilities { + + /** 支持的生成模式 */ + @Builder.Default + private Set modes = Set.of(VideoCapability.GENERATE); + + /** 支持的画面比例 */ + @Builder.Default + private List aspectRatios = List.of("16:9", "9:16", "1:1"); + + /** 支持的时长（秒），如 [5, 10] 表示只支持 5 秒和 10 秒 */ + @Builder.Default + private List supportedDurations = List.of(5); + + /** 最大时长（秒） */ + @Builder.Default + private int maxDurationSeconds = 10; + + /** 最大并发视频数 */ + @Builder.Default + private int maxVideos = 1; + + /** 是否支持音频 */ + @Builder.Default + private boolean supportsAudio = false; + + /** 默认模型 */ + private String defaultModel; + + /** 可用模型列表 */ + @Builder.Default + private List models = List.of(); + + /** + * 将请求的 duration 就近匹配到 provider 支持的值 + */ + public int normalizeDuration(int requested) { + if (supportedDurations == null || supportedDurations.isEmpty()) { + return Math.min(requested, maxDurationSeconds); + } + int closest = supportedDurations.get(0); + int minDiff = Math.abs(requested - closest); + for (int d : supportedDurations) { + int diff = Math.abs(requested - d); + if (diff < minDiff) { + minDiff = diff; + closest = d; + } + } + return closest; + } + + /** + * 将请求的 aspectRatio 就近匹配或回退到默认 + */ + public String normalizeAspectRatio(String requested) { + if (aspectRatios.contains(requested)) { + return requested; + } + return aspectRatios.isEmpty() ? "16:9" : aspectRatios.get(0); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderRegistry.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderRegistry.java new file mode 100644 index 00000000..5f61c117 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoProviderRegistry.java @@ -0,0 +1,90 @@ +package vip.mate.tool.video; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 视频生成提供商注册表 — 收集所有 {@link VideoGenerationProvider} 实现，提供优先级排序与自动探测 + *

+ * 借鉴 {@link vip.mate.tool.search.SearchProviderRegistry} 的设计模式。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class VideoProviderRegistry { + + private final List sortedProviders; + private final Map providerMap; + + public VideoProviderRegistry(List providers) { + this.sortedProviders = providers.stream() + .sorted(Comparator.comparingInt(VideoGenerationProvider::autoDetectOrder)) + .toList(); + this.providerMap = providers.stream() + .collect(Collectors.toMap(VideoGenerationProvider::id, Function.identity())); + log.info("注册视频生成提供商 {} 个: {}", sortedProviders.size(), + sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList()); + } + + /** 按 ID 获取指定 provider */ + public VideoGenerationProvider getById(String id) { + return providerMap.get(id); + } + + /** 获取按 autoDetectOrder 排序的全部 provider 列表 */ + public List allSorted() { + return sortedProviders; + } + + /** + * 根据当前配置，解析应使用的 provider + * + * @param config 系统配置 + * @param requiredCapability 需要的能力（如 GENERATE、IMAGE_TO_VIDEO） + * @return 选中的 provider，或 null + */ + public ResolvedProvider resolve(SystemSettingsDTO config, VideoCapability requiredCapability) { + // 1. 用户显式配置的 primary provider + String configuredId = config.getVideoProvider(); + if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) { + VideoGenerationProvider configured = providerMap.get(configuredId); + if (configured != null && configured.isAvailable(config) + && configured.capabilities().contains(requiredCapability)) { + return new ResolvedProvider(configured, "configured"); + } + } + + // 2. 自动探测：按优先级遍历，找第一个可用且支持该能力的 + for (VideoGenerationProvider p : sortedProviders) { + if (p.isAvailable(config) && p.capabilities().contains(requiredCapability)) { + return new ResolvedProvider(p, "auto-detect"); + } + } + + return null; + } + + /** + * 获取所有可用于 fallback 的 provider（按优先级排序，排除 primary） + */ + public List fallbackCandidates(SystemSettingsDTO config, + VideoCapability requiredCapability, + String excludeId) { + return sortedProviders.stream() + .filter(p -> !p.id().equals(excludeId)) + .filter(p -> p.isAvailable(config)) + .filter(p -> p.capabilities().contains(requiredCapability)) + .toList(); + } + + public record ResolvedProvider(VideoGenerationProvider provider, String source) { + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoSubmitResult.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoSubmitResult.java new file mode 100644 index 00000000..19148be7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoSubmitResult.java @@ -0,0 +1,42 @@ +package vip.mate.tool.video; + +import lombok.Builder; +import lombok.Data; + +/** + * Provider 提交视频生成任务的结果 + * + * @author MateClaw Team + */ +@Data +@Builder +public class VideoSubmitResult { + + /** provider 返回的任务 ID */ + private String providerTaskId; + + /** 提交的 provider 名称 */ + private String providerName; + + /** 是否被接受 */ + private boolean accepted; + + /** 错误信息（仅 accepted=false 时） */ + private String errorMessage; + + public static VideoSubmitResult success(String providerTaskId, String providerName) { + return VideoSubmitResult.builder() + .providerTaskId(providerTaskId) + .providerName(providerName) + .accepted(true) + .build(); + } + + public static VideoSubmitResult failure(String providerName, String errorMessage) { + return VideoSubmitResult.builder() + .providerName(providerName) + .accepted(false) + .errorMessage(errorMessage) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/CogVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/CogVideoProvider.java new file mode 100644 index 00000000..d496977d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/CogVideoProvider.java @@ -0,0 +1,180 @@ +package vip.mate.tool.video.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.video.*; + +import java.util.List; +import java.util.Set; + +/** + * 智谱 CogVideoX Provider — 支持 CogVideoX-Flash (免费) 和 CogVideoX (高质量) + *

+ * API 文档: https://bigmodel.cn/dev/api/video-generation/cogvideox + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class CogVideoProvider implements VideoGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String DEFAULT_BASE_URL = "https://open.bigmodel.cn"; + private static final String DEFAULT_MODEL = "cogvideox-flash"; + + @Override + public String id() { + return "zhipu-cogvideo"; + } + + @Override + public String label() { + return "智谱 CogVideoX"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 200; + } + + @Override + public Set capabilities() { + return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO); + } + + @Override + public VideoProviderCapabilities detailedCapabilities() { + return VideoProviderCapabilities.builder() + .modes(capabilities()) + .aspectRatios(List.of("16:9", "9:16", "1:1")) + .supportedDurations(List.of(6)) + .maxDurationSeconds(6) + .defaultModel(DEFAULT_MODEL) + .models(List.of("cogvideox-flash", "cogvideox")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return StringUtils.hasText(config.getZhipuApiKey()); + } + + @Override + public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { + String apiKey = config.getZhipuApiKey(); + String baseUrl = StringUtils.hasText(config.getZhipuBaseUrl()) + ? config.getZhipuBaseUrl() : DEFAULT_BASE_URL; + + try { + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("prompt", request.getPrompt()); + + if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { + body.put("image_url", request.getImageUrl()); + } + + // 智谱视频 API 使用 size 参数 + if (request.getAspectRatio() != null) { + String size = aspectRatioToSize(request.getAspectRatio()); + if (size != null) { + body.put("size", size); + } + } + if (request.getDurationSeconds() != null) { + body.put("duration", request.getDurationSeconds()); + } + + HttpResponse response = HttpRequest.post(baseUrl + "/api/paas/v4/videos/generations") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("id")) { + String taskId = result.get("id").asText(); + log.info("[CogVideo] Submitted task: {} (model={})", taskId, model); + return VideoSubmitResult.success(taskId, id()); + } else { + String errMsg = result.has("error") + ? result.get("error").path("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[CogVideo] Submit failed: {}", errMsg); + return VideoSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[CogVideo] Submit error: {}", e.getMessage(), e); + return VideoSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + String apiKey = config.getZhipuApiKey(); + String baseUrl = StringUtils.hasText(config.getZhipuBaseUrl()) + ? config.getZhipuBaseUrl() : DEFAULT_BASE_URL; + + try { + HttpResponse response = HttpRequest.get( + baseUrl + "/api/paas/v4/videos/" + providerTaskId) + .header("Authorization", "Bearer " + apiKey) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + String taskStatus = result.path("task_status").asText(); + + return switch (taskStatus) { + case "SUCCESS" -> { + // 智谱返回 video_result_list 数组 + JsonNode videoResults = result.path("video_result_list"); + String videoUrl = null; + if (videoResults.isArray() && !videoResults.isEmpty()) { + videoUrl = videoResults.get(0).path("url").asText(null); + } + // 兼容 video_result 字段 + if (videoUrl == null) { + videoUrl = result.path("video_result").path("url").asText(null); + } + yield TaskPollResult.succeeded(videoUrl, null, result.toString()); + } + case "FAIL" -> TaskPollResult.failed( + result.has("error") ? result.get("error").path("message").asText() : "任务失败"); + case "PROCESSING" -> TaskPollResult.running(null); + default -> TaskPollResult.pending(null); + }; + } catch (Exception e) { + log.error("[CogVideo] Poll error for task {}: {}", providerTaskId, e.getMessage()); + return null; + } + } + + private String aspectRatioToSize(String aspectRatio) { + return switch (aspectRatio) { + case "16:9" -> "1920x1080"; + case "9:16" -> "1080x1920"; + case "1:1" -> "1080x1080"; + default -> null; + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java new file mode 100644 index 00000000..e71bb868 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java @@ -0,0 +1,222 @@ +package vip.mate.tool.video.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.video.*; + +import java.util.List; +import java.util.Set; + +/** + * DashScope 视频生成 Provider — 支持通义万相 Wan 2.5 / Wanx 2.1 + *

+ * 复用已有的 DashScope LLM provider 的 API Key。 + * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/video-generation + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DashScopeVideoProvider implements VideoGenerationProvider { + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1"; + private static final String DEFAULT_T2V_MODEL = "wan2.5-t2v-turbo"; + private static final String DEFAULT_I2V_MODEL = "wan2.5-i2v-turbo"; + + @Override + public String id() { + return "dashscope"; + } + + @Override + public String label() { + return "DashScope (通义万相)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 100; + } + + @Override + public Set capabilities() { + return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO); + } + + @Override + public VideoProviderCapabilities detailedCapabilities() { + return VideoProviderCapabilities.builder() + .modes(capabilities()) + .aspectRatios(List.of("16:9", "9:16", "1:1")) + .supportedDurations(List.of(5, 10)) + .maxDurationSeconds(10) + .defaultModel(DEFAULT_T2V_MODEL) + .models(List.of("wan2.5-t2v-turbo", "wan2.5-i2v-turbo", "wanx2.1-t2v-turbo", "wanx2.1-i2v-turbo")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + try { + return modelProviderService.isProviderConfigured("dashscope"); + } catch (Exception e) { + return false; + } + } + + @Override + public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { + String apiKey = getDashScopeApiKey(); + if (apiKey == null) { + return VideoSubmitResult.failure(id(), "DashScope API Key 未配置"); + } + + try { + String model = resolveModel(request); + ObjectNode body = buildRequestBody(request, model); + + HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/video-generation/generation") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .header("X-DashScope-Async", "enable") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("output")) { + String taskId = result.path("output").path("task_id").asText(); + log.info("[DashScope Video] Submitted task: {} (model={})", taskId, model); + return VideoSubmitResult.success(taskId, id()); + } else { + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[DashScope Video] Submit failed: {}", errMsg); + return VideoSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[DashScope Video] Submit error: {}", e.getMessage(), e); + return VideoSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + String apiKey = getDashScopeApiKey(); + if (apiKey == null) { + return TaskPollResult.failed("DashScope API Key 未配置"); + } + + try { + HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId) + .header("Authorization", "Bearer " + apiKey) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + JsonNode output = result.path("output"); + String taskStatus = output.path("task_status").asText(); + + return switch (taskStatus) { + case "SUCCEEDED" -> { + String videoUrl = extractVideoUrl(output); + yield TaskPollResult.succeeded(videoUrl, null, output.toString()); + } + case "FAILED" -> { + String errMsg = output.has("message") ? output.get("message").asText() : "任务失败"; + yield TaskPollResult.failed(errMsg); + } + case "RUNNING" -> TaskPollResult.running(null); + default -> TaskPollResult.pending(null); + }; + } catch (Exception e) { + log.error("[DashScope Video] Poll error for task {}: {}", providerTaskId, e.getMessage()); + return null; // 轮询异常不终止，等下次重试 + } + } + + // ==================== 内部方法 ==================== + + private String getDashScopeApiKey() { + try { + var providerEntity = modelProviderService.getProviderConfig("dashscope"); + return providerEntity.getApiKey(); + } catch (Exception e) { + return null; + } + } + + private String resolveModel(VideoGenerationRequest request) { + if (request.getModel() != null && !request.getModel().isBlank()) { + return request.getModel(); + } + return request.getMode() == VideoCapability.IMAGE_TO_VIDEO + ? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL; + } + + private ObjectNode buildRequestBody(VideoGenerationRequest request, String model) { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt()); + + if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { + input.put("img_url", request.getImageUrl()); + } + + ObjectNode parameters = body.putObject("parameters"); + if (request.getAspectRatio() != null) { + // DashScope 使用 size 参数，如 "1280*720" + String size = aspectRatioToSize(request.getAspectRatio()); + if (size != null) { + parameters.put("size", size); + } + } + if (request.getDurationSeconds() != null) { + parameters.put("duration", String.valueOf(request.getDurationSeconds())); + } + + return body; + } + + private String aspectRatioToSize(String aspectRatio) { + return switch (aspectRatio) { + case "16:9" -> "1280*720"; + case "9:16" -> "720*1280"; + case "1:1" -> "720*720"; + default -> null; + }; + } + + private String extractVideoUrl(JsonNode output) { + JsonNode results = output.path("results"); + if (results.isArray() && !results.isEmpty()) { + return results.get(0).path("url").asText(null); + } + // 有些模型返回 video_url + if (output.has("video_url")) { + return output.get("video_url").asText(); + } + return null; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/FalVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/FalVideoProvider.java new file mode 100644 index 00000000..0a28e403 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/FalVideoProvider.java @@ -0,0 +1,194 @@ +package vip.mate.tool.video.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.video.*; + +import java.util.List; +import java.util.Set; + +/** + * fal.ai 视频生成 Provider — 通过 fal.ai 中转访问 Kling、Runway、Luma、MiniMax 等模型 + *

+ * API 文档: https://fal.ai/docs + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class FalVideoProvider implements VideoGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://queue.fal.run"; + private static final String STATUS_URL = "https://queue.fal.run"; + private static final String DEFAULT_MODEL = "fal-ai/kling-video/v1.6/pro/text-to-video"; + + @Override + public String id() { + return "fal"; + } + + @Override + public String label() { + return "fal.ai (Kling/Runway/Luma)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 300; + } + + @Override + public Set capabilities() { + return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO); + } + + @Override + public VideoProviderCapabilities detailedCapabilities() { + return VideoProviderCapabilities.builder() + .modes(capabilities()) + .aspectRatios(List.of("16:9", "9:16", "1:1", "3:4", "4:3")) + .supportedDurations(List.of(5, 10)) + .maxDurationSeconds(10) + .defaultModel(DEFAULT_MODEL) + .models(List.of( + "fal-ai/kling-video/v1.6/pro/text-to-video", + "fal-ai/runway-gen3/turbo/text-to-video", + "fal-ai/luma-dream-machine", + "fal-ai/minimax/video-01-live")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return StringUtils.hasText(config.getFalApiKey()); + } + + @Override + public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { + String apiKey = config.getFalApiKey(); + + try { + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + ObjectNode body = objectMapper.createObjectNode(); + body.put("prompt", request.getPrompt()); + + if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { + body.put("image_url", request.getImageUrl()); + } + if (request.getAspectRatio() != null) { + body.put("aspect_ratio", request.getAspectRatio()); + } + if (request.getDurationSeconds() != null) { + body.put("duration", String.valueOf(request.getDurationSeconds())); + } + + HttpResponse response = HttpRequest.post(BASE_URL + "/" + model) + .header("Authorization", "Key " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (response.getStatus() == 200 && result.has("request_id")) { + String requestId = result.get("request_id").asText(); + // 把 model 编码到 taskId 中，轮询时需要 + String compositeId = model + "|" + requestId; + log.info("[fal.ai] Submitted task: {} (model={})", requestId, model); + return VideoSubmitResult.success(compositeId, id()); + } else { + String errMsg = result.has("detail") ? result.get("detail").asText() + : "HTTP " + response.getStatus(); + log.warn("[fal.ai] Submit failed: {}", errMsg); + return VideoSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[fal.ai] Submit error: {}", e.getMessage(), e); + return VideoSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + String apiKey = config.getFalApiKey(); + + // 从复合 ID 中解析 model 和 requestId + String[] parts = providerTaskId.split("\\|", 2); + if (parts.length != 2) { + return TaskPollResult.failed("无效的任务 ID 格式"); + } + String model = parts[0]; + String requestId = parts[1]; + + try { + HttpResponse response = HttpRequest.get( + STATUS_URL + "/" + model + "/requests/" + requestId + "/status") + .header("Authorization", "Key " + apiKey) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + String status = result.path("status").asText(); + + return switch (status) { + case "COMPLETED" -> { + // 完成后需要再获取结果 + HttpResponse resultResponse = HttpRequest.get( + STATUS_URL + "/" + model + "/requests/" + requestId) + .header("Authorization", "Key " + apiKey) + .timeout(15_000) + .execute(); + JsonNode resultBody = objectMapper.readTree(resultResponse.body()); + String videoUrl = extractVideoUrl(resultBody); + yield TaskPollResult.succeeded(videoUrl, null, resultBody.toString()); + } + case "FAILED" -> TaskPollResult.failed( + result.has("error") ? result.get("error").asText() : "任务失败"); + case "IN_PROGRESS" -> { + Integer progress = result.has("progress") + ? (int) (result.get("progress").asDouble() * 100) : null; + yield TaskPollResult.running(progress); + } + default -> TaskPollResult.pending(null); + }; + } catch (Exception e) { + log.error("[fal.ai] Poll error for task {}: {}", requestId, e.getMessage()); + return null; + } + } + + private String extractVideoUrl(JsonNode resultBody) { + // fal.ai 通常返回 video.url 或 video[0].url + JsonNode video = resultBody.path("video"); + if (video.has("url")) { + return video.get("url").asText(); + } + if (video.isArray() && !video.isEmpty()) { + return video.get(0).path("url").asText(null); + } + // 有些模型用 output.video + JsonNode output = resultBody.path("output"); + if (output.has("video")) { + return output.path("video").path("url").asText(null); + } + return null; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/KlingVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/KlingVideoProvider.java new file mode 100644 index 00000000..06565486 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/KlingVideoProvider.java @@ -0,0 +1,191 @@ +package vip.mate.tool.video.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.jsonwebtoken.Jwts; +import io.jsonwebtoken.security.Keys; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.task.AsyncTaskService.TaskPollResult; +import vip.mate.tool.video.*; + +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 快手可灵 Kling 视频生成 Provider + *

+ * API 文档: https://docs.qingque.cn/d/home/eZQBXy5cfgN4H-YSZjCE1c_5w + * 鉴权方式: JWT (access_key + secret_key 签发短时 token) + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class KlingVideoProvider implements VideoGenerationProvider { + + private final ObjectMapper objectMapper; + + private static final String BASE_URL = "https://api.klingai.com"; + private static final String DEFAULT_MODEL = "kling-v1.6-pro"; + + @Override + public String id() { + return "kling"; + } + + @Override + public String label() { + return "快手可灵 Kling"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 250; + } + + @Override + public Set capabilities() { + return Set.of(VideoCapability.GENERATE, VideoCapability.IMAGE_TO_VIDEO); + } + + @Override + public VideoProviderCapabilities detailedCapabilities() { + return VideoProviderCapabilities.builder() + .modes(capabilities()) + .aspectRatios(List.of("16:9", "9:16", "1:1")) + .supportedDurations(List.of(5, 10)) + .maxDurationSeconds(10) + .defaultModel(DEFAULT_MODEL) + .models(List.of("kling-v1.6-pro", "kling-v2.0-master")) + .build(); + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + return StringUtils.hasText(config.getKlingAccessKey()) + && StringUtils.hasText(config.getKlingSecretKey()); + } + + @Override + public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { + try { + String token = generateJwtToken(config.getKlingAccessKey(), config.getKlingSecretKey()); + String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; + + String endpoint = request.getMode() == VideoCapability.IMAGE_TO_VIDEO + ? "/v1/videos/image2video" : "/v1/videos/text2video"; + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model_name", model); + body.put("prompt", request.getPrompt()); + + if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { + body.put("image", request.getImageUrl()); + } + if (request.getAspectRatio() != null) { + body.put("aspect_ratio", request.getAspectRatio()); + } + if (request.getDurationSeconds() != null) { + body.put("duration", String.valueOf(request.getDurationSeconds())); + } + + HttpResponse response = HttpRequest.post(BASE_URL + endpoint) + .header("Authorization", "Bearer " + token) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + + if (result.path("code").asInt() == 0 && result.has("data")) { + String taskId = result.path("data").path("task_id").asText(); + log.info("[Kling] Submitted task: {} (model={})", taskId, model); + // 复合 ID 编码 endpoint 类型 + String compositeId = endpoint + "|" + taskId; + return VideoSubmitResult.success(compositeId, id()); + } else { + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[Kling] Submit failed: {}", errMsg); + return VideoSubmitResult.failure(id(), errMsg); + } + } catch (Exception e) { + log.error("[Kling] Submit error: {}", e.getMessage(), e); + return VideoSubmitResult.failure(id(), e.getMessage()); + } + } + + @Override + public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { + String[] parts = providerTaskId.split("\\|", 2); + if (parts.length != 2) { + return TaskPollResult.failed("无效的任务 ID 格式"); + } + String endpoint = parts[0]; + String taskId = parts[1]; + + try { + String token = generateJwtToken(config.getKlingAccessKey(), config.getKlingSecretKey()); + + HttpResponse response = HttpRequest.get(BASE_URL + endpoint + "/" + taskId) + .header("Authorization", "Bearer " + token) + .timeout(15_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + JsonNode data = result.path("data"); + String taskStatus = data.path("task_status").asText(); + + return switch (taskStatus) { + case "succeed" -> { + JsonNode works = data.path("task_result").path("videos"); + String videoUrl = null; + if (works.isArray() && !works.isEmpty()) { + videoUrl = works.get(0).path("url").asText(null); + } + yield TaskPollResult.succeeded(videoUrl, null, data.toString()); + } + case "failed" -> TaskPollResult.failed( + data.has("task_status_msg") ? data.get("task_status_msg").asText() : "任务失败"); + case "processing" -> TaskPollResult.running(null); + default -> TaskPollResult.pending(null); + }; + } catch (Exception e) { + log.error("[Kling] Poll error for task {}: {}", taskId, e.getMessage()); + return null; + } + } + + /** + * 使用 access_key + secret_key 签发短时 JWT token + */ + private String generateJwtToken(String accessKey, String secretKey) { + long now = System.currentTimeMillis(); + return Jwts.builder() + .header().add("alg", "HS256").add("typ", "JWT").and() + .claims(Map.of( + "iss", accessKey, + "exp", (now / 1000) + 1800, // 30 分钟过期 + "nbf", (now / 1000) - 5 + )) + .signWith(Keys.hmacShaKeyFor(secretKey.getBytes(StandardCharsets.UTF_8))) + .compact(); + } +} diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index 5a3091ee..dcb6720f 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -372,6 +372,11 @@ MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, KEY (id) VALUES (1000000014, 'DelegateAgentTool', 'Agent Delegation', 'Delegate tasks to other Agents for multi-agent collaboration. Call target Agent by name, run in isolated session and return result.', 'builtin', 'delegateAgentTool', '🤝', TRUE, TRUE, NOW(), NOW(), 0); +-- Built-in tool: Video Generation +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000015, 'VideoGenerateTool', 'Video Generation', 'Generate videos using AI. Supports text-to-video and image-to-video modes. Video generation is asynchronous and will appear in conversation when complete.', 'builtin', 'videoGenerateTool', '🎬', TRUE, TRUE, NOW(), NOW(), 0); + -- Example MCP Server: Filesystem (see MateClaw docs mcpServers.filesystem) MERGE INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index 671a72fb..6f690fd2 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -378,6 +378,11 @@ MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, KEY (id) VALUES (1000000014, 'DelegateAgentTool', 'Agent 委派', '委派任务给其他 Agent 执行，实现多 Agent 协作。支持按名称调用目标 Agent，在独立会话中运行并返回结果。', 'builtin', 'delegateAgentTool', '🤝', TRUE, TRUE, NOW(), NOW(), 0); +-- 内置工具：视频生成 +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000015, 'VideoGenerateTool', '视频生成', '使用 AI 生成视频，支持文字生成视频和图片生成视频两种模式。视频生成是异步过程，完成后自动显示在对话中。', 'builtin', 'videoGenerateTool', '🎬', TRUE, TRUE, NOW(), NOW(), 0); + -- 示例 MCP Server：Filesystem（参考 MateClaw 文档中的 mcpServers.filesystem） MERGE INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-server/src/main/resources/db/schema.sql b/mateclaw-server/src/main/resources/db/schema.sql index b9bd82e1..632cc9dd 100644 --- a/mateclaw-server/src/main/resources/db/schema.sql +++ b/mateclaw-server/src/main/resources/db/schema.sql @@ -446,3 +446,27 @@ ALTER TABLE mate_model_provider ADD COLUMN IF NOT EXISTS oauth_account_id VARCHA -- 清理 Codex 不支持的 ChatGPT OAuth 模型（gpt-4o, o3, o4-mini 在 Codex 模式下不可用） DELETE FROM mate_model_config WHERE provider = 'openai-chatgpt' AND model_name IN ('gpt-4o', 'o3', 'o4-mini'); + +-- ==================== 异步任务（视频/图片生成等长耗时操作） ==================== + +CREATE TABLE IF NOT EXISTS mate_async_task ( + id BIGINT NOT NULL PRIMARY KEY, + task_id VARCHAR(64) NOT NULL UNIQUE, + task_type VARCHAR(32) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'pending', + conversation_id VARCHAR(128), + message_id BIGINT, + provider_name VARCHAR(64), + provider_task_id VARCHAR(128), + request_json TEXT, + result_json TEXT, + error_message VARCHAR(512), + progress INT DEFAULT 0, + created_by VARCHAR(64), + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_async_task_taskid ON mate_async_task(task_id); +CREATE INDEX IF NOT EXISTS idx_async_task_conv ON mate_async_task(conversation_id); +CREATE INDEX IF NOT EXISTS idx_async_task_status ON mate_async_task(status); diff --git a/mateclaw-ui/src/composables/chat/useChat.ts b/mateclaw-ui/src/composables/chat/useChat.ts index 3cc35949..f9bd1193 100644 --- a/mateclaw-ui/src/composables/chat/useChat.ts +++ b/mateclaw-ui/src/composables/chat/useChat.ts @@ -618,6 +618,26 @@ export function useChat(options: UseChatOptions): UseChatReturn { phaseInfo.value = null }) + // ===== 异步任务完成事件（视频生成等） ===== + stream.on('async_task_completed', (data) => { + console.log('[useChat] Async task completed:', data) + if (data.success && data.videoUrl && streamConversationId) { + // 在消息列表中追加一条包含视频的 assistant 消息 + addMessage({ + role: 'assistant', + content: '', + contentParts: [{ + type: 'video', + fileUrl: data.videoUrl, + fileName: `video_${data.taskId}.mp4`, + contentType: 'video/mp4', + }] as MessageContentPart[], + status: 'completed', + conversationId: streamConversationId, + }) + } + }) + // ===== 发送消息（支持运行中继续发送） ===== const sendMessage = async (content: string, options: SendMessageOptions) => { diff --git a/mateclaw-ui/src/composables/chat/useStream.ts b/mateclaw-ui/src/composables/chat/useStream.ts index 83375303..685370e0 100644 --- a/mateclaw-ui/src/composables/chat/useStream.ts +++ b/mateclaw-ui/src/composables/chat/useStream.ts @@ -32,6 +32,9 @@ export type SSEEventType = | 'turn_interrupted' | 'queued_input_accepted' | 'queued_input_started' + // 异步任务事件 + | 'async_task_progress' + | 'async_task_completed' export interface SSEEvent { type: SSEEventType diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index fba3fedb..7c1e4452 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -204,6 +204,7 @@ export default { sections: { model: 'Model Management', system: 'System', + video: 'Video Generation', about: 'About', }, modelTitle: 'Model Management', @@ -332,6 +333,15 @@ export default { tavilyBaseUrl: 'Tavily Base URL', duckduckgoEnabled: 'DuckDuckGo (Keyless)', searxngBaseUrl: 'SearXNG Base URL', + videoEnabled: 'Enable Video Generation', + videoProvider: 'Preferred Video Provider', + videoFallbackEnabled: 'Provider Fallback', + dashscopeStatus: 'DashScope Status', + zhipuApiKey: 'Zhipu API Key', + zhipuBaseUrl: 'Zhipu API Base URL', + falApiKey: 'fal.ai API Key', + klingAccessKey: 'Kling Access Key', + klingSecretKey: 'Kling Secret Key', }, hints: { provider: 'Current implementation applies DashScope model options at runtime.', @@ -347,9 +357,28 @@ export default { tavilyBaseUrl: 'Usually no need to change unless using a custom proxy.', duckduckgoEnabled: 'Free keyless search fallback. No API key required. Enabled by default as a zero-config fallback.', searxngBaseUrl: 'Self-hosted SearXNG instance URL. Auto-configured when deploying via Docker.', + videoEnabled: 'Enable to let Agent use the video generation tool. Requires at least one configured video provider.', + videoProvider: 'Select preferred video provider. Auto mode picks the first available one.', + videoFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.', + dashscopeVideoStatus: 'Reuses the DashScope API Key configured in Model Management. No extra setup needed.', + zhipuApiKey: 'Get from bigmodel.cn. CogVideoX-Flash model is free.', + zhipuBaseUrl: 'Usually no need to change unless using a custom proxy.', + falApiKey: 'Get from fal.ai. One key accesses Kling, Runway, Luma and more.', + klingAccessKey: 'Get from Kuaishou Open Platform for Kling video generation API.', + klingSecretKey: 'Paired with Access Key for JWT authentication.', }, searchTitle: 'Search Service', searchDesc: 'Configure the built-in search tool provider and API credentials', + videoTitle: 'Video Generation', + videoDesc: 'Configure AI video generation capabilities, supporting text-to-video and image-to-video', + videoProviderOptions: { + auto: 'Auto Select', + }, + videoProviderTags: { + reuseLlmKey: 'Reuses LLM API Key', + freeQuota: 'Free Quota Available', + configuredInModels: 'See Model Settings', + }, actions: { setDefault: 'Set Default', saveSystem: 'Save System Settings', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 9bdfba35..161e8cc2 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -204,6 +204,7 @@ export default { sections: { model: '模型管理', system: '系统设置', + video: '视频生成', about: '关于', }, modelTitle: '模型管理', @@ -332,6 +333,16 @@ export default { tavilyBaseUrl: 'Tavily 接口地址', duckduckgoEnabled: 'DuckDuckGo（免 Key）', searxngBaseUrl: 'SearXNG 地址', + // 视频生成 + videoEnabled: '启用视频生成', + videoProvider: '首选视频 Provider', + videoFallbackEnabled: 'Provider 回退', + dashscopeStatus: 'DashScope 状态', + zhipuApiKey: '智谱 API Key', + zhipuBaseUrl: '智谱 API 地址', + falApiKey: 'fal.ai API Key', + klingAccessKey: '可灵 Access Key', + klingSecretKey: '可灵 Secret Key', }, hints: { provider: '当前版本会把 DashScope 模型参数真实应用到 Agent 调用链路。', @@ -347,9 +358,29 @@ export default { tavilyBaseUrl: '通常无需修改，除非使用自定义代理地址。', duckduckgoEnabled: '免费搜索兜底，无需 API Key。默认开启，作为零配置下的搜索降级方案。', searxngBaseUrl: '自部署 SearXNG 实例地址。Docker 部署时自动配置。', + // 视频生成 + videoEnabled: '开启后 Agent 可使用视频生成工具。需至少配置一个视频 Provider 的 API Key。', + videoProvider: '选择首选视频生成 Provider，auto 模式自动选择第一个可用的。', + videoFallbackEnabled: '首选 Provider 失败时自动尝试其他已配置的 Provider。', + dashscopeVideoStatus: '复用模型管理中配置的 DashScope API Key，无需额外配置。', + zhipuApiKey: '从 bigmodel.cn 获取，CogVideoX-Flash 模型免费。', + zhipuBaseUrl: '通常无需修改，除非使用自定义代理地址。', + falApiKey: '从 fal.ai 获取，一个 Key 可访问 Kling、Runway、Luma 等多个模型。', + klingAccessKey: '从快手开放平台获取，用于调用可灵视频生成 API。', + klingSecretKey: '与 Access Key 配对使用，用于 JWT 签名鉴权。', }, searchTitle: '搜索服务', searchDesc: '配置内置搜索工具的提供商与 API 凭证', + videoTitle: '视频生成', + videoDesc: '配置 AI 视频生成能力，支持文字生成视频和图片生成视频', + videoProviderOptions: { + auto: '自动选择', + }, + videoProviderTags: { + reuseLlmKey: '复用 LLM API Key', + freeQuota: '有免费额度', + configuredInModels: '前往模型设置查看', + }, actions: { setDefault: '设为默认', saveSystem: '保存系统设置', diff --git a/mateclaw-ui/src/router/index.ts b/mateclaw-ui/src/router/index.ts index 723a53c5..996b5efc 100644 --- a/mateclaw-ui/src/router/index.ts +++ b/mateclaw-ui/src/router/index.ts @@ -85,6 +85,12 @@ const router = createRouter({ component: () => import('@/views/Settings/System/index.vue'), meta: { title: 'Settings - System' }, }, + { + path: 'video', + name: 'SettingsVideo', + component: () => import('@/views/Settings/Video/index.vue'), + meta: { title: 'Settings - Video' }, + }, { path: 'about', name: 'SettingsAbout', diff --git a/mateclaw-ui/src/types/index.ts b/mateclaw-ui/src/types/index.ts index e7bd425a..46ddfc63 100644 --- a/mateclaw-ui/src/types/index.ts +++ b/mateclaw-ui/src/types/index.ts @@ -502,6 +502,19 @@ export interface SystemSettings { // Keyless 搜索 provider duckduckgoEnabled: boolean searxngBaseUrl: string + // 视频生成配置 + videoEnabled?: boolean + videoProvider?: string + videoFallbackEnabled?: boolean + zhipuApiKey?: string + zhipuBaseUrl?: string + zhipuApiKeyMasked?: string + falApiKey?: string + falApiKeyMasked?: string + klingAccessKey?: string + klingSecretKey?: string + klingAccessKeyMasked?: string + klingSecretKeyMasked?: string } export interface ProviderModelInfo { diff --git a/mateclaw-ui/src/views/Settings/Layout.vue b/mateclaw-ui/src/views/Settings/Layout.vue index e4ccc57b..3ceabeff 100644 --- a/mateclaw-ui/src/views/Settings/Layout.vue +++ b/mateclaw-ui/src/views/Settings/Layout.vue @@ -41,6 +41,12 @@ const sections = computed(() => [ label: t('settings.sections.system'), icon: '', }, + { + id: 'video', + path: '/settings/video', + label: t('settings.sections.video'), + icon: '', + }, { id: 'about', path: '/settings/about', diff --git a/mateclaw-ui/src/views/Settings/Video/index.vue b/mateclaw-ui/src/views/Settings/Video/index.vue new file mode 100644 index 00000000..d34fd682 --- /dev/null +++ b/mateclaw-ui/src/views/Settings/Video/index.vue @@ -0,0 +1,307 @@ + + + + +