From c2aecf18ef2a18b3882b17a9c94293383bdcc1b4 Mon Sep 17 00:00:00 2001 From: matevip Date: Sat, 9 May 2026 16:41:26 +0800 Subject: [PATCH] feat(agent,llm): multimodal sidecar routing for unsupported attachments (#87) --- .../vip/mate/agent/AgentGraphBuilder.java | 26 +++ .../main/java/vip/mate/agent/BaseAgent.java | 210 +++++++++++++++--- .../vip/mate/agent/GraphEventPublisher.java | 9 + .../agent/controller/AgentController.java | 60 +++++ .../agent/graph/StateGraphReActAgent.java | 21 +- .../plan/StateGraphPlanExecuteAgent.java | 13 +- .../agent/graph/state/MateClawStateKeys.java | 9 + .../mate/agent/vo/AgentCapabilitiesVO.java | 43 ++++ .../vip/mate/channel/web/ChatController.java | 18 ++ .../llm/controller/ModelConfigController.java | 8 +- .../mate/llm/routing/MediaCaptionService.java | 141 ++++++++++++ .../mate/llm/routing/MultimodalRouter.java | 166 ++++++++++++++ .../model/MultimodalRoutingDecision.java | 89 ++++++++ .../mate/llm/service/ModelConfigService.java | 34 ++- .../mate/system/model/SystemSettingsDTO.java | 18 ++ .../system/service/SystemSettingService.java | 26 +++ .../h2/V100__multimodal_default_models.sql | 16 ++ .../mysql/V100__multimodal_default_models.sql | 16 ++ 18 files changed, 882 insertions(+), 41 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 15be0dc1..749cfbce 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -133,6 +133,8 @@ public class AgentGraphBuilder { private final vip.mate.llm.failover.AvailableProviderPool providerPool; /** PR-0b: DashScope-specific construction lives here now; we only call into it for the search-on log. */ private final vip.mate.agent.chatmodel.AgentDashScopeChatModelBuilder dashScopeBuilder; + private final vip.mate.llm.routing.MultimodalRouter multimodalRouter; + private final vip.mate.llm.routing.MediaCaptionService mediaCaptionService; /** * Optional audit pipeline. Setter injection (rather than a constructor @@ -293,6 +295,11 @@ public class AgentGraphBuilder { agent.modelCapabilities = modelCapabilityService.resolve( runtimeModel.getModelName(), runtimeModel.getModalities()); agent.runtimeProviderId = provider != null ? provider.getProviderId() : ""; + agent.runtimeModelConfig = runtimeModel; + agent.toolSet = toolSet; + agent.multimodalRouter = multimodalRouter; + agent.mediaCaptionService = mediaCaptionService; + agent.userLocale = resolveLocale(); agent.temperature = runtimeModel.getTemperature(); agent.maxTokens = runtimeModel.getMaxTokens(); agent.maxInputTokens = runtimeModel.getMaxInputTokens(); @@ -450,6 +457,8 @@ public class AgentGraphBuilder { // 丢这个键，evidence_insufficient 检查会"静默地不生效" —— // StateKeyRegistrationCoverageTest 专门兜这条。 .addStrategy(MateClawStateKeys.SOURCE_EVIDENCE_LEDGER, KeyStrategy.REPLACE) + // Multimodal sidecar routing decision for the current turn. + .addStrategy(MateClawStateKeys.ROUTING_DECISION, KeyStrategy.REPLACE) .build(); // Graph 拓扑： @@ -636,6 +645,8 @@ public class AgentGraphBuilder { // 丢这个键，evidence_insufficient 检查会"静默地不生效" —— // StateKeyRegistrationCoverageTest 专门兜这条。 .addStrategy(MateClawStateKeys.SOURCE_EVIDENCE_LEDGER, KeyStrategy.REPLACE) + // Multimodal sidecar routing decision for the current turn. + .addStrategy(MateClawStateKeys.ROUTING_DECISION, KeyStrategy.REPLACE) .build(); StateGraph graph = new StateGraph("react-agent-v2", keyStrategyFactory) @@ -701,6 +712,21 @@ public class AgentGraphBuilder { return buildRuntimeChatModel(runtimeModel, this.retryTemplate); } + /** + * Resolve the user-facing locale used for sidecar caption prompts. + * Reads {@code language} from system settings; falls back to + * {@code zh-CN} so CN deployments stay consistent with the chat UI. + */ + private java.util.Locale resolveLocale() { + try { + String lang = systemSettingService.getLanguage(); + if (lang == null || lang.isBlank()) return java.util.Locale.SIMPLIFIED_CHINESE; + return java.util.Locale.forLanguageTag(lang); + } catch (Exception e) { + return java.util.Locale.SIMPLIFIED_CHINESE; + } + } + /** * 构建运行时 ChatModel，并指定自定义的 Spring AI {@link RetryTemplate}。 *

diff --git a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java index bcbe8d38..979cbe59 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java @@ -10,6 +10,10 @@ import org.springframework.core.io.FileSystemResource; import org.springframework.util.MimeType; import reactor.core.publisher.Flux; import vip.mate.approval.ApprovalPlaceholderUtil; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.MediaCaptionService; +import vip.mate.llm.routing.MultimodalRouter; +import vip.mate.llm.routing.model.MultimodalRoutingDecision; import vip.mate.llm.service.ModelCapabilityService; import vip.mate.workspace.conversation.ConversationService; import vip.mate.workspace.conversation.model.MessageContentPart; @@ -85,6 +89,31 @@ public abstract class BaseAgent { /** 构建时使用的 provider ID（运行时快照） */ protected String runtimeProviderId; + /** + * Full runtime model configuration used by the multimodal router to + * decide whether the primary model can handle attachments natively. + * Set by {@code AgentGraphBuilder} alongside {@link #modelCapabilities}. + */ + protected ModelConfigEntity runtimeModelConfig; + + /** + * The agent's effective tool set. Lifted from subclasses so + * {@link #buildUserMessage} can ask whether the agent has any media-capable + * tool when the primary model rejects an attachment. + */ + protected vip.mate.agent.AgentToolSet toolSet; + + /** + * Optional sidecar routing services. Null when not wired (e.g. tests with + * minimal builders); the routing path then degrades to the legacy + * skip-with-text-hint behavior without any extra LLM calls. + */ + protected MultimodalRouter multimodalRouter; + protected MediaCaptionService mediaCaptionService; + + /** Locale used when prompting the vision sidecar. Defaults to zh-CN when unset. */ + protected java.util.Locale userLocale = java.util.Locale.SIMPLIFIED_CHINESE; + protected BaseAgent(ChatClient chatClient, ConversationService conversationService) { this.chatClient = chatClient; @@ -510,34 +539,87 @@ public abstract class BaseAgent { } /** - * 构建 UserMessage，支持 multimodal：如果消息包含图片/视频附件，直接注入 Spring AI Media 对象， - * 让模型在 prompt 中直接看到媒体内容，不需要再调 MCP read_media_file 工具。 + * Build a {@link UserMessage} for the current turn, including any image/video + * media the agent's primary model can handle natively. Returns the message + * paired with the {@link MultimodalRoutingDecision} taken so the caller can + * persist it as message metadata and emit a routing event. */ - protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { - return buildUserMessage(message, renderedContent, true); + protected CurrentTurnUserMessage buildUserMessageForCurrentTurn(MessageEntity message, String renderedContent) { + return buildUserMessageInternal(message, renderedContent, true); } /** - * @param injectMedia when {@code false} (history replay), skip the Media-loading - * branch entirely and return text-only — providers like Zhipu - * GLM-5V cap at 1 video per request, so re-injecting historical - * attachments on every turn breaks the call. + * History-replay variant: text-only, no media reinjected, no routing decision. + * Many providers cap at one video per request, so re-injecting old attachments + * on every replay would break the call. */ + protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { + return buildUserMessageInternal(message, renderedContent, true).userMessage(); + } + protected UserMessage buildUserMessage(MessageEntity message, String renderedContent, boolean injectMedia) { + return buildUserMessageInternal(message, renderedContent, injectMedia).userMessage(); + } + + private CurrentTurnUserMessage buildUserMessageInternal(MessageEntity message, String renderedContent, boolean injectMedia) { if (!injectMedia) { - return new UserMessage(renderedContent == null ? "" : renderedContent); + return new CurrentTurnUserMessage( + new UserMessage(renderedContent == null ? "" : renderedContent), + null); } List parts = conversationService.parseMessageParts(message); + + // Sidecar routing — runs first so caption text gets folded into finalText + // before native media injection considers the same parts again. + MultimodalRoutingDecision decision = multimodalRouter != null + ? multimodalRouter.route(parts, runtimeModelConfig) + : MultimodalRoutingDecision.none(); + + StringBuilder textBuilder = new StringBuilder(renderedContent == null ? "" : renderedContent); + java.util.Set sidecarHandledIdentifiers = new java.util.HashSet<>(); + if (decision.strategy() == MultimodalRoutingDecision.Strategy.SIDECAR + && mediaCaptionService != null + && decision.sidecarModel() != null) { + for (MessageContentPart part : parts) { + if (part == null) continue; + String contentType = part.getContentType(); + boolean isImage = ("image".equals(part.getType()) || "file".equals(part.getType())) + && contentType != null && contentType.startsWith("image/") + && !contentType.contains("svg"); + if (!isImage) continue; + MediaCaptionService.CaptionResult result = mediaCaptionService.caption( + decision.sidecarModel(), part, userLocale); + if (result.isFailure()) { + log.warn("[{}] Sidecar caption failed for {}: {}", + agentName, part.getFileName(), result.failure().getMessage()); + textBuilder.append("\n\n[系统提示] 视觉模型未能解析附件 ") + .append(part.getFileName()) + .append("，请稍后重试或在「设置 → 模型」检查视觉模型配置。"); + continue; + } + textBuilder.append("\n\n[图片附件描述: ") + .append(part.getFileName() == null ? "image" : part.getFileName()) + .append("]\n") + .append(result.description()) + .append("\n[/图片附件描述]"); + String identifier = identifyPart(part); + if (identifier != null) sidecarHandledIdentifiers.add(identifier); + } + } + List mediaList = new ArrayList<>(); - // Reasons for attachments that the model cannot consume — surfaced to the agent - // via the user message text so it does not hallucinate a tool call to read them. - // See issue #44. List skippedAttachments = new ArrayList<>(); boolean videoSupported = modelSupportsVideo(); boolean visionSupported = modelSupportsVision(); for (MessageContentPart part : parts) { if (part == null) continue; + // Sidecar already produced text for this image; never inject the + // raw bytes — the primary model would receive them and try to + // process natively, defeating the cost-saving purpose. + String identifier = identifyPart(part); + if (identifier != null && sidecarHandledIdentifiers.contains(identifier)) continue; + String partType = part.getType(); String contentType = part.getContentType(); // image 类型的 part 可能没有精确 contentType，补全为 image/jpeg @@ -607,21 +689,88 @@ public abstract class BaseAgent { } } - String finalText = renderedContent; if (!skippedAttachments.isEmpty()) { - finalText = (renderedContent == null ? "" : renderedContent) - + "\n\n[系统提示] 以下附件未能传入当前模型：" + String.join("、", skippedAttachments) - + "。\n请用对话语言清晰、友好地告诉用户：当前模型无法处理这类附件，建议切换到具备相应能力的多模态模型（图片需视觉模型，视频需视频理解模型）后重新上传。" - + "不要调用任何工具（包括 ffmpeg、浏览器、文件读取等）尝试解析这些附件。"; + textBuilder.append("\n\n[系统提示] 以下附件未能传入当前模型：") + .append(String.join("、", skippedAttachments)) + .append("。"); + // Only suggest switching models when no media-capable tool is bound + // either. With a media tool the LLM may legitimately choose to + // delegate to the tool — never instruct it not to use tools. + if (!hasMediaCapableTools()) { + textBuilder.append("\n请用对话语言清晰、友好地告诉用户：当前模型无法处理这类附件，建议切换到具备相应能力的多模态模型，或在「设置 → 模型」中配置视觉/视频模型作为旁路。"); + } } - if (mediaList.isEmpty()) { - return new UserMessage(finalText); + String finalText = textBuilder.toString(); + UserMessage built = mediaList.isEmpty() + ? new UserMessage(finalText) + : UserMessage.builder().text(finalText).media(mediaList).build(); + return new CurrentTurnUserMessage(built, decision); + } + + /** + * Stable identifier for de-duplicating parts already handled by the sidecar + * pass. Falls back across {@code path → mediaId → fileName} since not every + * channel populates the same field. + */ + private static String identifyPart(MessageContentPart part) { + if (part == null) return null; + if (part.getPath() != null && !part.getPath().isBlank()) return "p:" + part.getPath(); + if (part.getMediaId() != null && !part.getMediaId().isBlank()) return "m:" + part.getMediaId(); + if (part.getFileName() != null && !part.getFileName().isBlank()) return "f:" + part.getFileName(); + return null; + } + + /** + * True if the agent has at least one tool whose name or description + * suggests it can read images / video / audio. The check is intentionally + * loose — false positives just mean the agent is allowed to attempt media + * processing on its own, which is the safer default. + */ + private static final Set MEDIA_TOOL_KEYWORDS = Set.of( + "image", "图片", "vision", "视觉", + "video", "视频", "ffmpeg", + "ocr", "caption", "media", "audio", "音频"); + + private boolean hasMediaCapableTools() { + if (toolSet == null) return false; + var callbacks = toolSet.callbacks(); + if (callbacks == null || callbacks.isEmpty()) return false; + return callbacks.stream().anyMatch(cb -> { + try { + String name = String.valueOf(cb.getToolDefinition().name()).toLowerCase(); + String desc = String.valueOf(cb.getToolDefinition().description()).toLowerCase(); + return MEDIA_TOOL_KEYWORDS.stream().anyMatch(k -> name.contains(k) || desc.contains(k)); + } catch (Exception e) { + return false; + } + }); + } + + /** + * Pair returned from the current-turn user message build path: the assembled + * {@link UserMessage} and the routing decision the caller should persist as + * {@code metadata.routing} and surface to the SSE consumer. + */ + public record CurrentTurnUserMessage(UserMessage userMessage, MultimodalRoutingDecision routingDecision) {} + + /** + * Extract a routing-decision payload from the graph input map (placed there + * by {@code buildInitialState}) and turn it into a startup + * {@link vip.mate.agent.AgentService.StreamDelta} the SSE accumulator can + * persist. Returns an empty Flux when no routing happened this turn so we + * don't emit zero-value events. + */ + @SuppressWarnings("unchecked") + public static reactor.core.publisher.Flux routingStartupDelta( + java.util.Map inputs) { + Object decision = inputs.get(vip.mate.agent.graph.state.MateClawStateKeys.ROUTING_DECISION); + if (decision instanceof java.util.Map map && !map.isEmpty()) { + return reactor.core.publisher.Flux.just(vip.mate.agent.AgentService.StreamDelta.event( + vip.mate.agent.GraphEventPublisher.EVENT_ROUTING_DECISION, + (java.util.Map) map)); } - return UserMessage.builder() - .text(finalText) - .media(mediaList) - .build(); + return reactor.core.publisher.Flux.empty(); } /** @@ -642,6 +791,17 @@ public abstract class BaseAgent { * @return 带图片 Media 的 UserMessage（如果有图片附件），否则纯文本 UserMessage */ protected UserMessage buildCurrentUserMessage(String conversationId, String userMessageText) { + return buildCurrentUserMessageWithRouting(conversationId, userMessageText).userMessage(); + } + + /** + * Same as {@link #buildCurrentUserMessage} but also returns the multimodal + * routing decision taken for this turn so the caller can persist it as + * {@code metadata.routing} and emit a SSE-side event for the chat UI. + * Returns a decision with NONE strategy when the message has no attachments + * the primary model can't already handle. + */ + protected CurrentTurnUserMessage buildCurrentUserMessageWithRouting(String conversationId, String userMessageText) { try { List history = conversationService.listMessages(conversationId); // 倒序取最后一条 user 消息（buildInitialState 在 saveMessage 后调用，所以最后一条就是当前消息） @@ -650,14 +810,14 @@ public abstract class BaseAgent { if ("user".equals(msg.getRole())) { // 用 DB 中的实际内容（可能包含 contentParts），不用传入的 text String content = conversationService.renderMessageContent(msg); - return buildUserMessage(msg, content != null && !content.isBlank() ? content : userMessageText); + return buildUserMessageForCurrentTurn(msg, content != null && !content.isBlank() ? content : userMessageText); } } } catch (Exception e) { log.debug("[{}] Failed to load current user message parts for multimodal: {}", agentName, e.getMessage()); } - return new UserMessage(userMessageText); + return new CurrentTurnUserMessage(new UserMessage(userMessageText), null); } protected Path resolveImagePath(String relativePath) { diff --git a/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java b/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java index 278d747b..e63d8865 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java @@ -45,6 +45,15 @@ public final class GraphEventPublisher { */ public static final String EVENT_FINISH_REASON = "finish_reason"; + /** + * Multimodal sidecar routing decision for the current turn. Emitted once + * per turn before the graph starts streaming; the channel-side accumulator + * stores it under {@code metadata.routing} so the chat UI can show which + * sidecar (if any) was invoked. Underscore-prefixed name keeps it out of + * IM channel rebroadcast (see {@code ChannelMessageRouter}). + */ + public static final String EVENT_ROUTING_DECISION = "_routing_decision"; + /** * 事件记录 */ diff --git a/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java b/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java index 1526041b..9f04b02f 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java @@ -11,7 +11,13 @@ import vip.mate.channel.web.Utf8SseEmitter; import vip.mate.agent.AgentService; import vip.mate.agent.AgentState; import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.vo.AgentCapabilitiesVO; import vip.mate.audit.service.AuditEventService; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; import vip.mate.auth.model.UserEntity; import vip.mate.auth.service.AuthService; import vip.mate.common.result.R; @@ -40,6 +46,9 @@ public class AgentController { private final AuditEventService auditEventService; private final AuthService authService; private final WorkspaceService workspaceService; + private final ModelConfigService modelConfigService; + private final ModelCapabilityService modelCapabilityService; + private final SystemSettingService systemSettingService; private final ExecutorService sseExecutor = Executors.newCachedThreadPool(); @Operation(summary = "获取Agent列表") @@ -62,6 +71,57 @@ public class AgentController { return R.ok(agent); } + @Operation(summary = "获取Agent当前能力（modality 集合 + sidecar 配置），用于聊天页提示条") + @GetMapping("/{id}/capabilities") + @RequireWorkspaceRole("viewer") + public R capabilities( + @PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + AgentEntity agent = agentService.getAgent(id); + verifyResourceWorkspace(agent.getWorkspaceId(), workspaceId); + + ModelConfigEntity primary; + try { + primary = modelConfigService.resolveModel(agent.getModelName()); + } catch (Exception e) { + // No default model configured yet — return a capabilities snapshot that + // tells the UI "we can't say anything about this agent's modalities". + return R.ok(AgentCapabilitiesVO.builder() + .agentId(id) + .modelName("") + .providerId("") + .modalities(List.of()) + .build()); + } + java.util.Set modalities = + modelCapabilityService.resolve(primary.getModelName(), primary.getModalities()); + + SystemSettingsDTO settings = systemSettingService.getSettings(); + Long visionId = settings.getDefaultVisionModelId(); + Long videoId = settings.getDefaultVideoModelId(); + + return R.ok(AgentCapabilitiesVO.builder() + .agentId(id) + .modelName(primary.getModelName()) + .providerId(primary.getProvider()) + .modalities(modalities.stream().map(Enum::name).toList()) + .defaultVisionModelId(visionId) + .defaultVisionModelLabel(resolveSidecarLabel(visionId)) + .defaultVideoModelId(videoId) + .defaultVideoModelLabel(resolveSidecarLabel(videoId)) + .build()); + } + + private String resolveSidecarLabel(Long modelId) { + if (modelId == null) return null; + try { + ModelConfigEntity m = modelConfigService.getModel(modelId); + return m == null ? null : m.getProvider() + " / " + m.getModelName(); + } catch (Exception e) { + return null; + } + } + @Operation(summary = "创建Agent") @PostMapping @RequireWorkspaceRole("member") diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java index aa040600..6d1b5913 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java @@ -198,7 +198,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC AtomicInteger lastSoftCap = new AtomicInteger(0); AtomicBoolean sawLegitimateExit = new AtomicBoolean(false); - return compiledGraph.stream(inputs, config) + return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config) .flatMapIterable(output -> { List deltas = new ArrayList<>(); List allEvents = GraphEventPublisher.extractEvents(output); @@ -263,7 +263,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC )); } return null; - }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())) + }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))) .doOnComplete(() -> { setState(AgentState.IDLE); if (!sawLegitimateExit.get()) { @@ -326,7 +326,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC AtomicInteger lastSoftCap = new AtomicInteger(0); AtomicBoolean sawLegitimateExit = new AtomicBoolean(false); - return compiledGraph.stream(inputs, config) + return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config) .flatMapIterable(output -> { List deltas = new ArrayList<>(); // 1. 提取所有累积的事件，只发送新增部分 @@ -402,7 +402,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC )); } return null; - }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())) + }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))) .doOnComplete(() -> { setState(AgentState.IDLE); if (!sawLegitimateExit.get()) { @@ -444,7 +444,9 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC List messages = new ArrayList<>(historyMessages); // 构建当前用户消息：支持 multimodal（如果有图片附件，直接注入 Media） - messages.add(buildCurrentUserMessage(conversationId, userMessage)); + // 同步获取 routing decision，写入 state 供后续节点 / accumulator 读取。 + BaseAgent.CurrentTurnUserMessage currentTurn = buildCurrentUserMessageWithRouting(conversationId, userMessage); + messages.add(currentTurn.userMessage()); Map inputs = new HashMap<>(); // 输入 @@ -482,6 +484,15 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC inputs.put(RUNTIME_PROVIDER_ID, runtimeProviderId != null ? runtimeProviderId : ""); inputs.put(TRACE_ID, UUID.randomUUID().toString().substring(0, 8)); + // Multimodal sidecar routing — null when the turn carries no media or + // the primary model already covers the modalities. Stored as a Map so + // graph state stays JSON-friendly. + if (currentTurn.routingDecision() != null + && currentTurn.routingDecision().strategy() != vip.mate.llm.routing.model.MultimodalRoutingDecision.Strategy.NONE + || (currentTurn.routingDecision() != null && !currentTurn.routingDecision().skipped().isEmpty())) { + inputs.put(MateClawStateKeys.ROUTING_DECISION, currentTurn.routingDecision().toMap()); + } + // RFC-063r §2.5: enrich the originating ChatOrigin with this agent's id // and workspace, then write it into graph state so ActionNode + // StepExecutionNode can forward it to ToolExecutionExecutor → ToolContext. diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java index 39cd4ecf..138e3257 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java @@ -148,7 +148,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS AtomicReference lastPersistedStepResult = new AtomicReference<>(""); AtomicReference lastPersistedStepThinking = new AtomicReference<>(""); - return compiledGraph.stream(inputs, config) + return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config) .flatMapIterable(output -> { List deltas = new ArrayList<>(); // 1. 提取事件（只发送新增部分） @@ -218,7 +218,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS )); } return null; - }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())) + }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))) .doOnComplete(() -> setState(AgentState.IDLE)) .doOnError(e -> { log.error("[{}] Plan-Execute stream error: {}", agentName, e.getMessage()); @@ -271,7 +271,8 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS } List messages = new ArrayList<>(historyMessages); - messages.add(buildCurrentUserMessage(conversationId, userMessage)); + BaseAgent.CurrentTurnUserMessage currentTurn = buildCurrentUserMessageWithRouting(conversationId, userMessage); + messages.add(currentTurn.userMessage()); // 构建 working context：对历史消息做受控长度摘要 String workingContext = buildWorkingContext(historyMessages, List.of()); @@ -299,6 +300,12 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS inputs.put(MateClawStateKeys.RUNTIME_PROVIDER_ID, runtimeProviderId != null ? runtimeProviderId : ""); inputs.put(MateClawStateKeys.TRACE_ID, UUID.randomUUID().toString().substring(0, 8)); + if (currentTurn.routingDecision() != null + && (currentTurn.routingDecision().strategy() != vip.mate.llm.routing.model.MultimodalRoutingDecision.Strategy.NONE + || !currentTurn.routingDecision().skipped().isEmpty())) { + inputs.put(MateClawStateKeys.ROUTING_DECISION, currentTurn.routingDecision().toMap()); + } + // RFC-063r §2.5: same as ReAct path — enrich and store the ChatOrigin // so StepExecutionNode (and any sub-graphs spawned via DelegateAgentTool) // can read it back from state. diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java index d75f3dad..30e44678 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java @@ -83,6 +83,15 @@ public final class MateClawStateKeys { // ===== 事件流（APPEND 策略）===== public static final String PENDING_EVENTS = "pending_events"; + /** + * Multimodal routing decision for the current turn (REPLACE strategy). + * Stored as a Map ready for JSON serialization. Set by BaseAgent before + * the reasoning node runs; read back by FinalAnswerNode and (separately) + * emitted as a graph event for the SSE accumulator to write into the + * persisted message metadata under {@code metadata.routing}. + */ + public static final String ROUTING_DECISION = "routing_decision"; + // ===== 阶段标记（REPLACE 策略）===== public static final String CURRENT_PHASE = "current_phase"; diff --git a/mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java b/mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java new file mode 100644 index 00000000..5bae044e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java @@ -0,0 +1,43 @@ +package vip.mate.agent.vo; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * Lightweight capability snapshot for an agent — answers two questions + * the chat console needs synchronously while the user is composing a message: + * + *

+ * + *

Returned by {@code GET /api/v1/agents/{id}/capabilities}. Computed on + * each request — cheap because everything is cached service-side and we only + * read at most three rows. Not persisted on {@code mate_agent}; sidecar + * configuration is system-wide and {@code modelCapabilities} is derived from + * {@code mate_model_config.modalities}. + */ +@Data +@Builder +@AllArgsConstructor +public class AgentCapabilitiesVO { + private Long agentId; + private String modelName; + private String providerId; + /** Resolved modality set: any of {@code TEXT / VISION / VIDEO / AUDIO}. */ + private List modalities; + /** System-level vision sidecar model id, null when not configured. */ + private Long defaultVisionModelId; + /** Display name of the configured vision sidecar (provider/modelName), null when not configured. */ + private String defaultVisionModelLabel; + /** System-level video sidecar model id (reserved in v1; never wired). */ + private Long defaultVideoModelId; + private String defaultVideoModelLabel; +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java index 105fef0f..6a122a44 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java @@ -1650,6 +1650,14 @@ public class ChatController { private List planSteps = List.of(); private Integer currentPlanStep = null; private Map pendingApproval = null; + /** + * Multimodal sidecar routing decision for this turn (null when no + * routing happened). Captured from the {@code _routing_decision} + * event emitted before the graph stream and folded into + * {@code metadata.routing} on persistence so the chat UI can show + * which sidecar (if any) was invoked. + */ + private Map routingDecision = null; synchronized void accept(AgentService.StreamDelta delta, String conversationId) { if (delta == null) return; @@ -1683,6 +1691,13 @@ public class ChatController { finishReason = String.valueOf(reason); } } + if (vip.mate.agent.GraphEventPublisher.EVENT_ROUTING_DECISION.equals(delta.eventType())) { + // Captured at turn start; persisted under metadata.routing so the + // chat UI can render which sidecar (if any) was invoked. Internal + // event — return early to skip rebroadcast on IM channels. + routingDecision = delta.eventData(); + return; + } accumulateToolEvent(delta.eventType(), delta.eventData(), conversationId); try { broadcastEvent(conversationId, delta.eventType(), delta.eventData()); @@ -1967,6 +1982,9 @@ public class ChatController { // brittle text matching on the assistant content. metadata.put("finishReason", finishReason); } + if (routingDecision != null && !routingDecision.isEmpty()) { + metadata.put("routing", routingDecision); + } return objectMapper.writeValueAsString(metadata); } catch (Exception e) { log.warn("Failed to serialize metadata: {}", e.getMessage()); diff --git a/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java b/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java index 5384bc9c..960d5238 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java @@ -204,10 +204,12 @@ public class ModelConfigController { // ==================== Embedding 模型管理 ==================== - @Operation(summary = "按类型筛选模型（chat / embedding）") + @Operation(summary = "按类型筛选模型（chat / embedding），可选 modality 过滤") @GetMapping("/by-type") - public R> listByType(@RequestParam(defaultValue = "chat") String modelType) { - return R.ok(modelConfigService.listByType(modelType)); + public R> listByType( + @RequestParam(defaultValue = "chat") String modelType, + @RequestParam(required = false) String modality) { + return R.ok(modelConfigService.listByType(modelType, modality)); } @Operation(summary = "测试 Embedding 模型连通性（嵌入一个短文本验证 API key）") diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java new file mode 100644 index 00000000..6521009d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java @@ -0,0 +1,141 @@ +package vip.mate.llm.routing; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.content.Media; +import org.springframework.core.io.FileSystemResource; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Service; +import org.springframework.util.MimeType; +import vip.mate.llm.chatmodel.ProviderChatModelFactory; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Locale; + +/** + * Caption an image attachment using a configured vision model so a text-only + * primary model can still reason about it. + * + *

The service is the execution arm of the sidecar strategy chosen by + * {@link MultimodalRouter}: pick a vision-capable model, send a single + * structured prompt with the image attached, return the description text. + * + *

v1 has no caching layer — every call hits the vision model. The cache + * (keyed by {@code sha256(file_bytes) + visionModelId + locale}) is reserved + * for the next iteration; the API shape exposes {@code cacheHit} so callers + * already record the field in routing metadata. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MediaCaptionService { + + private final ProviderChatModelFactory chatModelFactory; + private final RetryTemplate retryTemplate; + + public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale) { + if (visionModel == null || imagePart == null) { + return CaptionResult.failure(0, new IllegalArgumentException("vision model or image part is null")); + } + Path mediaPath = resolveMediaPath(imagePart); + if (mediaPath == null) { + return CaptionResult.failure(0, new IllegalStateException( + "Image file not found for attachment: " + imagePart.getFileName())); + } + String contentType = imagePart.getContentType(); + if (contentType == null || "image/*".equals(contentType)) { + contentType = "image/jpeg"; + } + long start = System.currentTimeMillis(); + try { + ChatModel chatModel = chatModelFactory.buildFor(visionModel, retryTemplate); + ChatClient client = ChatClient.create(chatModel); + UserMessage userMessage = UserMessage.builder() + .text(buildPrompt(locale, imagePart.getFileName())) + .media(List.of(new Media(MimeType.valueOf(contentType), new FileSystemResource(mediaPath)))) + .build(); + String description = client.prompt() + .messages(userMessage) + .call() + .content(); + long elapsed = System.currentTimeMillis() - start; + String trimmed = description == null ? "" : description.trim(); + if (trimmed.isEmpty()) { + return CaptionResult.failure(elapsed, + new IllegalStateException("Vision model returned empty description")); + } + return CaptionResult.success(trimmed, elapsed, false); + } catch (Exception e) { + long elapsed = System.currentTimeMillis() - start; + log.warn("Caption call failed for {} via {}/{}: {}", + imagePart.getFileName(), visionModel.getProvider(), visionModel.getModelName(), + e.getMessage()); + return CaptionResult.failure(elapsed, e); + } + } + + /** + * Locale-aware prompt. Defaults to Chinese when the locale is null or unrecognized + * (matches the primary user base) but switches to English so vision-model output + * matches the chat language and avoids polluting English-only contexts. + */ + private String buildPrompt(Locale locale, String fileName) { + boolean english = locale != null && Locale.ENGLISH.getLanguage().equalsIgnoreCase(locale.getLanguage()); + String fileHint = (fileName == null || fileName.isBlank()) ? "" : " (" + fileName + ")"; + if (english) { + return "Describe this image" + fileHint + + " concisely: list the main objects, scene, any visible text (OCR), " + + "and notable actions or emotions. Keep the answer under 300 words. " + + "Reply with the description only — no preamble."; + } + return "请用一段简洁的中文描述这张图片" + fileHint + + "：列出主要物体、场景、画面中可见的文字（OCR）、以及人物动作或情绪。" + + "不超过 300 字。直接给出描述，不要寒暄。"; + } + + /** + * Mirrors {@code BaseAgent.resolveImagePath} but standalone — caption service + * is reused outside the agent context (e.g. tests, future preflight endpoint). + */ + private Path resolveMediaPath(MessageContentPart part) { + Path resolved = tryResolve(part.getPath()); + if (resolved != null) return resolved; + return tryResolve(part.getMediaId()); + } + + private Path tryResolve(String relativePath) { + if (relativePath == null || relativePath.isBlank()) return null; + Path path = Paths.get(relativePath); + if (path.isAbsolute() && Files.exists(path)) return path; + Path workdir = Paths.get(System.getProperty("user.dir")).resolve(relativePath); + if (Files.exists(workdir)) return workdir; + return null; + } + + public record CaptionResult( + String description, + boolean cacheHit, + long elapsedMs, + Throwable failure + ) { + public boolean isFailure() { + return failure != null; + } + + public static CaptionResult success(String description, long elapsedMs, boolean cacheHit) { + return new CaptionResult(description, cacheHit, elapsedMs, null); + } + + public static CaptionResult failure(long elapsedMs, Throwable failure) { + return new CaptionResult(null, false, elapsedMs, failure); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java new file mode 100644 index 00000000..9a51d46c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java @@ -0,0 +1,166 @@ +package vip.mate.llm.routing; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.model.MultimodalRoutingDecision; +import vip.mate.llm.routing.model.MultimodalRoutingDecision.SkippedAttachment; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.llm.service.ModelCapabilityService.Modality; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * Decides how to handle attachments whose modality outruns the agent's primary model. + * + *

The router is a pure decision step: it inspects the parts list and the primary + * model's capability set, then returns a {@link MultimodalRoutingDecision}. Caller is + * responsible for executing the decision (e.g. invoking the caption service when + * strategy is SIDECAR). + * + *

v1 only supports image sidecar. Video attachments fall through to the NONE + * branch with an explanatory skip reason — the next iteration will add a video + * captioning path once a strategy for frame sampling is in place. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MultimodalRouter { + + private final SystemSettingService systemSettingService; + private final ModelConfigService modelConfigService; + private final ModelCapabilityService capabilityService; + + public MultimodalRoutingDecision route(List parts, ModelConfigEntity primary) { + Set required = collectRequiredModalities(parts); + if (required.isEmpty()) return MultimodalRoutingDecision.none(); + + EnumSet primaryCaps = primary == null + ? EnumSet.noneOf(Modality.class) + : capabilityService.resolve(primary.getModelName(), primary.getModalities()); + if (primaryCaps.containsAll(required)) return MultimodalRoutingDecision.none(); + + EnumSet missing = EnumSet.copyOf(required); + missing.removeAll(primaryCaps); + + List skipped = new ArrayList<>(); + ModelConfigEntity sidecarModel = null; + + // VISION sidecar: resolve configured default vision model. + if (missing.contains(Modality.VISION)) { + ModelConfigEntity candidate = resolveSidecar(Modality.VISION); + if (candidate != null) { + sidecarModel = candidate; + } else { + String reason = describeMissingSidecar(Modality.VISION); + for (MessageContentPart p : imageParts(parts)) { + skipped.add(new SkippedAttachment("image", p.getFileName(), reason)); + } + } + } + + // VIDEO: v1 has no sidecar implementation. Mark as skipped so the UI can + // tell the user to switch to a video-capable primary model. Reserved for + // a follow-up RFC. + if (missing.contains(Modality.VIDEO)) { + for (MessageContentPart p : videoParts(parts)) { + skipped.add(new SkippedAttachment("video", p.getFileName(), + "video_sidecar_not_supported_in_v1")); + } + } + + if (sidecarModel != null) { + return MultimodalRoutingDecision.sidecar(sidecarModel, required, missing); + } + return MultimodalRoutingDecision.noneWithSkipped(required, missing, skipped); + } + + private Set collectRequiredModalities(List parts) { + if (parts == null || parts.isEmpty()) return Set.of(); + EnumSet required = EnumSet.noneOf(Modality.class); + for (MessageContentPart part : parts) { + if (part == null) continue; + String type = part.getType(); + String contentType = part.getContentType(); + if (isImagePart(type, contentType)) required.add(Modality.VISION); + else if (isVideoPart(type, contentType)) required.add(Modality.VIDEO); + else if (isAudioPart(type, contentType)) required.add(Modality.AUDIO); + } + return required; + } + + private boolean isImagePart(String type, String contentType) { + if ("image".equals(type)) return true; + return "file".equals(type) && contentType != null && contentType.startsWith("image/"); + } + + private boolean isVideoPart(String type, String contentType) { + if ("video".equals(type)) return true; + return "file".equals(type) && contentType != null && contentType.startsWith("video/"); + } + + private boolean isAudioPart(String type, String contentType) { + if ("audio".equals(type)) return true; + return "file".equals(type) && contentType != null && contentType.startsWith("audio/"); + } + + private List imageParts(List parts) { + return parts.stream() + .filter(p -> p != null && isImagePart(p.getType(), p.getContentType())) + .toList(); + } + + private List videoParts(List parts) { + return parts.stream() + .filter(p -> p != null && isVideoPart(p.getType(), p.getContentType())) + .toList(); + } + + /** + * Resolve the configured sidecar model for a modality. Returns null when: + * - the setting is empty / blank; + * - the referenced row no longer exists or has been disabled; + * - the row's resolved capability set does not actually contain the modality. + * The caller treats null as "ask the user to configure one." + */ + private ModelConfigEntity resolveSidecar(Modality modality) { + SystemSettingsDTO settings = systemSettingService.getSettings(); + Long modelId = switch (modality) { + case VISION -> settings.getDefaultVisionModelId(); + case VIDEO -> settings.getDefaultVideoModelId(); + default -> null; + }; + if (modelId == null) return null; + ModelConfigEntity model; + try { + model = modelConfigService.getModel(modelId); + } catch (Exception e) { + log.debug("Configured sidecar model id={} could not be loaded: {}", modelId, e.getMessage()); + return null; + } + if (model == null || !Boolean.TRUE.equals(model.getEnabled())) return null; + if (!capabilityService.supports(model.getModelName(), model.getModalities(), modality)) { + log.warn("Configured sidecar model {}/{} does not actually support {} — ignoring", + model.getProvider(), model.getModelName(), modality); + return null; + } + return model; + } + + private String describeMissingSidecar(Modality modality) { + SystemSettingsDTO settings = systemSettingService.getSettings(); + Long configured = modality == Modality.VISION + ? settings.getDefaultVisionModelId() + : settings.getDefaultVideoModelId(); + if (configured == null) return modality.name().toLowerCase() + "_model_not_configured"; + return modality.name().toLowerCase() + "_model_unavailable"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java new file mode 100644 index 00000000..ef3c6691 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java @@ -0,0 +1,89 @@ +package vip.mate.llm.routing.model; + +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelCapabilityService.Modality; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Outcome of routing a single user turn that may carry image / video / audio attachments. + * + *

The decision is purely descriptive — execution (loading captions, mutating + * the user message) happens in the caller. Treat instances as immutable; the + * static factories cover the only valid shapes. + */ +public record MultimodalRoutingDecision( + Strategy strategy, + ModelConfigEntity sidecarModel, + Set requiredModalities, + Set primaryMissing, + List skipped +) { + + public enum Strategy { + /** Primary model handles the turn directly (or no attachments at all). */ + NONE, + /** A separate vision/video model captions attachments; primary stays. */ + SIDECAR, + /** Reserved: switch the whole turn to a multimodal model. v1 does not emit. */ + NATIVE + } + + public record SkippedAttachment(String type, String fileName, String reason) {} + + public static MultimodalRoutingDecision none() { + return new MultimodalRoutingDecision( + Strategy.NONE, null, Set.of(), Set.of(), List.of()); + } + + public static MultimodalRoutingDecision noneWithSkipped( + Set required, + Set missing, + List skipped) { + return new MultimodalRoutingDecision( + Strategy.NONE, null, required, missing, skipped); + } + + public static MultimodalRoutingDecision sidecar( + ModelConfigEntity sidecarModel, + Set required, + Set missing) { + return new MultimodalRoutingDecision( + Strategy.SIDECAR, sidecarModel, required, missing, List.of()); + } + + /** + * Serialize to a flat map for emission as a graph event payload. + * Only includes keys present in this decision so the resulting + * {@code metadata.routing} JSON stays compact for the chat UI. + */ + public Map toMap() { + Map m = new LinkedHashMap<>(); + m.put("strategy", strategy.name().toLowerCase()); + if (sidecarModel != null) { + m.put("sidecarModelId", sidecarModel.getId()); + m.put("sidecarModel", sidecarModel.getModelName()); + m.put("sidecarProvider", sidecarModel.getProvider()); + } + if (!requiredModalities.isEmpty()) { + m.put("requiredModalities", requiredModalities.stream().map(Enum::name).toList()); + } + if (!primaryMissing.isEmpty()) { + m.put("primaryMissing", primaryMissing.stream().map(Enum::name).toList()); + } + if (!skipped.isEmpty()) { + m.put("skipped", skipped.stream().map(s -> { + Map entry = new LinkedHashMap<>(); + entry.put("type", s.type()); + if (s.fileName() != null) entry.put("fileName", s.fileName()); + entry.put("reason", s.reason()); + return entry; + }).toList()); + } + return Collections.unmodifiableMap(m); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java index 1759b5c3..66401c14 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java @@ -23,6 +23,7 @@ public class ModelConfigService { private final ModelConfigMapper modelConfigMapper; private final ApplicationEventPublisher eventPublisher; + private final ModelCapabilityService modelCapabilityService; /** * Lazy to break circular dependency: ModelProviderService → ModelConfigService. @@ -67,17 +68,40 @@ public class ModelConfigService { * */ public List listByType(String modelType) { + return listByType(modelType, null); + } + + /** + * Optional modality filter (case-insensitive: {@code "vision" / "video" / "audio"}). + * When non-null, only enabled rows whose resolved capability set contains the + * requested modality survive — used by the multimodal sidecar settings UI to + * populate "default vision model" / "default video model" dropdowns. + */ + public List listByType(String modelType, String modality) { + List rows; if ("chat".equals(modelType)) { - return modelConfigMapper.selectList(new LambdaQueryWrapper() + rows = modelConfigMapper.selectList(new LambdaQueryWrapper() .and(w -> w.isNull(ModelConfigEntity::getModelType) .or().eq(ModelConfigEntity::getModelType, "chat")) .orderByDesc(ModelConfigEntity::getIsDefault) .orderByAsc(ModelConfigEntity::getName)); + } else { + rows = modelConfigMapper.selectList(new LambdaQueryWrapper() + .eq(ModelConfigEntity::getModelType, modelType) + .orderByDesc(ModelConfigEntity::getIsDefault) + .orderByAsc(ModelConfigEntity::getName)); } - return modelConfigMapper.selectList(new LambdaQueryWrapper() - .eq(ModelConfigEntity::getModelType, modelType) - .orderByDesc(ModelConfigEntity::getIsDefault) - .orderByAsc(ModelConfigEntity::getName)); + if (modality == null || modality.isBlank()) return rows; + ModelCapabilityService.Modality required; + try { + required = ModelCapabilityService.Modality.valueOf(modality.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + return rows; + } + return rows.stream() + .filter(m -> Boolean.TRUE.equals(m.getEnabled())) + .filter(m -> modelCapabilityService.supports(m.getModelName(), m.getModalities(), required)) + .toList(); } /** diff --git a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java index ace13763..42f012a5 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java @@ -134,4 +134,22 @@ public class SystemSettingsDTO { /** 首选 3D provider: auto / hunyuan-3d */ private String model3dProvider; private Boolean model3dFallbackEnabled; + + // ===== Multimodal sidecar routing ===== + /** + * Default vision-capable model id used to caption image attachments when the + * agent's primary model lacks the VISION modality. References mate_model_config.id; + * provider+model_name pairs are not unique so we store the surrogate key. + * null / non-existent / disabled rows are treated as "not configured" — the + * runtime then leaves the attachment out and asks the user to pick a model. + */ + private Long defaultVisionModelId; + + /** + * Default video-capable model id used when the agent's primary model lacks + * the VIDEO modality. Same semantics as defaultVisionModelId. v1 routing does + * not yet implement video sidecar; this is reserved for the next iteration so + * the configuration surface is stable. + */ + private Long defaultVideoModelId; } diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index e67cb3d8..fa1232d5 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -63,6 +63,10 @@ public class SystemSettingService { private static final String MODEL3D_PROVIDER_KEY = "model3dProvider"; private static final String MODEL3D_FALLBACK_ENABLED_KEY = "model3dFallbackEnabled"; + // Multimodal sidecar routing keys (id values; references mate_model_config.id) + private static final String DEFAULT_VISION_MODEL_KEY = "default.vision_model"; + private static final String DEFAULT_VIDEO_MODEL_KEY = "default.video_model"; + private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey"; private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl"; private static final String FAL_API_KEY_KEY = "falApiKey"; @@ -151,9 +155,22 @@ public class SystemSettingService { dto.setModel3dEnabled(Boolean.parseBoolean(getValue(MODEL3D_ENABLED_KEY, "false"))); dto.setModel3dProvider(getValue(MODEL3D_PROVIDER_KEY, "auto")); dto.setModel3dFallbackEnabled(Boolean.parseBoolean(getValue(MODEL3D_FALLBACK_ENABLED_KEY, "true"))); + + // Multimodal sidecar routing — empty string means "not configured" + dto.setDefaultVisionModelId(parseIdOrNull(getValue(DEFAULT_VISION_MODEL_KEY, ""))); + dto.setDefaultVideoModelId(parseIdOrNull(getValue(DEFAULT_VIDEO_MODEL_KEY, ""))); return dto; } + private Long parseIdOrNull(String value) { + if (value == null || value.isBlank()) return null; + try { + return Long.parseLong(value.trim()); + } catch (NumberFormatException e) { + return null; + } + } + /** * 获取全部配置（内部使用，包含明文 API Key）— 供 VideoGenerationService 等后端服务使用 */ @@ -333,6 +350,15 @@ public class SystemSettingService { if (dto.getModel3dFallbackEnabled() != null) { saveValue(MODEL3D_FALLBACK_ENABLED_KEY, String.valueOf(dto.getModel3dFallbackEnabled()), "3D Provider 级 Fallback"); } + + // Multimodal sidecar routing — write empty string to clear (parse-back returns null) + // Always written so users can revert to "not configured" via the UI. + saveValue(DEFAULT_VISION_MODEL_KEY, + dto.getDefaultVisionModelId() == null ? "" : String.valueOf(dto.getDefaultVisionModelId()), + "Default vision-capable model id (mate_model_config.id) for sidecar routing"); + saveValue(DEFAULT_VIDEO_MODEL_KEY, + dto.getDefaultVideoModelId() == null ? "" : String.valueOf(dto.getDefaultVideoModelId()), + "Default video-capable model id (mate_model_config.id) for sidecar routing"); return getSettings(); } diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql b/mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql new file mode 100644 index 00000000..aa7acd8b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql @@ -0,0 +1,16 @@ +-- V100: System-level defaults for vision and video sidecar routing. +-- When the agent's primary model lacks the modality required by an attachment, +-- the runtime delegates a single caption call to the model recorded here. +-- Empty value = not configured; the UI then asks the user to pick one. +-- Setting value stores mate_model_config.id as a string (provider+model_name pairs are not unique). +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000002001, 'default.vision_model', '', + 'Default vision-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VISION modality', + NOW(), NOW()); + +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000002002, 'default.video_model', '', + 'Default video-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VIDEO modality', + NOW(), NOW()); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql new file mode 100644 index 00000000..5aa02d34 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql @@ -0,0 +1,16 @@ +-- V100: System-level defaults for vision and video sidecar routing. +-- When the agent's primary model lacks the modality required by an attachment, +-- the runtime delegates a single caption call to the model recorded here. +-- Empty value = not configured; the UI then asks the user to pick one. +-- Setting value stores mate_model_config.id as a string (provider+model_name pairs are not unique). +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000002001, 'default.vision_model', '', + 'Default vision-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VISION modality', + NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key = setting_key; + +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000002002, 'default.video_model', '', + 'Default video-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VIDEO modality', + NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key = setting_key;