mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(agent,llm): multimodal sidecar routing for unsupported attachments (#87)
This commit is contained in:
matevip
parent
97e2b12f71
commit
c2aecf18ef
@ -133,6 +133,8 @@ public class AgentGraphBuilder {
|
||||
private final vip.mate.llm.failover.AvailableProviderPool providerPool;
|
||||
/** PR-0b: DashScope-specific construction lives here now; we only call into it for the search-on log. */
|
||||
private final vip.mate.agent.chatmodel.AgentDashScopeChatModelBuilder dashScopeBuilder;
|
||||
private final vip.mate.llm.routing.MultimodalRouter multimodalRouter;
|
||||
private final vip.mate.llm.routing.MediaCaptionService mediaCaptionService;
|
||||
|
||||
/**
|
||||
* Optional audit pipeline. Setter injection (rather than a constructor
|
||||
@ -293,6 +295,11 @@ public class AgentGraphBuilder {
|
||||
agent.modelCapabilities = modelCapabilityService.resolve(
|
||||
runtimeModel.getModelName(), runtimeModel.getModalities());
|
||||
agent.runtimeProviderId = provider != null ? provider.getProviderId() : "";
|
||||
agent.runtimeModelConfig = runtimeModel;
|
||||
agent.toolSet = toolSet;
|
||||
agent.multimodalRouter = multimodalRouter;
|
||||
agent.mediaCaptionService = mediaCaptionService;
|
||||
agent.userLocale = resolveLocale();
|
||||
agent.temperature = runtimeModel.getTemperature();
|
||||
agent.maxTokens = runtimeModel.getMaxTokens();
|
||||
agent.maxInputTokens = runtimeModel.getMaxInputTokens();
|
||||
@ -450,6 +457,8 @@ public class AgentGraphBuilder {
|
||||
// 丢这个键，evidence_insufficient 检查会"静默地不生效" ——
|
||||
// StateKeyRegistrationCoverageTest 专门兜这条。
|
||||
.addStrategy(MateClawStateKeys.SOURCE_EVIDENCE_LEDGER, KeyStrategy.REPLACE)
|
||||
// Multimodal sidecar routing decision for the current turn.
|
||||
.addStrategy(MateClawStateKeys.ROUTING_DECISION, KeyStrategy.REPLACE)
|
||||
.build();
|
||||
|
||||
// Graph 拓扑：
|
||||
@ -636,6 +645,8 @@ public class AgentGraphBuilder {
|
||||
// 丢这个键，evidence_insufficient 检查会"静默地不生效" ——
|
||||
// StateKeyRegistrationCoverageTest 专门兜这条。
|
||||
.addStrategy(MateClawStateKeys.SOURCE_EVIDENCE_LEDGER, KeyStrategy.REPLACE)
|
||||
// Multimodal sidecar routing decision for the current turn.
|
||||
.addStrategy(MateClawStateKeys.ROUTING_DECISION, KeyStrategy.REPLACE)
|
||||
.build();
|
||||
|
||||
StateGraph graph = new StateGraph("react-agent-v2", keyStrategyFactory)
|
||||
@ -701,6 +712,21 @@ public class AgentGraphBuilder {
|
||||
return buildRuntimeChatModel(runtimeModel, this.retryTemplate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the user-facing locale used for sidecar caption prompts.
|
||||
* Reads {@code language} from system settings; falls back to
|
||||
* {@code zh-CN} so CN deployments stay consistent with the chat UI.
|
||||
*/
|
||||
private java.util.Locale resolveLocale() {
|
||||
try {
|
||||
String lang = systemSettingService.getLanguage();
|
||||
if (lang == null || lang.isBlank()) return java.util.Locale.SIMPLIFIED_CHINESE;
|
||||
return java.util.Locale.forLanguageTag(lang);
|
||||
} catch (Exception e) {
|
||||
return java.util.Locale.SIMPLIFIED_CHINESE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建运行时 ChatModel，并指定自定义的 Spring AI {@link RetryTemplate}。
|
||||
* <p>
|
||||
|
||||
@ -10,6 +10,10 @@ import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.util.MimeType;
|
||||
import reactor.core.publisher.Flux;
|
||||
import vip.mate.approval.ApprovalPlaceholderUtil;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.routing.MediaCaptionService;
|
||||
import vip.mate.llm.routing.MultimodalRouter;
|
||||
import vip.mate.llm.routing.model.MultimodalRoutingDecision;
|
||||
import vip.mate.llm.service.ModelCapabilityService;
|
||||
import vip.mate.workspace.conversation.ConversationService;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
@ -85,6 +89,31 @@ public abstract class BaseAgent {
|
||||
/** 构建时使用的 provider ID（运行时快照） */
|
||||
protected String runtimeProviderId;
|
||||
|
||||
/**
|
||||
* Full runtime model configuration used by the multimodal router to
|
||||
* decide whether the primary model can handle attachments natively.
|
||||
* Set by {@code AgentGraphBuilder} alongside {@link #modelCapabilities}.
|
||||
*/
|
||||
protected ModelConfigEntity runtimeModelConfig;
|
||||
|
||||
/**
|
||||
* The agent's effective tool set. Lifted from subclasses so
|
||||
* {@link #buildUserMessage} can ask whether the agent has any media-capable
|
||||
* tool when the primary model rejects an attachment.
|
||||
*/
|
||||
protected vip.mate.agent.AgentToolSet toolSet;
|
||||
|
||||
/**
|
||||
* Optional sidecar routing services. Null when not wired (e.g. tests with
|
||||
* minimal builders); the routing path then degrades to the legacy
|
||||
* skip-with-text-hint behavior without any extra LLM calls.
|
||||
*/
|
||||
protected MultimodalRouter multimodalRouter;
|
||||
protected MediaCaptionService mediaCaptionService;
|
||||
|
||||
/** Locale used when prompting the vision sidecar. Defaults to zh-CN when unset. */
|
||||
protected java.util.Locale userLocale = java.util.Locale.SIMPLIFIED_CHINESE;
|
||||
|
||||
|
||||
protected BaseAgent(ChatClient chatClient, ConversationService conversationService) {
|
||||
this.chatClient = chatClient;
|
||||
@ -510,34 +539,87 @@ public abstract class BaseAgent {
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建 UserMessage，支持 multimodal：如果消息包含图片/视频附件，直接注入 Spring AI Media 对象，
|
||||
* 让模型在 prompt 中直接看到媒体内容，不需要再调 MCP read_media_file 工具。
|
||||
* Build a {@link UserMessage} for the current turn, including any image/video
|
||||
* media the agent's primary model can handle natively. Returns the message
|
||||
* paired with the {@link MultimodalRoutingDecision} taken so the caller can
|
||||
* persist it as message metadata and emit a routing event.
|
||||
*/
|
||||
protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) {
|
||||
return buildUserMessage(message, renderedContent, true);
|
||||
protected CurrentTurnUserMessage buildUserMessageForCurrentTurn(MessageEntity message, String renderedContent) {
|
||||
return buildUserMessageInternal(message, renderedContent, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param injectMedia when {@code false} (history replay), skip the Media-loading
|
||||
* branch entirely and return text-only — providers like Zhipu
|
||||
* GLM-5V cap at 1 video per request, so re-injecting historical
|
||||
* attachments on every turn breaks the call.
|
||||
* History-replay variant: text-only, no media reinjected, no routing decision.
|
||||
* Many providers cap at one video per request, so re-injecting old attachments
|
||||
* on every replay would break the call.
|
||||
*/
|
||||
protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) {
|
||||
return buildUserMessageInternal(message, renderedContent, true).userMessage();
|
||||
}
|
||||
|
||||
protected UserMessage buildUserMessage(MessageEntity message, String renderedContent, boolean injectMedia) {
|
||||
return buildUserMessageInternal(message, renderedContent, injectMedia).userMessage();
|
||||
}
|
||||
|
||||
private CurrentTurnUserMessage buildUserMessageInternal(MessageEntity message, String renderedContent, boolean injectMedia) {
|
||||
if (!injectMedia) {
|
||||
return new UserMessage(renderedContent == null ? "" : renderedContent);
|
||||
return new CurrentTurnUserMessage(
|
||||
new UserMessage(renderedContent == null ? "" : renderedContent),
|
||||
null);
|
||||
}
|
||||
List<MessageContentPart> parts = conversationService.parseMessageParts(message);
|
||||
|
||||
// Sidecar routing — runs first so caption text gets folded into finalText
|
||||
// before native media injection considers the same parts again.
|
||||
MultimodalRoutingDecision decision = multimodalRouter != null
|
||||
? multimodalRouter.route(parts, runtimeModelConfig)
|
||||
: MultimodalRoutingDecision.none();
|
||||
|
||||
StringBuilder textBuilder = new StringBuilder(renderedContent == null ? "" : renderedContent);
|
||||
java.util.Set<String> sidecarHandledIdentifiers = new java.util.HashSet<>();
|
||||
if (decision.strategy() == MultimodalRoutingDecision.Strategy.SIDECAR
|
||||
&& mediaCaptionService != null
|
||||
&& decision.sidecarModel() != null) {
|
||||
for (MessageContentPart part : parts) {
|
||||
if (part == null) continue;
|
||||
String contentType = part.getContentType();
|
||||
boolean isImage = ("image".equals(part.getType()) || "file".equals(part.getType()))
|
||||
&& contentType != null && contentType.startsWith("image/")
|
||||
&& !contentType.contains("svg");
|
||||
if (!isImage) continue;
|
||||
MediaCaptionService.CaptionResult result = mediaCaptionService.caption(
|
||||
decision.sidecarModel(), part, userLocale);
|
||||
if (result.isFailure()) {
|
||||
log.warn("[{}] Sidecar caption failed for {}: {}",
|
||||
agentName, part.getFileName(), result.failure().getMessage());
|
||||
textBuilder.append("\n\n[系统提示] 视觉模型未能解析附件 ")
|
||||
.append(part.getFileName())
|
||||
.append("，请稍后重试或在「设置 → 模型」检查视觉模型配置。");
|
||||
continue;
|
||||
}
|
||||
textBuilder.append("\n\n[图片附件描述: ")
|
||||
.append(part.getFileName() == null ? "image" : part.getFileName())
|
||||
.append("]\n")
|
||||
.append(result.description())
|
||||
.append("\n[/图片附件描述]");
|
||||
String identifier = identifyPart(part);
|
||||
if (identifier != null) sidecarHandledIdentifiers.add(identifier);
|
||||
}
|
||||
}
|
||||
|
||||
List<Media> mediaList = new ArrayList<>();
|
||||
// Reasons for attachments that the model cannot consume — surfaced to the agent
|
||||
// via the user message text so it does not hallucinate a tool call to read them.
|
||||
// See issue #44.
|
||||
List<String> skippedAttachments = new ArrayList<>();
|
||||
boolean videoSupported = modelSupportsVideo();
|
||||
boolean visionSupported = modelSupportsVision();
|
||||
|
||||
for (MessageContentPart part : parts) {
|
||||
if (part == null) continue;
|
||||
// Sidecar already produced text for this image; never inject the
|
||||
// raw bytes — the primary model would receive them and try to
|
||||
// process natively, defeating the cost-saving purpose.
|
||||
String identifier = identifyPart(part);
|
||||
if (identifier != null && sidecarHandledIdentifiers.contains(identifier)) continue;
|
||||
|
||||
String partType = part.getType();
|
||||
String contentType = part.getContentType();
|
||||
// image 类型的 part 可能没有精确 contentType，补全为 image/jpeg
|
||||
@ -607,21 +689,88 @@ public abstract class BaseAgent {
|
||||
}
|
||||
}
|
||||
|
||||
String finalText = renderedContent;
|
||||
if (!skippedAttachments.isEmpty()) {
|
||||
finalText = (renderedContent == null ? "" : renderedContent)
|
||||
+ "\n\n[系统提示] 以下附件未能传入当前模型：" + String.join("、", skippedAttachments)
|
||||
+ "。\n请用对话语言清晰、友好地告诉用户：当前模型无法处理这类附件，建议切换到具备相应能力的多模态模型（图片需视觉模型，视频需视频理解模型）后重新上传。"
|
||||
+ "不要调用任何工具（包括 ffmpeg、浏览器、文件读取等）尝试解析这些附件。";
|
||||
textBuilder.append("\n\n[系统提示] 以下附件未能传入当前模型：")
|
||||
.append(String.join("、", skippedAttachments))
|
||||
.append("。");
|
||||
// Only suggest switching models when no media-capable tool is bound
|
||||
// either. With a media tool the LLM may legitimately choose to
|
||||
// delegate to the tool — never instruct it not to use tools.
|
||||
if (!hasMediaCapableTools()) {
|
||||
textBuilder.append("\n请用对话语言清晰、友好地告诉用户：当前模型无法处理这类附件，建议切换到具备相应能力的多模态模型，或在「设置 → 模型」中配置视觉/视频模型作为旁路。");
|
||||
}
|
||||
}
|
||||
|
||||
if (mediaList.isEmpty()) {
|
||||
return new UserMessage(finalText);
|
||||
String finalText = textBuilder.toString();
|
||||
UserMessage built = mediaList.isEmpty()
|
||||
? new UserMessage(finalText)
|
||||
: UserMessage.builder().text(finalText).media(mediaList).build();
|
||||
return new CurrentTurnUserMessage(built, decision);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable identifier for de-duplicating parts already handled by the sidecar
|
||||
* pass. Falls back across {@code path → mediaId → fileName} since not every
|
||||
* channel populates the same field.
|
||||
*/
|
||||
private static String identifyPart(MessageContentPart part) {
|
||||
if (part == null) return null;
|
||||
if (part.getPath() != null && !part.getPath().isBlank()) return "p:" + part.getPath();
|
||||
if (part.getMediaId() != null && !part.getMediaId().isBlank()) return "m:" + part.getMediaId();
|
||||
if (part.getFileName() != null && !part.getFileName().isBlank()) return "f:" + part.getFileName();
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* True if the agent has at least one tool whose name or description
|
||||
* suggests it can read images / video / audio. The check is intentionally
|
||||
* loose — false positives just mean the agent is allowed to attempt media
|
||||
* processing on its own, which is the safer default.
|
||||
*/
|
||||
private static final Set<String> MEDIA_TOOL_KEYWORDS = Set.of(
|
||||
"image", "图片", "vision", "视觉",
|
||||
"video", "视频", "ffmpeg",
|
||||
"ocr", "caption", "media", "audio", "音频");
|
||||
|
||||
private boolean hasMediaCapableTools() {
|
||||
if (toolSet == null) return false;
|
||||
var callbacks = toolSet.callbacks();
|
||||
if (callbacks == null || callbacks.isEmpty()) return false;
|
||||
return callbacks.stream().anyMatch(cb -> {
|
||||
try {
|
||||
String name = String.valueOf(cb.getToolDefinition().name()).toLowerCase();
|
||||
String desc = String.valueOf(cb.getToolDefinition().description()).toLowerCase();
|
||||
return MEDIA_TOOL_KEYWORDS.stream().anyMatch(k -> name.contains(k) || desc.contains(k));
|
||||
} catch (Exception e) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Pair returned from the current-turn user message build path: the assembled
|
||||
* {@link UserMessage} and the routing decision the caller should persist as
|
||||
* {@code metadata.routing} and surface to the SSE consumer.
|
||||
*/
|
||||
public record CurrentTurnUserMessage(UserMessage userMessage, MultimodalRoutingDecision routingDecision) {}
|
||||
|
||||
/**
|
||||
* Extract a routing-decision payload from the graph input map (placed there
|
||||
* by {@code buildInitialState}) and turn it into a startup
|
||||
* {@link vip.mate.agent.AgentService.StreamDelta} the SSE accumulator can
|
||||
* persist. Returns an empty Flux when no routing happened this turn so we
|
||||
* don't emit zero-value events.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public static reactor.core.publisher.Flux<vip.mate.agent.AgentService.StreamDelta> routingStartupDelta(
|
||||
java.util.Map<String, Object> inputs) {
|
||||
Object decision = inputs.get(vip.mate.agent.graph.state.MateClawStateKeys.ROUTING_DECISION);
|
||||
if (decision instanceof java.util.Map<?, ?> map && !map.isEmpty()) {
|
||||
return reactor.core.publisher.Flux.just(vip.mate.agent.AgentService.StreamDelta.event(
|
||||
vip.mate.agent.GraphEventPublisher.EVENT_ROUTING_DECISION,
|
||||
(java.util.Map<String, Object>) map));
|
||||
}
|
||||
return UserMessage.builder()
|
||||
.text(finalText)
|
||||
.media(mediaList)
|
||||
.build();
|
||||
return reactor.core.publisher.Flux.empty();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -642,6 +791,17 @@ public abstract class BaseAgent {
|
||||
* @return 带图片 Media 的 UserMessage（如果有图片附件），否则纯文本 UserMessage
|
||||
*/
|
||||
protected UserMessage buildCurrentUserMessage(String conversationId, String userMessageText) {
|
||||
return buildCurrentUserMessageWithRouting(conversationId, userMessageText).userMessage();
|
||||
}
|
||||
|
||||
/**
|
||||
* Same as {@link #buildCurrentUserMessage} but also returns the multimodal
|
||||
* routing decision taken for this turn so the caller can persist it as
|
||||
* {@code metadata.routing} and emit a SSE-side event for the chat UI.
|
||||
* Returns a decision with NONE strategy when the message has no attachments
|
||||
* the primary model can't already handle.
|
||||
*/
|
||||
protected CurrentTurnUserMessage buildCurrentUserMessageWithRouting(String conversationId, String userMessageText) {
|
||||
try {
|
||||
List<MessageEntity> history = conversationService.listMessages(conversationId);
|
||||
// 倒序取最后一条 user 消息（buildInitialState 在 saveMessage 后调用，所以最后一条就是当前消息）
|
||||
@ -650,14 +810,14 @@ public abstract class BaseAgent {
|
||||
if ("user".equals(msg.getRole())) {
|
||||
// 用 DB 中的实际内容（可能包含 contentParts），不用传入的 text
|
||||
String content = conversationService.renderMessageContent(msg);
|
||||
return buildUserMessage(msg, content != null && !content.isBlank() ? content : userMessageText);
|
||||
return buildUserMessageForCurrentTurn(msg, content != null && !content.isBlank() ? content : userMessageText);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("[{}] Failed to load current user message parts for multimodal: {}",
|
||||
agentName, e.getMessage());
|
||||
}
|
||||
return new UserMessage(userMessageText);
|
||||
return new CurrentTurnUserMessage(new UserMessage(userMessageText), null);
|
||||
}
|
||||
|
||||
protected Path resolveImagePath(String relativePath) {
|
||||
|
||||
@ -45,6 +45,15 @@ public final class GraphEventPublisher {
|
||||
*/
|
||||
public static final String EVENT_FINISH_REASON = "finish_reason";
|
||||
|
||||
/**
|
||||
* Multimodal sidecar routing decision for the current turn. Emitted once
|
||||
* per turn before the graph starts streaming; the channel-side accumulator
|
||||
* stores it under {@code metadata.routing} so the chat UI can show which
|
||||
* sidecar (if any) was invoked. Underscore-prefixed name keeps it out of
|
||||
* IM channel rebroadcast (see {@code ChannelMessageRouter}).
|
||||
*/
|
||||
public static final String EVENT_ROUTING_DECISION = "_routing_decision";
|
||||
|
||||
/**
|
||||
* 事件记录
|
||||
*/
|
||||
|
||||
@ -11,7 +11,13 @@ import vip.mate.channel.web.Utf8SseEmitter;
|
||||
import vip.mate.agent.AgentService;
|
||||
import vip.mate.agent.AgentState;
|
||||
import vip.mate.agent.model.AgentEntity;
|
||||
import vip.mate.agent.vo.AgentCapabilitiesVO;
|
||||
import vip.mate.audit.service.AuditEventService;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.service.ModelCapabilityService;
|
||||
import vip.mate.llm.service.ModelConfigService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.system.service.SystemSettingService;
|
||||
import vip.mate.auth.model.UserEntity;
|
||||
import vip.mate.auth.service.AuthService;
|
||||
import vip.mate.common.result.R;
|
||||
@ -40,6 +46,9 @@ public class AgentController {
|
||||
private final AuditEventService auditEventService;
|
||||
private final AuthService authService;
|
||||
private final WorkspaceService workspaceService;
|
||||
private final ModelConfigService modelConfigService;
|
||||
private final ModelCapabilityService modelCapabilityService;
|
||||
private final SystemSettingService systemSettingService;
|
||||
private final ExecutorService sseExecutor = Executors.newCachedThreadPool();
|
||||
|
||||
@Operation(summary = "获取Agent列表")
|
||||
@ -62,6 +71,57 @@ public class AgentController {
|
||||
return R.ok(agent);
|
||||
}
|
||||
|
||||
@Operation(summary = "获取Agent当前能力（modality 集合 + sidecar 配置），用于聊天页提示条")
|
||||
@GetMapping("/{id}/capabilities")
|
||||
@RequireWorkspaceRole("viewer")
|
||||
public R<AgentCapabilitiesVO> capabilities(
|
||||
@PathVariable Long id,
|
||||
@RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) {
|
||||
AgentEntity agent = agentService.getAgent(id);
|
||||
verifyResourceWorkspace(agent.getWorkspaceId(), workspaceId);
|
||||
|
||||
ModelConfigEntity primary;
|
||||
try {
|
||||
primary = modelConfigService.resolveModel(agent.getModelName());
|
||||
} catch (Exception e) {
|
||||
// No default model configured yet — return a capabilities snapshot that
|
||||
// tells the UI "we can't say anything about this agent's modalities".
|
||||
return R.ok(AgentCapabilitiesVO.builder()
|
||||
.agentId(id)
|
||||
.modelName("")
|
||||
.providerId("")
|
||||
.modalities(List.of())
|
||||
.build());
|
||||
}
|
||||
java.util.Set<ModelCapabilityService.Modality> modalities =
|
||||
modelCapabilityService.resolve(primary.getModelName(), primary.getModalities());
|
||||
|
||||
SystemSettingsDTO settings = systemSettingService.getSettings();
|
||||
Long visionId = settings.getDefaultVisionModelId();
|
||||
Long videoId = settings.getDefaultVideoModelId();
|
||||
|
||||
return R.ok(AgentCapabilitiesVO.builder()
|
||||
.agentId(id)
|
||||
.modelName(primary.getModelName())
|
||||
.providerId(primary.getProvider())
|
||||
.modalities(modalities.stream().map(Enum::name).toList())
|
||||
.defaultVisionModelId(visionId)
|
||||
.defaultVisionModelLabel(resolveSidecarLabel(visionId))
|
||||
.defaultVideoModelId(videoId)
|
||||
.defaultVideoModelLabel(resolveSidecarLabel(videoId))
|
||||
.build());
|
||||
}
|
||||
|
||||
private String resolveSidecarLabel(Long modelId) {
|
||||
if (modelId == null) return null;
|
||||
try {
|
||||
ModelConfigEntity m = modelConfigService.getModel(modelId);
|
||||
return m == null ? null : m.getProvider() + " / " + m.getModelName();
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Operation(summary = "创建Agent")
|
||||
@PostMapping
|
||||
@RequireWorkspaceRole("member")
|
||||
|
||||
@ -198,7 +198,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
|
||||
AtomicInteger lastSoftCap = new AtomicInteger(0);
|
||||
AtomicBoolean sawLegitimateExit = new AtomicBoolean(false);
|
||||
|
||||
return compiledGraph.stream(inputs, config)
|
||||
return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config)
|
||||
.flatMapIterable(output -> {
|
||||
List<AgentService.StreamDelta> deltas = new ArrayList<>();
|
||||
List<GraphEventPublisher.GraphEvent> allEvents = GraphEventPublisher.extractEvents(output);
|
||||
@ -263,7 +263,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
|
||||
));
|
||||
}
|
||||
return null;
|
||||
}).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))
|
||||
}).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())))
|
||||
.doOnComplete(() -> {
|
||||
setState(AgentState.IDLE);
|
||||
if (!sawLegitimateExit.get()) {
|
||||
@ -326,7 +326,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
|
||||
AtomicInteger lastSoftCap = new AtomicInteger(0);
|
||||
AtomicBoolean sawLegitimateExit = new AtomicBoolean(false);
|
||||
|
||||
return compiledGraph.stream(inputs, config)
|
||||
return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config)
|
||||
.flatMapIterable(output -> {
|
||||
List<AgentService.StreamDelta> deltas = new ArrayList<>();
|
||||
// 1. 提取所有累积的事件，只发送新增部分
|
||||
@ -402,7 +402,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
|
||||
));
|
||||
}
|
||||
return null;
|
||||
}).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))
|
||||
}).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())))
|
||||
.doOnComplete(() -> {
|
||||
setState(AgentState.IDLE);
|
||||
if (!sawLegitimateExit.get()) {
|
||||
@ -444,7 +444,9 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
|
||||
|
||||
List<Message> messages = new ArrayList<>(historyMessages);
|
||||
// 构建当前用户消息：支持 multimodal（如果有图片附件，直接注入 Media）
|
||||
messages.add(buildCurrentUserMessage(conversationId, userMessage));
|
||||
// 同步获取 routing decision，写入 state 供后续节点 / accumulator 读取。
|
||||
BaseAgent.CurrentTurnUserMessage currentTurn = buildCurrentUserMessageWithRouting(conversationId, userMessage);
|
||||
messages.add(currentTurn.userMessage());
|
||||
|
||||
Map<String, Object> inputs = new HashMap<>();
|
||||
// 输入
|
||||
@ -482,6 +484,15 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC
|
||||
inputs.put(RUNTIME_PROVIDER_ID, runtimeProviderId != null ? runtimeProviderId : "");
|
||||
inputs.put(TRACE_ID, UUID.randomUUID().toString().substring(0, 8));
|
||||
|
||||
// Multimodal sidecar routing — null when the turn carries no media or
|
||||
// the primary model already covers the modalities. Stored as a Map so
|
||||
// graph state stays JSON-friendly.
|
||||
if (currentTurn.routingDecision() != null
|
||||
&& currentTurn.routingDecision().strategy() != vip.mate.llm.routing.model.MultimodalRoutingDecision.Strategy.NONE
|
||||
|| (currentTurn.routingDecision() != null && !currentTurn.routingDecision().skipped().isEmpty())) {
|
||||
inputs.put(MateClawStateKeys.ROUTING_DECISION, currentTurn.routingDecision().toMap());
|
||||
}
|
||||
|
||||
// RFC-063r §2.5: enrich the originating ChatOrigin with this agent's id
|
||||
// and workspace, then write it into graph state so ActionNode +
|
||||
// StepExecutionNode can forward it to ToolExecutionExecutor → ToolContext.
|
||||
|
||||
@ -148,7 +148,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS
|
||||
AtomicReference<String> lastPersistedStepResult = new AtomicReference<>("");
|
||||
AtomicReference<String> lastPersistedStepThinking = new AtomicReference<>("");
|
||||
|
||||
return compiledGraph.stream(inputs, config)
|
||||
return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config)
|
||||
.flatMapIterable(output -> {
|
||||
List<AgentService.StreamDelta> deltas = new ArrayList<>();
|
||||
// 1. 提取事件（只发送新增部分）
|
||||
@ -218,7 +218,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS
|
||||
));
|
||||
}
|
||||
return null;
|
||||
}).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))
|
||||
}).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())))
|
||||
.doOnComplete(() -> setState(AgentState.IDLE))
|
||||
.doOnError(e -> {
|
||||
log.error("[{}] Plan-Execute stream error: {}", agentName, e.getMessage());
|
||||
@ -271,7 +271,8 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS
|
||||
}
|
||||
|
||||
List<Message> messages = new ArrayList<>(historyMessages);
|
||||
messages.add(buildCurrentUserMessage(conversationId, userMessage));
|
||||
BaseAgent.CurrentTurnUserMessage currentTurn = buildCurrentUserMessageWithRouting(conversationId, userMessage);
|
||||
messages.add(currentTurn.userMessage());
|
||||
|
||||
// 构建 working context：对历史消息做受控长度摘要
|
||||
String workingContext = buildWorkingContext(historyMessages, List.of());
|
||||
@ -299,6 +300,12 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS
|
||||
inputs.put(MateClawStateKeys.RUNTIME_PROVIDER_ID, runtimeProviderId != null ? runtimeProviderId : "");
|
||||
inputs.put(MateClawStateKeys.TRACE_ID, UUID.randomUUID().toString().substring(0, 8));
|
||||
|
||||
if (currentTurn.routingDecision() != null
|
||||
&& (currentTurn.routingDecision().strategy() != vip.mate.llm.routing.model.MultimodalRoutingDecision.Strategy.NONE
|
||||
|| !currentTurn.routingDecision().skipped().isEmpty())) {
|
||||
inputs.put(MateClawStateKeys.ROUTING_DECISION, currentTurn.routingDecision().toMap());
|
||||
}
|
||||
|
||||
// RFC-063r §2.5: same as ReAct path — enrich and store the ChatOrigin
|
||||
// so StepExecutionNode (and any sub-graphs spawned via DelegateAgentTool)
|
||||
// can read it back from state.
|
||||
|
||||
@ -83,6 +83,15 @@ public final class MateClawStateKeys {
|
||||
// ===== 事件流（APPEND 策略）=====
|
||||
public static final String PENDING_EVENTS = "pending_events";
|
||||
|
||||
/**
|
||||
* Multimodal routing decision for the current turn (REPLACE strategy).
|
||||
* Stored as a Map ready for JSON serialization. Set by BaseAgent before
|
||||
* the reasoning node runs; read back by FinalAnswerNode and (separately)
|
||||
* emitted as a graph event for the SSE accumulator to write into the
|
||||
* persisted message metadata under {@code metadata.routing}.
|
||||
*/
|
||||
public static final String ROUTING_DECISION = "routing_decision";
|
||||
|
||||
// ===== 阶段标记（REPLACE 策略）=====
|
||||
public static final String CURRENT_PHASE = "current_phase";
|
||||
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
package vip.mate.agent.vo;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Lightweight capability snapshot for an agent — answers two questions
|
||||
* the chat console needs synchronously while the user is composing a message:
|
||||
*
|
||||
* <ul>
|
||||
* <li>What modalities does the agent's primary model support? (drives the
|
||||
* attachment routing hint above the input box.)</li>
|
||||
* <li>Are sidecar models configured at the system level? (drives the
|
||||
* "configure a vision model" CTA when a user attaches an image to an
|
||||
* agent whose primary model can't process it.)</li>
|
||||
* </ul>
|
||||
*
|
||||
* <p>Returned by {@code GET /api/v1/agents/{id}/capabilities}. Computed on
|
||||
* each request — cheap because everything is cached service-side and we only
|
||||
* read at most three rows. Not persisted on {@code mate_agent}; sidecar
|
||||
* configuration is system-wide and {@code modelCapabilities} is derived from
|
||||
* {@code mate_model_config.modalities}.
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
public class AgentCapabilitiesVO {
|
||||
private Long agentId;
|
||||
private String modelName;
|
||||
private String providerId;
|
||||
/** Resolved modality set: any of {@code TEXT / VISION / VIDEO / AUDIO}. */
|
||||
private List<String> modalities;
|
||||
/** System-level vision sidecar model id, null when not configured. */
|
||||
private Long defaultVisionModelId;
|
||||
/** Display name of the configured vision sidecar (provider/modelName), null when not configured. */
|
||||
private String defaultVisionModelLabel;
|
||||
/** System-level video sidecar model id (reserved in v1; never wired). */
|
||||
private Long defaultVideoModelId;
|
||||
private String defaultVideoModelLabel;
|
||||
}
|
||||
@ -1650,6 +1650,14 @@ public class ChatController {
|
||||
private List<String> planSteps = List.of();
|
||||
private Integer currentPlanStep = null;
|
||||
private Map<String, Object> pendingApproval = null;
|
||||
/**
|
||||
* Multimodal sidecar routing decision for this turn (null when no
|
||||
* routing happened). Captured from the {@code _routing_decision}
|
||||
* event emitted before the graph stream and folded into
|
||||
* {@code metadata.routing} on persistence so the chat UI can show
|
||||
* which sidecar (if any) was invoked.
|
||||
*/
|
||||
private Map<String, Object> routingDecision = null;
|
||||
|
||||
synchronized void accept(AgentService.StreamDelta delta, String conversationId) {
|
||||
if (delta == null) return;
|
||||
@ -1683,6 +1691,13 @@ public class ChatController {
|
||||
finishReason = String.valueOf(reason);
|
||||
}
|
||||
}
|
||||
if (vip.mate.agent.GraphEventPublisher.EVENT_ROUTING_DECISION.equals(delta.eventType())) {
|
||||
// Captured at turn start; persisted under metadata.routing so the
|
||||
// chat UI can render which sidecar (if any) was invoked. Internal
|
||||
// event — return early to skip rebroadcast on IM channels.
|
||||
routingDecision = delta.eventData();
|
||||
return;
|
||||
}
|
||||
accumulateToolEvent(delta.eventType(), delta.eventData(), conversationId);
|
||||
try {
|
||||
broadcastEvent(conversationId, delta.eventType(), delta.eventData());
|
||||
@ -1967,6 +1982,9 @@ public class ChatController {
|
||||
// brittle text matching on the assistant content.
|
||||
metadata.put("finishReason", finishReason);
|
||||
}
|
||||
if (routingDecision != null && !routingDecision.isEmpty()) {
|
||||
metadata.put("routing", routingDecision);
|
||||
}
|
||||
return objectMapper.writeValueAsString(metadata);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to serialize metadata: {}", e.getMessage());
|
||||
|
||||
@ -204,10 +204,12 @@ public class ModelConfigController {
|
||||
|
||||
// ==================== Embedding 模型管理 ====================
|
||||
|
||||
@Operation(summary = "按类型筛选模型（chat / embedding）")
|
||||
@Operation(summary = "按类型筛选模型（chat / embedding），可选 modality 过滤")
|
||||
@GetMapping("/by-type")
|
||||
public R<List<ModelConfigEntity>> listByType(@RequestParam(defaultValue = "chat") String modelType) {
|
||||
return R.ok(modelConfigService.listByType(modelType));
|
||||
public R<List<ModelConfigEntity>> listByType(
|
||||
@RequestParam(defaultValue = "chat") String modelType,
|
||||
@RequestParam(required = false) String modality) {
|
||||
return R.ok(modelConfigService.listByType(modelType, modality));
|
||||
}
|
||||
|
||||
@Operation(summary = "测试 Embedding 模型连通性（嵌入一个短文本验证 API key）")
|
||||
|
||||
@ -0,0 +1,141 @@
|
||||
package vip.mate.llm.routing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.content.Media;
|
||||
import org.springframework.core.io.FileSystemResource;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.util.MimeType;
|
||||
import vip.mate.llm.chatmodel.ProviderChatModelFactory;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Caption an image attachment using a configured vision model so a text-only
|
||||
* primary model can still reason about it.
|
||||
*
|
||||
* <p>The service is the execution arm of the sidecar strategy chosen by
|
||||
* {@link MultimodalRouter}: pick a vision-capable model, send a single
|
||||
* structured prompt with the image attached, return the description text.
|
||||
*
|
||||
* <p>v1 has no caching layer — every call hits the vision model. The cache
|
||||
* (keyed by {@code sha256(file_bytes) + visionModelId + locale}) is reserved
|
||||
* for the next iteration; the API shape exposes {@code cacheHit} so callers
|
||||
* already record the field in routing metadata.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class MediaCaptionService {
|
||||
|
||||
private final ProviderChatModelFactory chatModelFactory;
|
||||
private final RetryTemplate retryTemplate;
|
||||
|
||||
public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale) {
|
||||
if (visionModel == null || imagePart == null) {
|
||||
return CaptionResult.failure(0, new IllegalArgumentException("vision model or image part is null"));
|
||||
}
|
||||
Path mediaPath = resolveMediaPath(imagePart);
|
||||
if (mediaPath == null) {
|
||||
return CaptionResult.failure(0, new IllegalStateException(
|
||||
"Image file not found for attachment: " + imagePart.getFileName()));
|
||||
}
|
||||
String contentType = imagePart.getContentType();
|
||||
if (contentType == null || "image/*".equals(contentType)) {
|
||||
contentType = "image/jpeg";
|
||||
}
|
||||
long start = System.currentTimeMillis();
|
||||
try {
|
||||
ChatModel chatModel = chatModelFactory.buildFor(visionModel, retryTemplate);
|
||||
ChatClient client = ChatClient.create(chatModel);
|
||||
UserMessage userMessage = UserMessage.builder()
|
||||
.text(buildPrompt(locale, imagePart.getFileName()))
|
||||
.media(List.of(new Media(MimeType.valueOf(contentType), new FileSystemResource(mediaPath))))
|
||||
.build();
|
||||
String description = client.prompt()
|
||||
.messages(userMessage)
|
||||
.call()
|
||||
.content();
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
String trimmed = description == null ? "" : description.trim();
|
||||
if (trimmed.isEmpty()) {
|
||||
return CaptionResult.failure(elapsed,
|
||||
new IllegalStateException("Vision model returned empty description"));
|
||||
}
|
||||
return CaptionResult.success(trimmed, elapsed, false);
|
||||
} catch (Exception e) {
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
log.warn("Caption call failed for {} via {}/{}: {}",
|
||||
imagePart.getFileName(), visionModel.getProvider(), visionModel.getModelName(),
|
||||
e.getMessage());
|
||||
return CaptionResult.failure(elapsed, e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Locale-aware prompt. Defaults to Chinese when the locale is null or unrecognized
|
||||
* (matches the primary user base) but switches to English so vision-model output
|
||||
* matches the chat language and avoids polluting English-only contexts.
|
||||
*/
|
||||
private String buildPrompt(Locale locale, String fileName) {
|
||||
boolean english = locale != null && Locale.ENGLISH.getLanguage().equalsIgnoreCase(locale.getLanguage());
|
||||
String fileHint = (fileName == null || fileName.isBlank()) ? "" : " (" + fileName + ")";
|
||||
if (english) {
|
||||
return "Describe this image" + fileHint
|
||||
+ " concisely: list the main objects, scene, any visible text (OCR), "
|
||||
+ "and notable actions or emotions. Keep the answer under 300 words. "
|
||||
+ "Reply with the description only — no preamble.";
|
||||
}
|
||||
return "请用一段简洁的中文描述这张图片" + fileHint
|
||||
+ "：列出主要物体、场景、画面中可见的文字（OCR）、以及人物动作或情绪。"
|
||||
+ "不超过 300 字。直接给出描述，不要寒暄。";
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirrors {@code BaseAgent.resolveImagePath} but standalone — caption service
|
||||
* is reused outside the agent context (e.g. tests, future preflight endpoint).
|
||||
*/
|
||||
private Path resolveMediaPath(MessageContentPart part) {
|
||||
Path resolved = tryResolve(part.getPath());
|
||||
if (resolved != null) return resolved;
|
||||
return tryResolve(part.getMediaId());
|
||||
}
|
||||
|
||||
private Path tryResolve(String relativePath) {
|
||||
if (relativePath == null || relativePath.isBlank()) return null;
|
||||
Path path = Paths.get(relativePath);
|
||||
if (path.isAbsolute() && Files.exists(path)) return path;
|
||||
Path workdir = Paths.get(System.getProperty("user.dir")).resolve(relativePath);
|
||||
if (Files.exists(workdir)) return workdir;
|
||||
return null;
|
||||
}
|
||||
|
||||
public record CaptionResult(
|
||||
String description,
|
||||
boolean cacheHit,
|
||||
long elapsedMs,
|
||||
Throwable failure
|
||||
) {
|
||||
public boolean isFailure() {
|
||||
return failure != null;
|
||||
}
|
||||
|
||||
public static CaptionResult success(String description, long elapsedMs, boolean cacheHit) {
|
||||
return new CaptionResult(description, cacheHit, elapsedMs, null);
|
||||
}
|
||||
|
||||
public static CaptionResult failure(long elapsedMs, Throwable failure) {
|
||||
return new CaptionResult(null, false, elapsedMs, failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,166 @@
|
||||
package vip.mate.llm.routing;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.routing.model.MultimodalRoutingDecision;
|
||||
import vip.mate.llm.routing.model.MultimodalRoutingDecision.SkippedAttachment;
|
||||
import vip.mate.llm.service.ModelCapabilityService;
|
||||
import vip.mate.llm.service.ModelCapabilityService.Modality;
|
||||
import vip.mate.llm.service.ModelConfigService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.system.service.SystemSettingService;
|
||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.EnumSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Decides how to handle attachments whose modality outruns the agent's primary model.
|
||||
*
|
||||
* <p>The router is a pure decision step: it inspects the parts list and the primary
|
||||
* model's capability set, then returns a {@link MultimodalRoutingDecision}. Caller is
|
||||
* responsible for executing the decision (e.g. invoking the caption service when
|
||||
* strategy is SIDECAR).
|
||||
*
|
||||
* <p>v1 only supports image sidecar. Video attachments fall through to the NONE
|
||||
* branch with an explanatory skip reason — the next iteration will add a video
|
||||
* captioning path once a strategy for frame sampling is in place.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class MultimodalRouter {
|
||||
|
||||
private final SystemSettingService systemSettingService;
|
||||
private final ModelConfigService modelConfigService;
|
||||
private final ModelCapabilityService capabilityService;
|
||||
|
||||
public MultimodalRoutingDecision route(List<MessageContentPart> parts, ModelConfigEntity primary) {
|
||||
Set<Modality> required = collectRequiredModalities(parts);
|
||||
if (required.isEmpty()) return MultimodalRoutingDecision.none();
|
||||
|
||||
EnumSet<Modality> primaryCaps = primary == null
|
||||
? EnumSet.noneOf(Modality.class)
|
||||
: capabilityService.resolve(primary.getModelName(), primary.getModalities());
|
||||
if (primaryCaps.containsAll(required)) return MultimodalRoutingDecision.none();
|
||||
|
||||
EnumSet<Modality> missing = EnumSet.copyOf(required);
|
||||
missing.removeAll(primaryCaps);
|
||||
|
||||
List<SkippedAttachment> skipped = new ArrayList<>();
|
||||
ModelConfigEntity sidecarModel = null;
|
||||
|
||||
// VISION sidecar: resolve configured default vision model.
|
||||
if (missing.contains(Modality.VISION)) {
|
||||
ModelConfigEntity candidate = resolveSidecar(Modality.VISION);
|
||||
if (candidate != null) {
|
||||
sidecarModel = candidate;
|
||||
} else {
|
||||
String reason = describeMissingSidecar(Modality.VISION);
|
||||
for (MessageContentPart p : imageParts(parts)) {
|
||||
skipped.add(new SkippedAttachment("image", p.getFileName(), reason));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// VIDEO: v1 has no sidecar implementation. Mark as skipped so the UI can
|
||||
// tell the user to switch to a video-capable primary model. Reserved for
|
||||
// a follow-up RFC.
|
||||
if (missing.contains(Modality.VIDEO)) {
|
||||
for (MessageContentPart p : videoParts(parts)) {
|
||||
skipped.add(new SkippedAttachment("video", p.getFileName(),
|
||||
"video_sidecar_not_supported_in_v1"));
|
||||
}
|
||||
}
|
||||
|
||||
if (sidecarModel != null) {
|
||||
return MultimodalRoutingDecision.sidecar(sidecarModel, required, missing);
|
||||
}
|
||||
return MultimodalRoutingDecision.noneWithSkipped(required, missing, skipped);
|
||||
}
|
||||
|
||||
private Set<Modality> collectRequiredModalities(List<MessageContentPart> parts) {
|
||||
if (parts == null || parts.isEmpty()) return Set.of();
|
||||
EnumSet<Modality> required = EnumSet.noneOf(Modality.class);
|
||||
for (MessageContentPart part : parts) {
|
||||
if (part == null) continue;
|
||||
String type = part.getType();
|
||||
String contentType = part.getContentType();
|
||||
if (isImagePart(type, contentType)) required.add(Modality.VISION);
|
||||
else if (isVideoPart(type, contentType)) required.add(Modality.VIDEO);
|
||||
else if (isAudioPart(type, contentType)) required.add(Modality.AUDIO);
|
||||
}
|
||||
return required;
|
||||
}
|
||||
|
||||
private boolean isImagePart(String type, String contentType) {
|
||||
if ("image".equals(type)) return true;
|
||||
return "file".equals(type) && contentType != null && contentType.startsWith("image/");
|
||||
}
|
||||
|
||||
private boolean isVideoPart(String type, String contentType) {
|
||||
if ("video".equals(type)) return true;
|
||||
return "file".equals(type) && contentType != null && contentType.startsWith("video/");
|
||||
}
|
||||
|
||||
private boolean isAudioPart(String type, String contentType) {
|
||||
if ("audio".equals(type)) return true;
|
||||
return "file".equals(type) && contentType != null && contentType.startsWith("audio/");
|
||||
}
|
||||
|
||||
private List<MessageContentPart> imageParts(List<MessageContentPart> parts) {
|
||||
return parts.stream()
|
||||
.filter(p -> p != null && isImagePart(p.getType(), p.getContentType()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private List<MessageContentPart> videoParts(List<MessageContentPart> parts) {
|
||||
return parts.stream()
|
||||
.filter(p -> p != null && isVideoPart(p.getType(), p.getContentType()))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the configured sidecar model for a modality. Returns null when:
|
||||
* - the setting is empty / blank;
|
||||
* - the referenced row no longer exists or has been disabled;
|
||||
* - the row's resolved capability set does not actually contain the modality.
|
||||
* The caller treats null as "ask the user to configure one."
|
||||
*/
|
||||
private ModelConfigEntity resolveSidecar(Modality modality) {
|
||||
SystemSettingsDTO settings = systemSettingService.getSettings();
|
||||
Long modelId = switch (modality) {
|
||||
case VISION -> settings.getDefaultVisionModelId();
|
||||
case VIDEO -> settings.getDefaultVideoModelId();
|
||||
default -> null;
|
||||
};
|
||||
if (modelId == null) return null;
|
||||
ModelConfigEntity model;
|
||||
try {
|
||||
model = modelConfigService.getModel(modelId);
|
||||
} catch (Exception e) {
|
||||
log.debug("Configured sidecar model id={} could not be loaded: {}", modelId, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
if (model == null || !Boolean.TRUE.equals(model.getEnabled())) return null;
|
||||
if (!capabilityService.supports(model.getModelName(), model.getModalities(), modality)) {
|
||||
log.warn("Configured sidecar model {}/{} does not actually support {} — ignoring",
|
||||
model.getProvider(), model.getModelName(), modality);
|
||||
return null;
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
private String describeMissingSidecar(Modality modality) {
|
||||
SystemSettingsDTO settings = systemSettingService.getSettings();
|
||||
Long configured = modality == Modality.VISION
|
||||
? settings.getDefaultVisionModelId()
|
||||
: settings.getDefaultVideoModelId();
|
||||
if (configured == null) return modality.name().toLowerCase() + "_model_not_configured";
|
||||
return modality.name().toLowerCase() + "_model_unavailable";
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,89 @@
|
||||
package vip.mate.llm.routing.model;
|
||||
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.service.ModelCapabilityService.Modality;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Outcome of routing a single user turn that may carry image / video / audio attachments.
|
||||
*
|
||||
* <p>The decision is purely descriptive — execution (loading captions, mutating
|
||||
* the user message) happens in the caller. Treat instances as immutable; the
|
||||
* static factories cover the only valid shapes.
|
||||
*/
|
||||
public record MultimodalRoutingDecision(
|
||||
Strategy strategy,
|
||||
ModelConfigEntity sidecarModel,
|
||||
Set<Modality> requiredModalities,
|
||||
Set<Modality> primaryMissing,
|
||||
List<SkippedAttachment> skipped
|
||||
) {
|
||||
|
||||
public enum Strategy {
|
||||
/** Primary model handles the turn directly (or no attachments at all). */
|
||||
NONE,
|
||||
/** A separate vision/video model captions attachments; primary stays. */
|
||||
SIDECAR,
|
||||
/** Reserved: switch the whole turn to a multimodal model. v1 does not emit. */
|
||||
NATIVE
|
||||
}
|
||||
|
||||
public record SkippedAttachment(String type, String fileName, String reason) {}
|
||||
|
||||
public static MultimodalRoutingDecision none() {
|
||||
return new MultimodalRoutingDecision(
|
||||
Strategy.NONE, null, Set.of(), Set.of(), List.of());
|
||||
}
|
||||
|
||||
public static MultimodalRoutingDecision noneWithSkipped(
|
||||
Set<Modality> required,
|
||||
Set<Modality> missing,
|
||||
List<SkippedAttachment> skipped) {
|
||||
return new MultimodalRoutingDecision(
|
||||
Strategy.NONE, null, required, missing, skipped);
|
||||
}
|
||||
|
||||
public static MultimodalRoutingDecision sidecar(
|
||||
ModelConfigEntity sidecarModel,
|
||||
Set<Modality> required,
|
||||
Set<Modality> missing) {
|
||||
return new MultimodalRoutingDecision(
|
||||
Strategy.SIDECAR, sidecarModel, required, missing, List.of());
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize to a flat map for emission as a graph event payload.
|
||||
* Only includes keys present in this decision so the resulting
|
||||
* {@code metadata.routing} JSON stays compact for the chat UI.
|
||||
*/
|
||||
public Map<String, Object> toMap() {
|
||||
Map<String, Object> m = new LinkedHashMap<>();
|
||||
m.put("strategy", strategy.name().toLowerCase());
|
||||
if (sidecarModel != null) {
|
||||
m.put("sidecarModelId", sidecarModel.getId());
|
||||
m.put("sidecarModel", sidecarModel.getModelName());
|
||||
m.put("sidecarProvider", sidecarModel.getProvider());
|
||||
}
|
||||
if (!requiredModalities.isEmpty()) {
|
||||
m.put("requiredModalities", requiredModalities.stream().map(Enum::name).toList());
|
||||
}
|
||||
if (!primaryMissing.isEmpty()) {
|
||||
m.put("primaryMissing", primaryMissing.stream().map(Enum::name).toList());
|
||||
}
|
||||
if (!skipped.isEmpty()) {
|
||||
m.put("skipped", skipped.stream().map(s -> {
|
||||
Map<String, Object> entry = new LinkedHashMap<>();
|
||||
entry.put("type", s.type());
|
||||
if (s.fileName() != null) entry.put("fileName", s.fileName());
|
||||
entry.put("reason", s.reason());
|
||||
return entry;
|
||||
}).toList());
|
||||
}
|
||||
return Collections.unmodifiableMap(m);
|
||||
}
|
||||
}
|
||||
@ -23,6 +23,7 @@ public class ModelConfigService {
|
||||
|
||||
private final ModelConfigMapper modelConfigMapper;
|
||||
private final ApplicationEventPublisher eventPublisher;
|
||||
private final ModelCapabilityService modelCapabilityService;
|
||||
|
||||
/**
|
||||
* Lazy to break circular dependency: ModelProviderService → ModelConfigService.
|
||||
@ -67,17 +68,40 @@ public class ModelConfigService {
|
||||
* </ul>
|
||||
*/
|
||||
public List<ModelConfigEntity> listByType(String modelType) {
|
||||
return listByType(modelType, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional modality filter (case-insensitive: {@code "vision" / "video" / "audio"}).
|
||||
* When non-null, only enabled rows whose resolved capability set contains the
|
||||
* requested modality survive — used by the multimodal sidecar settings UI to
|
||||
* populate "default vision model" / "default video model" dropdowns.
|
||||
*/
|
||||
public List<ModelConfigEntity> listByType(String modelType, String modality) {
|
||||
List<ModelConfigEntity> rows;
|
||||
if ("chat".equals(modelType)) {
|
||||
return modelConfigMapper.selectList(new LambdaQueryWrapper<ModelConfigEntity>()
|
||||
rows = modelConfigMapper.selectList(new LambdaQueryWrapper<ModelConfigEntity>()
|
||||
.and(w -> w.isNull(ModelConfigEntity::getModelType)
|
||||
.or().eq(ModelConfigEntity::getModelType, "chat"))
|
||||
.orderByDesc(ModelConfigEntity::getIsDefault)
|
||||
.orderByAsc(ModelConfigEntity::getName));
|
||||
} else {
|
||||
rows = modelConfigMapper.selectList(new LambdaQueryWrapper<ModelConfigEntity>()
|
||||
.eq(ModelConfigEntity::getModelType, modelType)
|
||||
.orderByDesc(ModelConfigEntity::getIsDefault)
|
||||
.orderByAsc(ModelConfigEntity::getName));
|
||||
}
|
||||
return modelConfigMapper.selectList(new LambdaQueryWrapper<ModelConfigEntity>()
|
||||
.eq(ModelConfigEntity::getModelType, modelType)
|
||||
.orderByDesc(ModelConfigEntity::getIsDefault)
|
||||
.orderByAsc(ModelConfigEntity::getName));
|
||||
if (modality == null || modality.isBlank()) return rows;
|
||||
ModelCapabilityService.Modality required;
|
||||
try {
|
||||
required = ModelCapabilityService.Modality.valueOf(modality.trim().toUpperCase());
|
||||
} catch (IllegalArgumentException e) {
|
||||
return rows;
|
||||
}
|
||||
return rows.stream()
|
||||
.filter(m -> Boolean.TRUE.equals(m.getEnabled()))
|
||||
.filter(m -> modelCapabilityService.supports(m.getModelName(), m.getModalities(), required))
|
||||
.toList();
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -134,4 +134,22 @@ public class SystemSettingsDTO {
|
||||
/** 首选 3D provider: auto / hunyuan-3d */
|
||||
private String model3dProvider;
|
||||
private Boolean model3dFallbackEnabled;
|
||||
|
||||
// ===== Multimodal sidecar routing =====
|
||||
/**
|
||||
* Default vision-capable model id used to caption image attachments when the
|
||||
* agent's primary model lacks the VISION modality. References mate_model_config.id;
|
||||
* provider+model_name pairs are not unique so we store the surrogate key.
|
||||
* null / non-existent / disabled rows are treated as "not configured" — the
|
||||
* runtime then leaves the attachment out and asks the user to pick a model.
|
||||
*/
|
||||
private Long defaultVisionModelId;
|
||||
|
||||
/**
|
||||
* Default video-capable model id used when the agent's primary model lacks
|
||||
* the VIDEO modality. Same semantics as defaultVisionModelId. v1 routing does
|
||||
* not yet implement video sidecar; this is reserved for the next iteration so
|
||||
* the configuration surface is stable.
|
||||
*/
|
||||
private Long defaultVideoModelId;
|
||||
}
|
||||
|
||||
@ -63,6 +63,10 @@ public class SystemSettingService {
|
||||
private static final String MODEL3D_PROVIDER_KEY = "model3dProvider";
|
||||
private static final String MODEL3D_FALLBACK_ENABLED_KEY = "model3dFallbackEnabled";
|
||||
|
||||
// Multimodal sidecar routing keys (id values; references mate_model_config.id)
|
||||
private static final String DEFAULT_VISION_MODEL_KEY = "default.vision_model";
|
||||
private static final String DEFAULT_VIDEO_MODEL_KEY = "default.video_model";
|
||||
|
||||
private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey";
|
||||
private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl";
|
||||
private static final String FAL_API_KEY_KEY = "falApiKey";
|
||||
@ -151,9 +155,22 @@ public class SystemSettingService {
|
||||
dto.setModel3dEnabled(Boolean.parseBoolean(getValue(MODEL3D_ENABLED_KEY, "false")));
|
||||
dto.setModel3dProvider(getValue(MODEL3D_PROVIDER_KEY, "auto"));
|
||||
dto.setModel3dFallbackEnabled(Boolean.parseBoolean(getValue(MODEL3D_FALLBACK_ENABLED_KEY, "true")));
|
||||
|
||||
// Multimodal sidecar routing — empty string means "not configured"
|
||||
dto.setDefaultVisionModelId(parseIdOrNull(getValue(DEFAULT_VISION_MODEL_KEY, "")));
|
||||
dto.setDefaultVideoModelId(parseIdOrNull(getValue(DEFAULT_VIDEO_MODEL_KEY, "")));
|
||||
return dto;
|
||||
}
|
||||
|
||||
private Long parseIdOrNull(String value) {
|
||||
if (value == null || value.isBlank()) return null;
|
||||
try {
|
||||
return Long.parseLong(value.trim());
|
||||
} catch (NumberFormatException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取全部配置（内部使用，包含明文 API Key）— 供 VideoGenerationService 等后端服务使用
|
||||
*/
|
||||
@ -333,6 +350,15 @@ public class SystemSettingService {
|
||||
if (dto.getModel3dFallbackEnabled() != null) {
|
||||
saveValue(MODEL3D_FALLBACK_ENABLED_KEY, String.valueOf(dto.getModel3dFallbackEnabled()), "3D Provider 级 Fallback");
|
||||
}
|
||||
|
||||
// Multimodal sidecar routing — write empty string to clear (parse-back returns null)
|
||||
// Always written so users can revert to "not configured" via the UI.
|
||||
saveValue(DEFAULT_VISION_MODEL_KEY,
|
||||
dto.getDefaultVisionModelId() == null ? "" : String.valueOf(dto.getDefaultVisionModelId()),
|
||||
"Default vision-capable model id (mate_model_config.id) for sidecar routing");
|
||||
saveValue(DEFAULT_VIDEO_MODEL_KEY,
|
||||
dto.getDefaultVideoModelId() == null ? "" : String.valueOf(dto.getDefaultVideoModelId()),
|
||||
"Default video-capable model id (mate_model_config.id) for sidecar routing");
|
||||
return getSettings();
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,16 @@
|
||||
-- V100: System-level defaults for vision and video sidecar routing.
|
||||
-- When the agent's primary model lacks the modality required by an attachment,
|
||||
-- the runtime delegates a single caption call to the model recorded here.
|
||||
-- Empty value = not configured; the UI then asks the user to pick one.
|
||||
-- Setting value stores mate_model_config.id as a string (provider+model_name pairs are not unique).
|
||||
MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
KEY (id)
|
||||
VALUES (1000002001, 'default.vision_model', '',
|
||||
'Default vision-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VISION modality',
|
||||
NOW(), NOW());
|
||||
|
||||
MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
KEY (id)
|
||||
VALUES (1000002002, 'default.video_model', '',
|
||||
'Default video-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VIDEO modality',
|
||||
NOW(), NOW());
|
||||
@ -0,0 +1,16 @@
|
||||
-- V100: System-level defaults for vision and video sidecar routing.
|
||||
-- When the agent's primary model lacks the modality required by an attachment,
|
||||
-- the runtime delegates a single caption call to the model recorded here.
|
||||
-- Empty value = not configured; the UI then asks the user to pick one.
|
||||
-- Setting value stores mate_model_config.id as a string (provider+model_name pairs are not unique).
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
VALUES (1000002001, 'default.vision_model', '',
|
||||
'Default vision-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VISION modality',
|
||||
NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE setting_key = setting_key;
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
VALUES (1000002002, 'default.video_model', '',
|
||||
'Default video-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VIDEO modality',
|
||||
NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE setting_key = setting_key;
|
||||
Loading…
Reference in New Issue
Block a user