package vip.mate.llm.routing; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import vip.mate.llm.model.ModelConfigEntity; import vip.mate.llm.routing.model.MultimodalRoutingDecision; import vip.mate.llm.routing.model.MultimodalRoutingDecision.SkippedAttachment; import vip.mate.llm.service.ModelCapabilityService; import vip.mate.llm.service.ModelCapabilityService.Modality; import vip.mate.llm.service.ModelConfigService; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.system.service.SystemSettingService; import vip.mate.workspace.conversation.model.MessageContentPart; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; import java.util.Set; /** * Decides how to handle attachments whose modality outruns the agent's primary model. * *
The router is a pure decision step: it inspects the parts list and the primary * model's capability set, then returns a {@link MultimodalRoutingDecision}. Caller is * responsible for executing the decision (e.g. invoking the caption service when * strategy is SIDECAR). * *
v1 only supports image sidecar. Video attachments fall through to the NONE
* branch with an explanatory skip reason — the next iteration will add a video
* captioning path once a strategy for frame sampling is in place.
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class MultimodalRouter {
private final SystemSettingService systemSettingService;
private final ModelConfigService modelConfigService;
private final ModelCapabilityService capabilityService;
public MultimodalRoutingDecision route(List
* An explicit sidecar selection is treated as the user's own capability
* declaration: a provider-compatible model can be vision-capable in practice
* even when the built-in heuristics don't recognize its name and it carries no
* declared {@code modalities}. Rejecting such a model here made it impossible to
* use a perfectly good compatible-mode vision model as the sidecar. We therefore
* honour the explicit choice and only emit a diagnostic when the heuristics
* can't confirm it — a wrong pick degrades gracefully (the caption call fails and
* the attachment is reported as un-processed) rather than being silently ignored.
*/
private ModelConfigEntity resolveSidecar(Modality modality) {
SystemSettingsDTO settings = systemSettingService.getSettings();
Long modelId = switch (modality) {
case VISION -> settings.getDefaultVisionModelId();
case VIDEO -> settings.getDefaultVideoModelId();
default -> null;
};
if (modelId == null) return null;
ModelConfigEntity model;
try {
model = modelConfigService.getModel(modelId);
} catch (Exception e) {
log.debug("Configured sidecar model id={} could not be loaded: {}", modelId, e.getMessage());
return null;
}
if (model == null || !Boolean.TRUE.equals(model.getEnabled())) return null;
if (!capabilityService.supports(model.getModelName(), model.getModalities(), modality)) {
log.info("Configured sidecar model {}/{} is not recognized as {}-capable by the "
+ "built-in heuristics; honouring the explicit selection anyway",
model.getProvider(), model.getModelName(), modality.name().toLowerCase());
}
return model;
}
private String describeMissingSidecar(Modality modality) {
SystemSettingsDTO settings = systemSettingService.getSettings();
Long configured = modality == Modality.VISION
? settings.getDefaultVisionModelId()
: settings.getDefaultVideoModelId();
if (configured == null) return modality.name().toLowerCase() + "_model_not_configured";
return modality.name().toLowerCase() + "_model_unavailable";
}
}