mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(llm): provider health tracker + UI editor for failover priority
UI — Failover priority editor - ProviderConfigRequest + ProviderInfoDTO carry fallbackPriority - ModelProviderService.updateProviderConfig persists it (null = unchanged); toProviderInfo exposes the current value to the UI (defaults to 0) - ProviderConfigModal advanced panel exposes a number input with hint - ProviderCard shows a "Fallback #N" badge for chain members so the priority order is visible at a glance without opening the modal - 5 new i18n keys (zh + en) — verified to resolve at runtime via i18n.global.t Backend — Per-provider health tracker - ProviderHealthTracker: ConcurrentHashMap-backed counters; N consecutive failures (default 3) push the provider into a cooldown window (default 5 min) during which the chain walker skips it. Success resets both counter and cooldown atomically. Lazy expiry on lookup so dead entries do not accumulate. - ProviderHealthProperties exposed under mateclaw.llm.failover.health.* with sane production defaults - New FallbackEntry record (providerId + ChatModel) replaces raw List<ChatModel> in the chain so the walker can correlate cooldown state to entries; AgentGraphBuilder.buildFallbackChain returns the new type - NodeStreamingChatHelper takes the tracker through a new 4-arg constructor and consults it before each fallback call; records success/failure on each chain attempt. Legacy 2/3-arg constructors preserved as @Deprecated wrappers (synthetic providerId means no health tracking on the legacy path — that path is opt-out anyway) Tests - ProviderHealthTrackerTest (9 tests): below/at threshold, success reset, cooldown expiry (via reflection on the min-clamp setter), disabled-tracker no-op, null-providerId safety, per-provider isolation, snapshot output - NodeStreamingChatHelperFallbackChainTest updated to FallbackEntry field type — verifies providerId + ChatModel survive the chain - 168 tests pass (was 159 + 9 new) Verification - mvn test green; vue-tsc clean; live UI confirms i18n resolution
This commit is contained in:
matevip
parent
ed37e81e7e
commit
7b12c5f0c9
@ -130,6 +130,7 @@ public class AgentGraphBuilder {
|
||||
private final vip.mate.agent.graph.executor.ToolResultStorage toolResultStorage;
|
||||
private final vip.mate.tool.ToolConcurrencyRegistry toolConcurrencyRegistry;
|
||||
private final vip.mate.i18n.I18nService i18nService;
|
||||
private final vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker;
|
||||
|
||||
/**
|
||||
* 根据 AgentEntity 构建完整的 Agent 实例
|
||||
@ -258,8 +259,8 @@ public class AgentGraphBuilder {
|
||||
CompiledGraph buildPlanExecuteGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations,
|
||||
String reasoningEffort, ModelConfigEntity primaryModelConfig) {
|
||||
try {
|
||||
List<ChatModel> fallbackChain = buildFallbackChain(primaryModelConfig);
|
||||
NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator);
|
||||
List<vip.mate.llm.failover.FallbackEntry> fallbackChain = buildFallbackChain(primaryModelConfig);
|
||||
NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator, providerHealthTracker);
|
||||
ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, toolGuardService, approvalService, streamTracker, toolTimeoutProperties, toolResultStorage, toolConcurrencyRegistry);
|
||||
PlanGenerationNode planGenerationNode = new PlanGenerationNode(chatModel, planningService, streamingHelper, conversationWindowManager, toolSet);
|
||||
StepExecutionNode stepExecutionNode = new StepExecutionNode(chatModel, toolSet, executor, planningService, streamTracker, reasoningEffort, streamingHelper, conversationWindowManager);
|
||||
@ -357,8 +358,8 @@ public class AgentGraphBuilder {
|
||||
CompiledGraph buildReActGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations,
|
||||
String reasoningEffort, ModelConfigEntity primaryModelConfig) {
|
||||
try {
|
||||
List<ChatModel> fallbackChain = buildFallbackChain(primaryModelConfig);
|
||||
NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator);
|
||||
List<vip.mate.llm.failover.FallbackEntry> fallbackChain = buildFallbackChain(primaryModelConfig);
|
||||
NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator, providerHealthTracker);
|
||||
ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, toolGuardService, approvalService, streamTracker, toolTimeoutProperties, toolResultStorage, toolConcurrencyRegistry);
|
||||
ReasoningNode reasoningNode = new ReasoningNode(chatModel, toolSet, reasoningEffort, streamingHelper, conversationWindowManager, streamTracker, 0, wikiContextService);
|
||||
ActionNode actionNode = new ActionNode(executor, streamTracker);
|
||||
@ -560,7 +561,7 @@ public class AgentGraphBuilder {
|
||||
* the primary model; used to identity-filter the chain
|
||||
* @return ordered, possibly-empty list of fallback {@link ChatModel}s
|
||||
*/
|
||||
List<ChatModel> buildFallbackChain(ModelConfigEntity primaryModelConfig) {
|
||||
List<vip.mate.llm.failover.FallbackEntry> buildFallbackChain(ModelConfigEntity primaryModelConfig) {
|
||||
List<ModelProviderEntity> providers;
|
||||
try {
|
||||
providers = modelProviderService.listFallbackChain();
|
||||
@ -575,7 +576,7 @@ public class AgentGraphBuilder {
|
||||
String primaryProviderId = primaryModelConfig != null ? primaryModelConfig.getProvider() : null;
|
||||
String primaryModelName = primaryModelConfig != null ? primaryModelConfig.getModelName() : null;
|
||||
|
||||
List<ChatModel> chain = new ArrayList<>();
|
||||
List<vip.mate.llm.failover.FallbackEntry> chain = new ArrayList<>();
|
||||
for (ModelProviderEntity p : providers) {
|
||||
ModelConfigEntity fallbackConfig;
|
||||
try {
|
||||
@ -600,7 +601,7 @@ public class AgentGraphBuilder {
|
||||
}
|
||||
try {
|
||||
ChatModel m = buildRuntimeChatModel(fallbackConfig, RetryTemplate.builder().maxAttempts(1).build());
|
||||
chain.add(m);
|
||||
chain.add(new vip.mate.llm.failover.FallbackEntry(p.getProviderId(), m));
|
||||
log.info("[LlmFailover] chain[{}] = {}/{} (priority={})",
|
||||
chain.size(), p.getProviderId(), fallbackConfig.getModelName(),
|
||||
p.getFallbackPriority());
|
||||
|
||||
@ -48,40 +48,67 @@ public class NodeStreamingChatHelper {
|
||||
* Ordered fallback chain tried after the primary model exhausts retries.
|
||||
* Each entry is attempted once (no retry); the first successful response
|
||||
* wins. Empty list disables fallover entirely.
|
||||
*
|
||||
* <p>Stored as {@link vip.mate.llm.failover.FallbackEntry} (providerId +
|
||||
* ChatModel) so the chain walker can consult {@link vip.mate.llm.failover.ProviderHealthTracker}
|
||||
* — cooldown state is keyed by providerId, not by ChatModel instance.</p>
|
||||
*/
|
||||
private final List<ChatModel> fallbackChain;
|
||||
private final List<vip.mate.llm.failover.FallbackEntry> fallbackChain;
|
||||
|
||||
/** Optional cache-metrics aggregator; {@code null} in tests or when the bean is absent. */
|
||||
private final vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics;
|
||||
|
||||
/** Optional per-provider health tracker; {@code null} in tests or when bean absent. */
|
||||
private final vip.mate.llm.failover.ProviderHealthTracker healthTracker;
|
||||
|
||||
public NodeStreamingChatHelper(ChatStreamTracker streamTracker) {
|
||||
this(streamTracker, List.of(), null);
|
||||
this(streamTracker, List.of(), null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated use the list-based constructor — a single fallback cannot
|
||||
* express the ordered multi-provider chain
|
||||
* @deprecated use the list-based constructor with FallbackEntry — a single
|
||||
* fallback cannot express the ordered multi-provider chain
|
||||
*/
|
||||
@Deprecated
|
||||
public NodeStreamingChatHelper(ChatStreamTracker streamTracker, ChatModel fallbackModel) {
|
||||
this(streamTracker, fallbackModel == null ? List.of() : List.of(fallbackModel), null);
|
||||
this(streamTracker, wrap(fallbackModel), null, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated use the list-based constructor — a single fallback cannot
|
||||
* express the ordered multi-provider chain
|
||||
* @deprecated use the list-based constructor with FallbackEntry — a single
|
||||
* fallback cannot express the ordered multi-provider chain
|
||||
*/
|
||||
@Deprecated
|
||||
public NodeStreamingChatHelper(ChatStreamTracker streamTracker, ChatModel fallbackModel,
|
||||
vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics) {
|
||||
this(streamTracker, fallbackModel == null ? List.of() : List.of(fallbackModel), cacheMetrics);
|
||||
this(streamTracker, wrap(fallbackModel), cacheMetrics, null);
|
||||
}
|
||||
|
||||
public NodeStreamingChatHelper(ChatStreamTracker streamTracker, List<ChatModel> fallbackChain,
|
||||
/**
|
||||
* Full chain constructor without health tracker — primarily for tests and
|
||||
* legacy wiring. Production callers should use the 4-arg variant so
|
||||
* cooldown state is honored.
|
||||
*/
|
||||
public NodeStreamingChatHelper(ChatStreamTracker streamTracker,
|
||||
List<vip.mate.llm.failover.FallbackEntry> fallbackChain,
|
||||
vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics) {
|
||||
this(streamTracker, fallbackChain, cacheMetrics, null);
|
||||
}
|
||||
|
||||
public NodeStreamingChatHelper(ChatStreamTracker streamTracker,
|
||||
List<vip.mate.llm.failover.FallbackEntry> fallbackChain,
|
||||
vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics,
|
||||
vip.mate.llm.failover.ProviderHealthTracker healthTracker) {
|
||||
this.streamTracker = streamTracker;
|
||||
this.fallbackChain = fallbackChain == null ? List.of() : List.copyOf(fallbackChain);
|
||||
this.cacheMetrics = cacheMetrics;
|
||||
this.healthTracker = healthTracker;
|
||||
}
|
||||
|
||||
private static List<vip.mate.llm.failover.FallbackEntry> wrap(ChatModel m) {
|
||||
// Legacy single-fallback path: providerId is unknown so health tracking
|
||||
// is silently disabled for that one entry (it gets a synthetic id).
|
||||
return m == null ? List.of() : List.of(new vip.mate.llm.failover.FallbackEntry("__legacy__", m));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -276,11 +303,19 @@ public class NodeStreamingChatHelper {
|
||||
// Each fallback gets a single shot (no retry); first successful result wins.
|
||||
// Same-instance entries (e.g., primary accidentally included in the chain)
|
||||
// are skipped so we don't re-try the exact model that just failed.
|
||||
// Providers in cooldown are also skipped so a known-bad
|
||||
// provider doesn't add latency to every conversation turn.
|
||||
for (int i = 0; i < fallbackChain.size(); i++) {
|
||||
ChatModel fallback = fallbackChain.get(i);
|
||||
vip.mate.llm.failover.FallbackEntry entry = fallbackChain.get(i);
|
||||
ChatModel fallback = entry.chatModel();
|
||||
if (fallback == chatModel) continue;
|
||||
log.warn("[{}] Primary exhausted, trying fallback {}/{} ({}) for conversation {}",
|
||||
phase, i + 1, fallbackChain.size(),
|
||||
if (healthTracker != null && healthTracker.isInCooldown(entry.providerId())) {
|
||||
log.info("[{}] Skipping fallback {}/{} provider={} — in cooldown",
|
||||
phase, i + 1, fallbackChain.size(), entry.providerId());
|
||||
continue;
|
||||
}
|
||||
log.warn("[{}] Primary exhausted, trying fallback {}/{} provider={} ({}) for conversation {}",
|
||||
phase, i + 1, fallbackChain.size(), entry.providerId(),
|
||||
fallback.getClass().getSimpleName(), conversationId);
|
||||
if (broadcast) {
|
||||
broadcastDelta(conversationId, "warning",
|
||||
@ -294,8 +329,10 @@ public class NodeStreamingChatHelper {
|
||||
if (fallbackResult != null
|
||||
&& fallbackResult.errorType() == ErrorType.NONE
|
||||
&& fallbackResult.errorMessage() == null) {
|
||||
if (healthTracker != null) healthTracker.recordSuccess(entry.providerId());
|
||||
return fallbackResult;
|
||||
}
|
||||
if (healthTracker != null) healthTracker.recordFailure(entry.providerId());
|
||||
if (fallbackResult != null) {
|
||||
lastResult = fallbackResult; // remember most recent to report if the whole chain fails
|
||||
}
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
package vip.mate.llm.failover;
|
||||
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
|
||||
/**
|
||||
* a single rung of the multi-model failover chain. Pairs a
|
||||
* {@link ChatModel} with the {@code providerId} that built it so the
|
||||
* chain walker can consult {@link ProviderHealthTracker} (which keys cooldown
|
||||
* state by provider id, not by ChatModel instance).
|
||||
*
|
||||
* @param providerId db key of the provider — must match {@code mate_model_provider.provider_id}
|
||||
* @param chatModel the actual model client to invoke
|
||||
*/
|
||||
public record FallbackEntry(String providerId, ChatModel chatModel) {}
|
||||
@ -0,0 +1,42 @@
|
||||
package vip.mate.llm.failover;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
/**
|
||||
* thresholds for the per-provider health tracker.
|
||||
*
|
||||
* <pre>
|
||||
* mateclaw:
|
||||
* llm:
|
||||
* failover:
|
||||
* health:
|
||||
* enabled: true
|
||||
* failure-threshold: 3 # consecutive failures before cooldown
|
||||
* cooldown-ms: 300000 # 5 minutes
|
||||
* </pre>
|
||||
*/
|
||||
@ConfigurationProperties(prefix = "mateclaw.llm.failover.health")
|
||||
public class ProviderHealthProperties {
|
||||
|
||||
/** Master switch. {@code false} disables tracking entirely; the chain walker tries every provider every time. */
|
||||
private boolean enabled = true;
|
||||
|
||||
/** Consecutive failures (any kind) at which a provider enters cooldown. */
|
||||
private int failureThreshold = 3;
|
||||
|
||||
/** Cooldown window in milliseconds. */
|
||||
private long cooldownMs = 300_000L;
|
||||
|
||||
public boolean isEnabled() { return enabled; }
|
||||
public void setEnabled(boolean enabled) { this.enabled = enabled; }
|
||||
|
||||
public int getFailureThreshold() { return failureThreshold; }
|
||||
public void setFailureThreshold(int failureThreshold) {
|
||||
this.failureThreshold = Math.max(1, failureThreshold);
|
||||
}
|
||||
|
||||
public long getCooldownMs() { return cooldownMs; }
|
||||
public void setCooldownMs(long cooldownMs) {
|
||||
this.cooldownMs = Math.max(1000, cooldownMs);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,120 @@
|
||||
package vip.mate.llm.failover;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* per-provider health tracker for the multi-model failover chain.
|
||||
*
|
||||
* <p>Tracks consecutive failure counts per provider id. When a provider hits
|
||||
* {@link ProviderHealthProperties#getFailureThreshold} consecutive failures,
|
||||
* it enters a cooldown window of {@link ProviderHealthProperties#getCooldownMs}
|
||||
* milliseconds during which the chain walker skips it entirely. A successful
|
||||
* call resets the counter and clears the cooldown immediately.</p>
|
||||
*
|
||||
* <p>Without this guard, a provider whose API key has been revoked or whose
|
||||
* service is degraded would be re-tried on every conversation turn, eating
|
||||
* the full retry budget and adding seconds of latency before the chain
|
||||
* advances. With it, the first round trips through the broken provider; the
|
||||
* next N turns skip it instantly and try the next entry directly.</p>
|
||||
*
|
||||
* <p>State is in-memory only — process restart resets all counters. That is
|
||||
* intentional for v1: per-process tracking is enough for desktop / single-node
|
||||
* deployments, and avoids the complication of distributed coordination. A
|
||||
* future RFC may upgrade this to a {@code mate_provider_health} row.</p>
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@Configuration
|
||||
@EnableConfigurationProperties(ProviderHealthProperties.class)
|
||||
public class ProviderHealthTracker {
|
||||
|
||||
private final ProviderHealthProperties props;
|
||||
private final ConcurrentHashMap<String, AtomicLong> consecutiveFailures = new ConcurrentHashMap<>();
|
||||
private final ConcurrentHashMap<String, Long> cooldownUntilMs = new ConcurrentHashMap<>();
|
||||
|
||||
public ProviderHealthTracker(ProviderHealthProperties props) {
|
||||
this.props = props;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when {@code providerId} is in cooldown and should be
|
||||
* skipped by the fallback chain walker. Lazily expires stale cooldowns
|
||||
* during the lookup so we don't accumulate dead entries.
|
||||
*/
|
||||
public boolean isInCooldown(String providerId) {
|
||||
if (!props.isEnabled() || providerId == null) return false;
|
||||
Long until = cooldownUntilMs.get(providerId);
|
||||
if (until == null) return false;
|
||||
if (System.currentTimeMillis() >= until) {
|
||||
// Cooldown expired — clear so the provider can be tried again.
|
||||
cooldownUntilMs.remove(providerId);
|
||||
consecutiveFailures.computeIfPresent(providerId, (k, v) -> { v.set(0); return v; });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a single failure against {@code providerId}. When the counter
|
||||
* reaches the configured threshold, the provider enters cooldown.
|
||||
*/
|
||||
public void recordFailure(String providerId) {
|
||||
if (!props.isEnabled() || providerId == null) return;
|
||||
AtomicLong counter = consecutiveFailures.computeIfAbsent(providerId, k -> new AtomicLong());
|
||||
long failures = counter.incrementAndGet();
|
||||
if (failures >= props.getFailureThreshold()) {
|
||||
long cooldownEnd = System.currentTimeMillis() + props.getCooldownMs();
|
||||
cooldownUntilMs.put(providerId, cooldownEnd);
|
||||
log.warn("[ProviderHealth] provider={} hit {} consecutive failures, entering cooldown for {}s",
|
||||
providerId, failures, props.getCooldownMs() / 1000);
|
||||
}
|
||||
}
|
||||
|
||||
/** Successful call: reset the failure counter and clear any active cooldown. */
|
||||
public void recordSuccess(String providerId) {
|
||||
if (!props.isEnabled() || providerId == null) return;
|
||||
AtomicLong counter = consecutiveFailures.get(providerId);
|
||||
if (counter != null) counter.set(0);
|
||||
if (cooldownUntilMs.remove(providerId) != null) {
|
||||
log.info("[ProviderHealth] provider={} recovered, cooldown cleared", providerId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Diagnostic snapshot for admin endpoints / tests. Returns a stable view
|
||||
* with provider id → (failures, cooldown remaining ms; 0 if not in cooldown).
|
||||
*/
|
||||
public Map<String, ProviderHealthSnapshot> snapshot() {
|
||||
Map<String, ProviderHealthSnapshot> out = new LinkedHashMap<>();
|
||||
long now = System.currentTimeMillis();
|
||||
consecutiveFailures.forEach((id, counter) -> {
|
||||
Long until = cooldownUntilMs.get(id);
|
||||
long remaining = until != null && until > now ? until - now : 0;
|
||||
out.put(id, new ProviderHealthSnapshot(counter.get(), remaining));
|
||||
});
|
||||
// Include cooldown-only entries (failure counter may have been zeroed by snapshot timing race).
|
||||
cooldownUntilMs.forEach((id, until) -> {
|
||||
if (!out.containsKey(id)) {
|
||||
long remaining = until > now ? until - now : 0;
|
||||
out.put(id, new ProviderHealthSnapshot(0, remaining));
|
||||
}
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Test-only: drop all state. Not exposed via any public bean method. */
|
||||
void reset() {
|
||||
consecutiveFailures.clear();
|
||||
cooldownUntilMs.clear();
|
||||
}
|
||||
|
||||
public record ProviderHealthSnapshot(long consecutiveFailures, long cooldownRemainingMs) {}
|
||||
}
|
||||
@ -11,4 +11,10 @@ public class ProviderConfigRequest {
|
||||
private String protocol;
|
||||
private String chatModel;
|
||||
private Map<String, Object> generateKwargs;
|
||||
/**
|
||||
* provider's position in the multi-model failover chain.
|
||||
* {@code 0} = excluded; positive ints define ascending try-order. When
|
||||
* {@code null} the field is left untouched on update.
|
||||
*/
|
||||
private Integer fallbackPriority;
|
||||
}
|
||||
|
||||
@ -29,4 +29,6 @@ public class ProviderInfoDTO {
|
||||
private String authType;
|
||||
private Boolean oauthConnected;
|
||||
private Long oauthExpiresAt;
|
||||
/** position in the failover chain (0 = excluded, 1..N = priority). */
|
||||
private Integer fallbackPriority;
|
||||
}
|
||||
|
||||
@ -72,6 +72,12 @@ public class ModelProviderService {
|
||||
provider.setBaseUrl(request.getBaseUrl());
|
||||
provider.setChatModel(ModelProtocol.resolveChatModel(request.getProtocol(), request.getChatModel()));
|
||||
provider.setGenerateKwargs(writeJson(request.getGenerateKwargs()));
|
||||
// only update fallback priority when the caller explicitly
|
||||
// sends a value. null leaves it untouched (existing chain unchanged).
|
||||
if (request.getFallbackPriority() != null) {
|
||||
int p = Math.max(0, request.getFallbackPriority());
|
||||
provider.setFallbackPriority(p);
|
||||
}
|
||||
modelProviderMapper.updateById(provider);
|
||||
tryAutoActivateModel(providerId, provider);
|
||||
eventPublisher.publishEvent(new ModelConfigChangedEvent("provider-config-updated"));
|
||||
@ -238,6 +244,7 @@ public class ModelProviderService {
|
||||
dto.setAuthType(provider.getAuthType() != null ? provider.getAuthType() : "api_key");
|
||||
dto.setOauthConnected(StringUtils.hasText(provider.getOauthAccessToken()));
|
||||
dto.setOauthExpiresAt(provider.getOauthExpiresAt());
|
||||
dto.setFallbackPriority(provider.getFallbackPriority() != null ? provider.getFallbackPriority() : 0);
|
||||
List<ModelInfoDTO> builtinModels = new ArrayList<>();
|
||||
List<ModelInfoDTO> extraModels = new ArrayList<>();
|
||||
if (models != null) {
|
||||
|
||||
@ -133,6 +133,15 @@ mateclaw:
|
||||
enabled: true # 自适应降级（连续 miss 后短路到 NoOp，冷却后恢复）
|
||||
miss-threshold: 5
|
||||
cool-down-ms: 60000
|
||||
# per-provider health tracker for the multi-model failover chain.
|
||||
# When a provider hits failure-threshold consecutive failures it enters a cooldown
|
||||
# window during which the chain walker skips it, avoiding repeated 5-retry stalls
|
||||
# against a known-broken provider on every conversation turn.
|
||||
failover:
|
||||
health:
|
||||
enabled: true
|
||||
failure-threshold: 3
|
||||
cooldown-ms: 300000
|
||||
|
||||
# MateClaw Agent 配置
|
||||
mate:
|
||||
|
||||
@ -4,6 +4,7 @@ import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import vip.mate.channel.web.ChatStreamTracker;
|
||||
import vip.mate.llm.failover.FallbackEntry;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.List;
|
||||
@ -25,24 +26,31 @@ class NodeStreamingChatHelperFallbackChainTest {
|
||||
private final ChatStreamTracker streamTracker = mock(ChatStreamTracker.class);
|
||||
|
||||
@Test
|
||||
@DisplayName("List-based constructor preserves fallback chain order and contents")
|
||||
@DisplayName("List-based constructor preserves fallback chain order, providerId, and ChatModel")
|
||||
void listConstructorPreservesOrder() throws Exception {
|
||||
ChatModel a = mock(ChatModel.class);
|
||||
ChatModel b = mock(ChatModel.class);
|
||||
ChatModel c = mock(ChatModel.class);
|
||||
NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, List.of(a, b, c), null);
|
||||
List<FallbackEntry> input = List.of(
|
||||
new FallbackEntry("openai", a),
|
||||
new FallbackEntry("dashscope", b),
|
||||
new FallbackEntry("anthropic", c));
|
||||
NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, input, null);
|
||||
|
||||
List<ChatModel> chain = readFallbackChain(helper);
|
||||
List<FallbackEntry> chain = readFallbackChain(helper);
|
||||
assertEquals(3, chain.size(), "fallback chain should preserve all entries");
|
||||
assertSame(a, chain.get(0), "priority 1 must be first");
|
||||
assertSame(b, chain.get(1));
|
||||
assertSame(c, chain.get(2));
|
||||
assertEquals("openai", chain.get(0).providerId());
|
||||
assertSame(a, chain.get(0).chatModel());
|
||||
assertEquals("dashscope", chain.get(1).providerId());
|
||||
assertSame(b, chain.get(1).chatModel());
|
||||
assertEquals("anthropic", chain.get(2).providerId());
|
||||
assertSame(c, chain.get(2).chatModel());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Null fallback chain is normalized to empty list (defensive)")
|
||||
void nullChainNormalizedToEmpty() throws Exception {
|
||||
NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, (List<ChatModel>) null, null);
|
||||
NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, (List<FallbackEntry>) null, null);
|
||||
assertTrue(readFallbackChain(helper).isEmpty(),
|
||||
"null chain must not throw — it should be normalized to an empty list");
|
||||
}
|
||||
@ -55,15 +63,19 @@ class NodeStreamingChatHelperFallbackChainTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Deprecated single-fallback constructor wraps the model into a 1-element chain")
|
||||
@DisplayName("Deprecated single-fallback constructor wraps the model into a 1-entry synthetic chain")
|
||||
void deprecatedSingleFallbackConstructorBackCompat() throws Exception {
|
||||
ChatModel single = mock(ChatModel.class);
|
||||
@SuppressWarnings("deprecation")
|
||||
NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, single);
|
||||
|
||||
List<ChatModel> chain = readFallbackChain(helper);
|
||||
List<FallbackEntry> chain = readFallbackChain(helper);
|
||||
assertEquals(1, chain.size(), "deprecated overload should produce a 1-entry chain");
|
||||
assertSame(single, chain.get(0));
|
||||
assertSame(single, chain.get(0).chatModel(),
|
||||
"the single fallback ChatModel must survive wrapping intact");
|
||||
// Synthetic providerId is acceptable; just assert it's present so health
|
||||
// tracking won't NPE on lookup.
|
||||
assertNotNull(chain.get(0).providerId());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -85,9 +97,9 @@ class NodeStreamingChatHelperFallbackChainTest {
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<ChatModel> readFallbackChain(NodeStreamingChatHelper helper) throws Exception {
|
||||
private static List<FallbackEntry> readFallbackChain(NodeStreamingChatHelper helper) throws Exception {
|
||||
Field f = NodeStreamingChatHelper.class.getDeclaredField("fallbackChain");
|
||||
f.setAccessible(true);
|
||||
return (List<ChatModel>) f.get(helper);
|
||||
return (List<FallbackEntry>) f.get(helper);
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,117 @@
|
||||
package vip.mate.llm.failover;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* per-provider failure-count + cooldown logic.
|
||||
*/
|
||||
class ProviderHealthTrackerTest {
|
||||
|
||||
private ProviderHealthProperties props;
|
||||
private ProviderHealthTracker tracker;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
props = new ProviderHealthProperties();
|
||||
props.setFailureThreshold(3);
|
||||
props.setCooldownMs(60_000L);
|
||||
tracker = new ProviderHealthTracker(props);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("New provider is not in cooldown")
|
||||
void newProviderNotInCooldown() {
|
||||
assertFalse(tracker.isInCooldown("openai"));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Failures below threshold do not trigger cooldown")
|
||||
void belowThresholdNoCooldown() {
|
||||
tracker.recordFailure("openai");
|
||||
tracker.recordFailure("openai");
|
||||
assertFalse(tracker.isInCooldown("openai"),
|
||||
"two failures < threshold of 3 must not enter cooldown");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Failures hitting threshold enter cooldown")
|
||||
void thresholdReachedTriggersCooldown() {
|
||||
tracker.recordFailure("openai");
|
||||
tracker.recordFailure("openai");
|
||||
tracker.recordFailure("openai");
|
||||
assertTrue(tracker.isInCooldown("openai"),
|
||||
"third failure must enter cooldown");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Success resets failure counter and clears cooldown")
|
||||
void successResetsCounterAndCooldown() {
|
||||
tracker.recordFailure("openai");
|
||||
tracker.recordFailure("openai");
|
||||
tracker.recordFailure("openai");
|
||||
assertTrue(tracker.isInCooldown("openai"));
|
||||
|
||||
tracker.recordSuccess("openai");
|
||||
assertFalse(tracker.isInCooldown("openai"),
|
||||
"success must clear cooldown so the provider becomes eligible again");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("After cooldown expires, provider becomes eligible again")
|
||||
void cooldownExpires() throws Exception {
|
||||
// Bypass the min-1000ms clamp in setCooldownMs via reflection — the
|
||||
// clamp is there to prevent prod misconfiguration, but for this test
|
||||
// we want a fast-expiring window to avoid sleeping 1+ seconds.
|
||||
java.lang.reflect.Field f = ProviderHealthProperties.class.getDeclaredField("cooldownMs");
|
||||
f.setAccessible(true);
|
||||
f.setLong(props, 50L);
|
||||
|
||||
for (int i = 0; i < 3; i++) tracker.recordFailure("openai");
|
||||
assertTrue(tracker.isInCooldown("openai"), "sanity: still in cooldown right after trigger");
|
||||
Thread.sleep(120);
|
||||
assertFalse(tracker.isInCooldown("openai"),
|
||||
"cooldown should expire once the window has passed");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Disabled tracker never reports cooldown")
|
||||
void disabledTrackerInert() {
|
||||
props.setEnabled(false);
|
||||
for (int i = 0; i < 10; i++) tracker.recordFailure("openai");
|
||||
assertFalse(tracker.isInCooldown("openai"),
|
||||
"disabled tracker must report no cooldown regardless of failures");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Null providerId is a safe no-op")
|
||||
void nullProviderIdSafe() {
|
||||
tracker.recordFailure(null);
|
||||
tracker.recordSuccess(null);
|
||||
assertFalse(tracker.isInCooldown(null),
|
||||
"null providerId must not crash and must report no cooldown");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Per-provider isolation: cooldown on A does not affect B")
|
||||
void perProviderIsolation() {
|
||||
for (int i = 0; i < 3; i++) tracker.recordFailure("openai");
|
||||
assertTrue(tracker.isInCooldown("openai"));
|
||||
assertFalse(tracker.isInCooldown("dashscope"),
|
||||
"cooldown must be scoped per provider id");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Snapshot reports both failure count and remaining cooldown")
|
||||
void snapshotReportsState() {
|
||||
for (int i = 0; i < 3; i++) tracker.recordFailure("openai");
|
||||
var snap = tracker.snapshot();
|
||||
assertNotNull(snap.get("openai"));
|
||||
assertEquals(3L, snap.get("openai").consecutiveFailures());
|
||||
assertTrue(snap.get("openai").cooldownRemainingMs() > 0,
|
||||
"cooldown remaining ms must be positive while active");
|
||||
}
|
||||
}
|
||||
@ -327,6 +327,9 @@ export default {
|
||||
protocolGemini: 'Gemini Native',
|
||||
protocolDashScope: 'DashScope Native',
|
||||
advancedHint: 'Use this for generation options such as temperature, max_tokens, and top_p.',
|
||||
fallbackPriorityHint: 'Failover order after the primary model fails: 0 = excluded, 1 = first in line, 2 = second, and so on. Providers sharing the same value are tried in alphabetical order of their ID.',
|
||||
fallbackBadge: 'Fallback #{priority}',
|
||||
fallbackBadgeTitle: 'Position in the multi-model failover chain — lower numbers are tried first',
|
||||
searchHint: 'When enabled, the LLM will use its built-in search engine to retrieve real-time information (DashScope/Kimi/OpenAI supported).',
|
||||
searchStrategyDefault: 'Default',
|
||||
oauthTitle: 'OpenAI OAuth Login',
|
||||
@ -344,6 +347,7 @@ export default {
|
||||
apiKeyPrefix: 'API Key Prefix',
|
||||
protocol: 'Protocol',
|
||||
generateKwargs: 'Generate Kwargs (JSON)',
|
||||
fallbackPriority: 'Failover Priority',
|
||||
enableSearch: 'Built-in Search',
|
||||
searchStrategy: 'Search Strategy',
|
||||
modelId: 'Model ID',
|
||||
|
||||
@ -317,6 +317,9 @@ export default {
|
||||
protocolGemini: 'Gemini 原生',
|
||||
protocolDashScope: 'DashScope 原生',
|
||||
advancedHint: '用于补充 temperature、max_tokens、top_p 等生成参数。',
|
||||
fallbackPriorityHint: '主模型失败后的失败切换顺序：0 = 不参与；1 = 第一顺位；2 = 第二顺位，依此类推。多个 provider 共用同一数字时按 ID 字典序。',
|
||||
fallbackBadge: '兜底 #{priority}',
|
||||
fallbackBadgeTitle: '该 provider 在多模型失败切换链中，数字越小越先尝试',
|
||||
searchHint: '开启后，大模型将在回答时自动调用内置搜索引擎获取实时信息（DashScope/Kimi/OpenAI 支持）。',
|
||||
searchStrategyDefault: '默认',
|
||||
oauthTitle: 'OpenAI OAuth 登录',
|
||||
@ -334,6 +337,7 @@ export default {
|
||||
apiKeyPrefix: 'API Key 前缀',
|
||||
protocol: '协议',
|
||||
generateKwargs: 'Generate Kwargs (JSON)',
|
||||
fallbackPriority: '失败切换优先级',
|
||||
enableSearch: '内置搜索',
|
||||
searchStrategy: '搜索策略',
|
||||
modelId: '模型 ID',
|
||||
|
||||
@ -602,6 +602,8 @@ export interface ProviderInfo {
|
||||
authType?: string
|
||||
oauthConnected?: boolean
|
||||
oauthExpiresAt?: number
|
||||
/** RFC-009 P3.5: position in the multi-model failover chain (0 = excluded). */
|
||||
fallbackPriority?: number
|
||||
}
|
||||
|
||||
export interface ActiveModelsInfo {
|
||||
|
||||
@ -18,6 +18,15 @@
|
||||
<span v-if="isProviderActive(provider)" class="provider-badge active">
|
||||
{{ t('settings.model.active') }}
|
||||
</span>
|
||||
<!-- RFC-009 P3.5: surface failover priority so users can see at a glance
|
||||
which providers participate in the chain and in what order. -->
|
||||
<span
|
||||
v-if="provider.fallbackPriority && provider.fallbackPriority > 0"
|
||||
class="provider-badge fallback"
|
||||
:title="t('settings.model.fallbackBadgeTitle')"
|
||||
>
|
||||
{{ t('settings.model.fallbackBadge', { priority: provider.fallbackPriority }) }}
|
||||
</span>
|
||||
</div>
|
||||
<p class="provider-id">{{ provider.id }}</p>
|
||||
</div>
|
||||
@ -156,6 +165,7 @@ const { t } = useI18n()
|
||||
.provider-badge.builtin { background: var(--mc-primary-bg); color: var(--mc-primary); }
|
||||
.provider-badge.custom { background: var(--mc-primary-bg); color: var(--mc-primary-hover); }
|
||||
.provider-badge.active { background: rgba(217, 119, 87, 0.12); color: var(--mc-primary-light); }
|
||||
.provider-badge.fallback { background: rgba(99, 102, 241, 0.12); color: #6366f1; cursor: help; }
|
||||
.provider-status { flex-shrink: 0; padding: 4px 10px; border-radius: 999px; font-size: 12px; font-weight: 700; }
|
||||
.provider-status.configured { background: var(--mc-primary-bg); color: var(--mc-primary); }
|
||||
.provider-status.partial { background: var(--mc-primary-bg); color: var(--mc-primary-hover); }
|
||||
|
||||
@ -118,7 +118,21 @@
|
||||
<span>{{ advancedOpen ? '−' : '+' }}</span>
|
||||
</button>
|
||||
<div v-if="advancedOpen" class="advanced-panel">
|
||||
<label class="form-label">{{ t('settings.model.fields.generateKwargs') }}</label>
|
||||
<!-- RFC-009 P3.5: failover chain priority editor.
|
||||
0 = excluded; 1..N defines try-order after primary fails. -->
|
||||
<label class="form-label">{{ t('settings.model.fields.fallbackPriority') }}</label>
|
||||
<input
|
||||
v-model.number="form.fallbackPriority"
|
||||
type="number"
|
||||
min="0"
|
||||
max="99"
|
||||
step="1"
|
||||
class="form-input"
|
||||
style="max-width: 160px"
|
||||
/>
|
||||
<div class="field-hint">{{ t('settings.model.fallbackPriorityHint') }}</div>
|
||||
|
||||
<label class="form-label" style="margin-top: 14px">{{ t('settings.model.fields.generateKwargs') }}</label>
|
||||
<textarea v-model="form.generateKwargsText" rows="6" class="form-textarea mono"></textarea>
|
||||
<div class="field-hint">{{ t('settings.model.advancedHint') }}</div>
|
||||
</div>
|
||||
@ -155,6 +169,7 @@ defineProps<{
|
||||
generateKwargsText: string
|
||||
enableSearch: boolean
|
||||
searchStrategy: string
|
||||
fallbackPriority: number
|
||||
}
|
||||
advancedOpen: boolean
|
||||
protocolOptions: Array<{ value: string; label: string }>
|
||||
|
||||
@ -36,6 +36,9 @@ export function useProviders() {
|
||||
generateKwargsText: '{}',
|
||||
enableSearch: false,
|
||||
searchStrategy: '',
|
||||
// RFC-009 P3.5: position in the multi-model failover chain.
|
||||
// 0 = excluded; positive int = ascending try-order.
|
||||
fallbackPriority: 0,
|
||||
})
|
||||
|
||||
const providerModelForm = reactive({
|
||||
@ -81,6 +84,7 @@ export function useProviders() {
|
||||
generateKwargsText: '{}',
|
||||
enableSearch: false,
|
||||
searchStrategy: '',
|
||||
fallbackPriority: 0,
|
||||
})
|
||||
showProviderModal.value = true
|
||||
}
|
||||
@ -104,6 +108,7 @@ export function useProviders() {
|
||||
generateKwargsText: JSON.stringify(kwargs, null, 2),
|
||||
enableSearch: searchDefault,
|
||||
searchStrategy: (kwargs.searchStrategy as string) || '',
|
||||
fallbackPriority: provider.fallbackPriority ?? 0,
|
||||
})
|
||||
showProviderModal.value = true
|
||||
}
|
||||
@ -128,6 +133,8 @@ export function useProviders() {
|
||||
delete kwargs.enableSearch
|
||||
delete kwargs.searchStrategy
|
||||
}
|
||||
// RFC-009 P3.5: clamp to non-negative, coerce string input back to integer.
|
||||
const fallbackPriority = Math.max(0, Math.floor(Number(providerForm.fallbackPriority) || 0))
|
||||
if (editingProvider.value) {
|
||||
await modelApi.updateProviderConfig(editingProvider.value.id, {
|
||||
apiKey: providerForm.apiKey,
|
||||
@ -135,6 +142,7 @@ export function useProviders() {
|
||||
protocol: providerForm.protocol,
|
||||
chatModel: protocolToChatModel(providerForm.protocol),
|
||||
generateKwargs: kwargs,
|
||||
fallbackPriority,
|
||||
})
|
||||
} else {
|
||||
await modelApi.createCustomProvider({
|
||||
@ -146,13 +154,14 @@ export function useProviders() {
|
||||
chatModel: protocolToChatModel(providerForm.protocol),
|
||||
models: [],
|
||||
})
|
||||
if (providerForm.apiKey || providerForm.generateKwargsText) {
|
||||
if (providerForm.apiKey || providerForm.generateKwargsText || fallbackPriority > 0) {
|
||||
await modelApi.updateProviderConfig(providerForm.id, {
|
||||
apiKey: providerForm.apiKey,
|
||||
baseUrl: providerForm.baseUrl,
|
||||
protocol: providerForm.protocol,
|
||||
chatModel: protocolToChatModel(providerForm.protocol),
|
||||
generateKwargs: kwargs,
|
||||
fallbackPriority,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user