diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 742cda7f..209ad338 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -130,6 +130,7 @@ public class AgentGraphBuilder { private final vip.mate.agent.graph.executor.ToolResultStorage toolResultStorage; private final vip.mate.tool.ToolConcurrencyRegistry toolConcurrencyRegistry; private final vip.mate.i18n.I18nService i18nService; + private final vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker; /** * 根据 AgentEntity 构建完整的 Agent 实例 @@ -258,8 +259,8 @@ public class AgentGraphBuilder { CompiledGraph buildPlanExecuteGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, String reasoningEffort, ModelConfigEntity primaryModelConfig) { try { - List fallbackChain = buildFallbackChain(primaryModelConfig); - NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator); + List fallbackChain = buildFallbackChain(primaryModelConfig); + NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator, providerHealthTracker); ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, toolGuardService, approvalService, streamTracker, toolTimeoutProperties, toolResultStorage, toolConcurrencyRegistry); PlanGenerationNode planGenerationNode = new PlanGenerationNode(chatModel, planningService, streamingHelper, conversationWindowManager, toolSet); StepExecutionNode stepExecutionNode = new StepExecutionNode(chatModel, toolSet, executor, planningService, streamTracker, reasoningEffort, streamingHelper, conversationWindowManager); @@ -357,8 +358,8 @@ public class AgentGraphBuilder { CompiledGraph buildReActGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, String reasoningEffort, ModelConfigEntity primaryModelConfig) { try { - List fallbackChain = buildFallbackChain(primaryModelConfig); - NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator); + List fallbackChain = buildFallbackChain(primaryModelConfig); + NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper(streamTracker, fallbackChain, llmCacheMetricsAggregator, providerHealthTracker); ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, toolGuardService, approvalService, streamTracker, toolTimeoutProperties, toolResultStorage, toolConcurrencyRegistry); ReasoningNode reasoningNode = new ReasoningNode(chatModel, toolSet, reasoningEffort, streamingHelper, conversationWindowManager, streamTracker, 0, wikiContextService); ActionNode actionNode = new ActionNode(executor, streamTracker); @@ -560,7 +561,7 @@ public class AgentGraphBuilder { * the primary model; used to identity-filter the chain * @return ordered, possibly-empty list of fallback {@link ChatModel}s */ - List buildFallbackChain(ModelConfigEntity primaryModelConfig) { + List buildFallbackChain(ModelConfigEntity primaryModelConfig) { List providers; try { providers = modelProviderService.listFallbackChain(); @@ -575,7 +576,7 @@ public class AgentGraphBuilder { String primaryProviderId = primaryModelConfig != null ? primaryModelConfig.getProvider() : null; String primaryModelName = primaryModelConfig != null ? primaryModelConfig.getModelName() : null; - List chain = new ArrayList<>(); + List chain = new ArrayList<>(); for (ModelProviderEntity p : providers) { ModelConfigEntity fallbackConfig; try { @@ -600,7 +601,7 @@ public class AgentGraphBuilder { } try { ChatModel m = buildRuntimeChatModel(fallbackConfig, RetryTemplate.builder().maxAttempts(1).build()); - chain.add(m); + chain.add(new vip.mate.llm.failover.FallbackEntry(p.getProviderId(), m)); log.info("[LlmFailover] chain[{}] = {}/{} (priority={})", chain.size(), p.getProviderId(), fallbackConfig.getModelName(), p.getFallbackPriority()); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java index 9a533cec..e0e3c175 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java @@ -48,40 +48,67 @@ public class NodeStreamingChatHelper { * Ordered fallback chain tried after the primary model exhausts retries. * Each entry is attempted once (no retry); the first successful response * wins. Empty list disables fallover entirely. + * + *

Stored as {@link vip.mate.llm.failover.FallbackEntry} (providerId + + * ChatModel) so the chain walker can consult {@link vip.mate.llm.failover.ProviderHealthTracker} + * — cooldown state is keyed by providerId, not by ChatModel instance.

*/ - private final List fallbackChain; + private final List fallbackChain; /** Optional cache-metrics aggregator; {@code null} in tests or when the bean is absent. */ private final vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics; + /** Optional per-provider health tracker; {@code null} in tests or when bean absent. */ + private final vip.mate.llm.failover.ProviderHealthTracker healthTracker; + public NodeStreamingChatHelper(ChatStreamTracker streamTracker) { - this(streamTracker, List.of(), null); + this(streamTracker, List.of(), null, null); } /** - * @deprecated use the list-based constructor — a single fallback cannot - * express the ordered multi-provider chain + * @deprecated use the list-based constructor with FallbackEntry — a single + * fallback cannot express the ordered multi-provider chain */ @Deprecated public NodeStreamingChatHelper(ChatStreamTracker streamTracker, ChatModel fallbackModel) { - this(streamTracker, fallbackModel == null ? List.of() : List.of(fallbackModel), null); + this(streamTracker, wrap(fallbackModel), null, null); } /** - * @deprecated use the list-based constructor — a single fallback cannot - * express the ordered multi-provider chain + * @deprecated use the list-based constructor with FallbackEntry — a single + * fallback cannot express the ordered multi-provider chain */ @Deprecated public NodeStreamingChatHelper(ChatStreamTracker streamTracker, ChatModel fallbackModel, vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics) { - this(streamTracker, fallbackModel == null ? List.of() : List.of(fallbackModel), cacheMetrics); + this(streamTracker, wrap(fallbackModel), cacheMetrics, null); } - public NodeStreamingChatHelper(ChatStreamTracker streamTracker, List fallbackChain, + /** + * Full chain constructor without health tracker — primarily for tests and + * legacy wiring. Production callers should use the 4-arg variant so + * cooldown state is honored. + */ + public NodeStreamingChatHelper(ChatStreamTracker streamTracker, + List fallbackChain, vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics) { + this(streamTracker, fallbackChain, cacheMetrics, null); + } + + public NodeStreamingChatHelper(ChatStreamTracker streamTracker, + List fallbackChain, + vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics, + vip.mate.llm.failover.ProviderHealthTracker healthTracker) { this.streamTracker = streamTracker; this.fallbackChain = fallbackChain == null ? List.of() : List.copyOf(fallbackChain); this.cacheMetrics = cacheMetrics; + this.healthTracker = healthTracker; + } + + private static List wrap(ChatModel m) { + // Legacy single-fallback path: providerId is unknown so health tracking + // is silently disabled for that one entry (it gets a synthetic id). + return m == null ? List.of() : List.of(new vip.mate.llm.failover.FallbackEntry("__legacy__", m)); } /** @@ -276,11 +303,19 @@ public class NodeStreamingChatHelper { // Each fallback gets a single shot (no retry); first successful result wins. // Same-instance entries (e.g., primary accidentally included in the chain) // are skipped so we don't re-try the exact model that just failed. + // Providers in cooldown are also skipped so a known-bad + // provider doesn't add latency to every conversation turn. for (int i = 0; i < fallbackChain.size(); i++) { - ChatModel fallback = fallbackChain.get(i); + vip.mate.llm.failover.FallbackEntry entry = fallbackChain.get(i); + ChatModel fallback = entry.chatModel(); if (fallback == chatModel) continue; - log.warn("[{}] Primary exhausted, trying fallback {}/{} ({}) for conversation {}", - phase, i + 1, fallbackChain.size(), + if (healthTracker != null && healthTracker.isInCooldown(entry.providerId())) { + log.info("[{}] Skipping fallback {}/{} provider={} — in cooldown", + phase, i + 1, fallbackChain.size(), entry.providerId()); + continue; + } + log.warn("[{}] Primary exhausted, trying fallback {}/{} provider={} ({}) for conversation {}", + phase, i + 1, fallbackChain.size(), entry.providerId(), fallback.getClass().getSimpleName(), conversationId); if (broadcast) { broadcastDelta(conversationId, "warning", @@ -294,8 +329,10 @@ public class NodeStreamingChatHelper { if (fallbackResult != null && fallbackResult.errorType() == ErrorType.NONE && fallbackResult.errorMessage() == null) { + if (healthTracker != null) healthTracker.recordSuccess(entry.providerId()); return fallbackResult; } + if (healthTracker != null) healthTracker.recordFailure(entry.providerId()); if (fallbackResult != null) { lastResult = fallbackResult; // remember most recent to report if the whole chain fails } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java new file mode 100644 index 00000000..61d2104c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java @@ -0,0 +1,14 @@ +package vip.mate.llm.failover; + +import org.springframework.ai.chat.model.ChatModel; + +/** + * a single rung of the multi-model failover chain. Pairs a + * {@link ChatModel} with the {@code providerId} that built it so the + * chain walker can consult {@link ProviderHealthTracker} (which keys cooldown + * state by provider id, not by ChatModel instance). + * + * @param providerId db key of the provider — must match {@code mate_model_provider.provider_id} + * @param chatModel the actual model client to invoke + */ +public record FallbackEntry(String providerId, ChatModel chatModel) {} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java new file mode 100644 index 00000000..4233b6d2 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java @@ -0,0 +1,42 @@ +package vip.mate.llm.failover; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * thresholds for the per-provider health tracker. + * + * 
+ * mateclaw:
+ *   llm:
+ *     failover:
+ *       health:
+ *         enabled: true
+ *         failure-threshold: 3      # consecutive failures before cooldown
+ *         cooldown-ms: 300000       # 5 minutes
+ *
+ */ +@ConfigurationProperties(prefix = "mateclaw.llm.failover.health") +public class ProviderHealthProperties { + + /** Master switch. {@code false} disables tracking entirely; the chain walker tries every provider every time. */ + private boolean enabled = true; + + /** Consecutive failures (any kind) at which a provider enters cooldown. */ + private int failureThreshold = 3; + + /** Cooldown window in milliseconds. */ + private long cooldownMs = 300_000L; + + public boolean isEnabled() { return enabled; } + public void setEnabled(boolean enabled) { this.enabled = enabled; } + + public int getFailureThreshold() { return failureThreshold; } + public void setFailureThreshold(int failureThreshold) { + this.failureThreshold = Math.max(1, failureThreshold); + } + + public long getCooldownMs() { return cooldownMs; } + public void setCooldownMs(long cooldownMs) { + this.cooldownMs = Math.max(1000, cooldownMs); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java new file mode 100644 index 00000000..02d04dec --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java @@ -0,0 +1,120 @@ +package vip.mate.llm.failover; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * per-provider health tracker for the multi-model failover chain. + * + *

Tracks consecutive failure counts per provider id. When a provider hits + * {@link ProviderHealthProperties#getFailureThreshold} consecutive failures, + * it enters a cooldown window of {@link ProviderHealthProperties#getCooldownMs} + * milliseconds during which the chain walker skips it entirely. A successful + * call resets the counter and clears the cooldown immediately.

+ * + *

Without this guard, a provider whose API key has been revoked or whose + * service is degraded would be re-tried on every conversation turn, eating + * the full retry budget and adding seconds of latency before the chain + * advances. With it, the first round trips through the broken provider; the + * next N turns skip it instantly and try the next entry directly.

+ * + *

State is in-memory only — process restart resets all counters. That is + * intentional for v1: per-process tracking is enough for desktop / single-node + * deployments, and avoids the complication of distributed coordination. A + * future RFC may upgrade this to a {@code mate_provider_health} row.

+ */ +@Slf4j +@Component +@Configuration +@EnableConfigurationProperties(ProviderHealthProperties.class) +public class ProviderHealthTracker { + + private final ProviderHealthProperties props; + private final ConcurrentHashMap consecutiveFailures = new ConcurrentHashMap<>(); + private final ConcurrentHashMap cooldownUntilMs = new ConcurrentHashMap<>(); + + public ProviderHealthTracker(ProviderHealthProperties props) { + this.props = props; + } + + /** + * Returns true when {@code providerId} is in cooldown and should be + * skipped by the fallback chain walker. Lazily expires stale cooldowns + * during the lookup so we don't accumulate dead entries. + */ + public boolean isInCooldown(String providerId) { + if (!props.isEnabled() || providerId == null) return false; + Long until = cooldownUntilMs.get(providerId); + if (until == null) return false; + if (System.currentTimeMillis() >= until) { + // Cooldown expired — clear so the provider can be tried again. + cooldownUntilMs.remove(providerId); + consecutiveFailures.computeIfPresent(providerId, (k, v) -> { v.set(0); return v; }); + return false; + } + return true; + } + + /** + * Record a single failure against {@code providerId}. When the counter + * reaches the configured threshold, the provider enters cooldown. + */ + public void recordFailure(String providerId) { + if (!props.isEnabled() || providerId == null) return; + AtomicLong counter = consecutiveFailures.computeIfAbsent(providerId, k -> new AtomicLong()); + long failures = counter.incrementAndGet(); + if (failures >= props.getFailureThreshold()) { + long cooldownEnd = System.currentTimeMillis() + props.getCooldownMs(); + cooldownUntilMs.put(providerId, cooldownEnd); + log.warn("[ProviderHealth] provider={} hit {} consecutive failures, entering cooldown for {}s", + providerId, failures, props.getCooldownMs() / 1000); + } + } + + /** Successful call: reset the failure counter and clear any active cooldown. */ + public void recordSuccess(String providerId) { + if (!props.isEnabled() || providerId == null) return; + AtomicLong counter = consecutiveFailures.get(providerId); + if (counter != null) counter.set(0); + if (cooldownUntilMs.remove(providerId) != null) { + log.info("[ProviderHealth] provider={} recovered, cooldown cleared", providerId); + } + } + + /** + * Diagnostic snapshot for admin endpoints / tests. Returns a stable view + * with provider id → (failures, cooldown remaining ms; 0 if not in cooldown). + */ + public Map snapshot() { + Map out = new LinkedHashMap<>(); + long now = System.currentTimeMillis(); + consecutiveFailures.forEach((id, counter) -> { + Long until = cooldownUntilMs.get(id); + long remaining = until != null && until > now ? until - now : 0; + out.put(id, new ProviderHealthSnapshot(counter.get(), remaining)); + }); + // Include cooldown-only entries (failure counter may have been zeroed by snapshot timing race). + cooldownUntilMs.forEach((id, until) -> { + if (!out.containsKey(id)) { + long remaining = until > now ? until - now : 0; + out.put(id, new ProviderHealthSnapshot(0, remaining)); + } + }); + return out; + } + + /** Test-only: drop all state. Not exposed via any public bean method. */ + void reset() { + consecutiveFailures.clear(); + cooldownUntilMs.clear(); + } + + public record ProviderHealthSnapshot(long consecutiveFailures, long cooldownRemainingMs) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java index 64b51580..e53d5772 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java @@ -11,4 +11,10 @@ public class ProviderConfigRequest { private String protocol; private String chatModel; private Map generateKwargs; + /** + * provider's position in the multi-model failover chain. + * {@code 0} = excluded; positive ints define ascending try-order. When + * {@code null} the field is left untouched on update. + */ + private Integer fallbackPriority; } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java index 09c48063..75fac2c7 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java @@ -29,4 +29,6 @@ public class ProviderInfoDTO { private String authType; private Boolean oauthConnected; private Long oauthExpiresAt; + /** position in the failover chain (0 = excluded, 1..N = priority). */ + private Integer fallbackPriority; } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java index 74da3e61..575331ae 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java @@ -72,6 +72,12 @@ public class ModelProviderService { provider.setBaseUrl(request.getBaseUrl()); provider.setChatModel(ModelProtocol.resolveChatModel(request.getProtocol(), request.getChatModel())); provider.setGenerateKwargs(writeJson(request.getGenerateKwargs())); + // only update fallback priority when the caller explicitly + // sends a value. null leaves it untouched (existing chain unchanged). + if (request.getFallbackPriority() != null) { + int p = Math.max(0, request.getFallbackPriority()); + provider.setFallbackPriority(p); + } modelProviderMapper.updateById(provider); tryAutoActivateModel(providerId, provider); eventPublisher.publishEvent(new ModelConfigChangedEvent("provider-config-updated")); @@ -238,6 +244,7 @@ public class ModelProviderService { dto.setAuthType(provider.getAuthType() != null ? provider.getAuthType() : "api_key"); dto.setOauthConnected(StringUtils.hasText(provider.getOauthAccessToken())); dto.setOauthExpiresAt(provider.getOauthExpiresAt()); + dto.setFallbackPriority(provider.getFallbackPriority() != null ? provider.getFallbackPriority() : 0); List builtinModels = new ArrayList<>(); List extraModels = new ArrayList<>(); if (models != null) { diff --git a/mateclaw-server/src/main/resources/application.yml b/mateclaw-server/src/main/resources/application.yml index 12adf66b..7863d8ec 100644 --- a/mateclaw-server/src/main/resources/application.yml +++ b/mateclaw-server/src/main/resources/application.yml @@ -133,6 +133,15 @@ mateclaw: enabled: true # 自适应降级（连续 miss 后短路到 NoOp，冷却后恢复） miss-threshold: 5 cool-down-ms: 60000 + # per-provider health tracker for the multi-model failover chain. + # When a provider hits failure-threshold consecutive failures it enters a cooldown + # window during which the chain walker skips it, avoiding repeated 5-retry stalls + # against a known-broken provider on every conversation turn. + failover: + health: + enabled: true + failure-threshold: 3 + cooldown-ms: 300000 # MateClaw Agent 配置 mate: diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java index f9fb59b5..0f6f4296 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.ai.chat.model.ChatModel; import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.llm.failover.FallbackEntry; import java.lang.reflect.Field; import java.util.List; @@ -25,24 +26,31 @@ class NodeStreamingChatHelperFallbackChainTest { private final ChatStreamTracker streamTracker = mock(ChatStreamTracker.class); @Test - @DisplayName("List-based constructor preserves fallback chain order and contents") + @DisplayName("List-based constructor preserves fallback chain order, providerId, and ChatModel") void listConstructorPreservesOrder() throws Exception { ChatModel a = mock(ChatModel.class); ChatModel b = mock(ChatModel.class); ChatModel c = mock(ChatModel.class); - NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, List.of(a, b, c), null); + List input = List.of( + new FallbackEntry("openai", a), + new FallbackEntry("dashscope", b), + new FallbackEntry("anthropic", c)); + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, input, null); - List chain = readFallbackChain(helper); + List chain = readFallbackChain(helper); assertEquals(3, chain.size(), "fallback chain should preserve all entries"); - assertSame(a, chain.get(0), "priority 1 must be first"); - assertSame(b, chain.get(1)); - assertSame(c, chain.get(2)); + assertEquals("openai", chain.get(0).providerId()); + assertSame(a, chain.get(0).chatModel()); + assertEquals("dashscope", chain.get(1).providerId()); + assertSame(b, chain.get(1).chatModel()); + assertEquals("anthropic", chain.get(2).providerId()); + assertSame(c, chain.get(2).chatModel()); } @Test @DisplayName("Null fallback chain is normalized to empty list (defensive)") void nullChainNormalizedToEmpty() throws Exception { - NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, (List) null, null); + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, (List) null, null); assertTrue(readFallbackChain(helper).isEmpty(), "null chain must not throw — it should be normalized to an empty list"); } @@ -55,15 +63,19 @@ class NodeStreamingChatHelperFallbackChainTest { } @Test - @DisplayName("Deprecated single-fallback constructor wraps the model into a 1-element chain") + @DisplayName("Deprecated single-fallback constructor wraps the model into a 1-entry synthetic chain") void deprecatedSingleFallbackConstructorBackCompat() throws Exception { ChatModel single = mock(ChatModel.class); @SuppressWarnings("deprecation") NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, single); - List chain = readFallbackChain(helper); + List chain = readFallbackChain(helper); assertEquals(1, chain.size(), "deprecated overload should produce a 1-entry chain"); - assertSame(single, chain.get(0)); + assertSame(single, chain.get(0).chatModel(), + "the single fallback ChatModel must survive wrapping intact"); + // Synthetic providerId is acceptable; just assert it's present so health + // tracking won't NPE on lookup. + assertNotNull(chain.get(0).providerId()); } @Test @@ -85,9 +97,9 @@ class NodeStreamingChatHelperFallbackChainTest { } @SuppressWarnings("unchecked") - private static List readFallbackChain(NodeStreamingChatHelper helper) throws Exception { + private static List readFallbackChain(NodeStreamingChatHelper helper) throws Exception { Field f = NodeStreamingChatHelper.class.getDeclaredField("fallbackChain"); f.setAccessible(true); - return (List) f.get(helper); + return (List) f.get(helper); } } diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java new file mode 100644 index 00000000..8908c8bd --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java @@ -0,0 +1,117 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * per-provider failure-count + cooldown logic. + */ +class ProviderHealthTrackerTest { + + private ProviderHealthProperties props; + private ProviderHealthTracker tracker; + + @BeforeEach + void setUp() { + props = new ProviderHealthProperties(); + props.setFailureThreshold(3); + props.setCooldownMs(60_000L); + tracker = new ProviderHealthTracker(props); + } + + @Test + @DisplayName("New provider is not in cooldown") + void newProviderNotInCooldown() { + assertFalse(tracker.isInCooldown("openai")); + } + + @Test + @DisplayName("Failures below threshold do not trigger cooldown") + void belowThresholdNoCooldown() { + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + assertFalse(tracker.isInCooldown("openai"), + "two failures < threshold of 3 must not enter cooldown"); + } + + @Test + @DisplayName("Failures hitting threshold enter cooldown") + void thresholdReachedTriggersCooldown() { + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai"), + "third failure must enter cooldown"); + } + + @Test + @DisplayName("Success resets failure counter and clears cooldown") + void successResetsCounterAndCooldown() { + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai")); + + tracker.recordSuccess("openai"); + assertFalse(tracker.isInCooldown("openai"), + "success must clear cooldown so the provider becomes eligible again"); + } + + @Test + @DisplayName("After cooldown expires, provider becomes eligible again") + void cooldownExpires() throws Exception { + // Bypass the min-1000ms clamp in setCooldownMs via reflection — the + // clamp is there to prevent prod misconfiguration, but for this test + // we want a fast-expiring window to avoid sleeping 1+ seconds. + java.lang.reflect.Field f = ProviderHealthProperties.class.getDeclaredField("cooldownMs"); + f.setAccessible(true); + f.setLong(props, 50L); + + for (int i = 0; i < 3; i++) tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai"), "sanity: still in cooldown right after trigger"); + Thread.sleep(120); + assertFalse(tracker.isInCooldown("openai"), + "cooldown should expire once the window has passed"); + } + + @Test + @DisplayName("Disabled tracker never reports cooldown") + void disabledTrackerInert() { + props.setEnabled(false); + for (int i = 0; i < 10; i++) tracker.recordFailure("openai"); + assertFalse(tracker.isInCooldown("openai"), + "disabled tracker must report no cooldown regardless of failures"); + } + + @Test + @DisplayName("Null providerId is a safe no-op") + void nullProviderIdSafe() { + tracker.recordFailure(null); + tracker.recordSuccess(null); + assertFalse(tracker.isInCooldown(null), + "null providerId must not crash and must report no cooldown"); + } + + @Test + @DisplayName("Per-provider isolation: cooldown on A does not affect B") + void perProviderIsolation() { + for (int i = 0; i < 3; i++) tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai")); + assertFalse(tracker.isInCooldown("dashscope"), + "cooldown must be scoped per provider id"); + } + + @Test + @DisplayName("Snapshot reports both failure count and remaining cooldown") + void snapshotReportsState() { + for (int i = 0; i < 3; i++) tracker.recordFailure("openai"); + var snap = tracker.snapshot(); + assertNotNull(snap.get("openai")); + assertEquals(3L, snap.get("openai").consecutiveFailures()); + assertTrue(snap.get("openai").cooldownRemainingMs() > 0, + "cooldown remaining ms must be positive while active"); + } +} diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index e0a5da74..763dda17 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -327,6 +327,9 @@ export default { protocolGemini: 'Gemini Native', protocolDashScope: 'DashScope Native', advancedHint: 'Use this for generation options such as temperature, max_tokens, and top_p.', + fallbackPriorityHint: 'Failover order after the primary model fails: 0 = excluded, 1 = first in line, 2 = second, and so on. Providers sharing the same value are tried in alphabetical order of their ID.', + fallbackBadge: 'Fallback #{priority}', + fallbackBadgeTitle: 'Position in the multi-model failover chain — lower numbers are tried first', searchHint: 'When enabled, the LLM will use its built-in search engine to retrieve real-time information (DashScope/Kimi/OpenAI supported).', searchStrategyDefault: 'Default', oauthTitle: 'OpenAI OAuth Login', @@ -344,6 +347,7 @@ export default { apiKeyPrefix: 'API Key Prefix', protocol: 'Protocol', generateKwargs: 'Generate Kwargs (JSON)', + fallbackPriority: 'Failover Priority', enableSearch: 'Built-in Search', searchStrategy: 'Search Strategy', modelId: 'Model ID', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 07f13689..8a99c3f1 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -317,6 +317,9 @@ export default { protocolGemini: 'Gemini 原生', protocolDashScope: 'DashScope 原生', advancedHint: '用于补充 temperature、max_tokens、top_p 等生成参数。', + fallbackPriorityHint: '主模型失败后的失败切换顺序：0 = 不参与；1 = 第一顺位；2 = 第二顺位，依此类推。多个 provider 共用同一数字时按 ID 字典序。', + fallbackBadge: '兜底 #{priority}', + fallbackBadgeTitle: '该 provider 在多模型失败切换链中，数字越小越先尝试', searchHint: '开启后，大模型将在回答时自动调用内置搜索引擎获取实时信息（DashScope/Kimi/OpenAI 支持）。', searchStrategyDefault: '默认', oauthTitle: 'OpenAI OAuth 登录', @@ -334,6 +337,7 @@ export default { apiKeyPrefix: 'API Key 前缀', protocol: '协议', generateKwargs: 'Generate Kwargs (JSON)', + fallbackPriority: '失败切换优先级', enableSearch: '内置搜索', searchStrategy: '搜索策略', modelId: '模型 ID', diff --git a/mateclaw-ui/src/types/index.ts b/mateclaw-ui/src/types/index.ts index a557087b..f9b319f5 100644 --- a/mateclaw-ui/src/types/index.ts +++ b/mateclaw-ui/src/types/index.ts @@ -602,6 +602,8 @@ export interface ProviderInfo { authType?: string oauthConnected?: boolean oauthExpiresAt?: number + /** RFC-009 P3.5: position in the multi-model failover chain (0 = excluded). */ + fallbackPriority?: number } export interface ActiveModelsInfo { diff --git a/mateclaw-ui/src/views/Settings/Models/ProviderCard.vue b/mateclaw-ui/src/views/Settings/Models/ProviderCard.vue index 032445bc..255da7e1 100644 --- a/mateclaw-ui/src/views/Settings/Models/ProviderCard.vue +++ b/mateclaw-ui/src/views/Settings/Models/ProviderCard.vue @@ -18,6 +18,15 @@ {{ t('settings.model.active') }} + + + {{ t('settings.model.fallbackBadge', { priority: provider.fallbackPriority }) }} +

{{ provider.id }}

@@ -156,6 +165,7 @@ const { t } = useI18n() .provider-badge.builtin { background: var(--mc-primary-bg); color: var(--mc-primary); } .provider-badge.custom { background: var(--mc-primary-bg); color: var(--mc-primary-hover); } .provider-badge.active { background: rgba(217, 119, 87, 0.12); color: var(--mc-primary-light); } +.provider-badge.fallback { background: rgba(99, 102, 241, 0.12); color: #6366f1; cursor: help; } .provider-status { flex-shrink: 0; padding: 4px 10px; border-radius: 999px; font-size: 12px; font-weight: 700; } .provider-status.configured { background: var(--mc-primary-bg); color: var(--mc-primary); } .provider-status.partial { background: var(--mc-primary-bg); color: var(--mc-primary-hover); } diff --git a/mateclaw-ui/src/views/Settings/Models/modals/ProviderConfigModal.vue b/mateclaw-ui/src/views/Settings/Models/modals/ProviderConfigModal.vue index be95bd21..2a494681 100644 --- a/mateclaw-ui/src/views/Settings/Models/modals/ProviderConfigModal.vue +++ b/mateclaw-ui/src/views/Settings/Models/modals/ProviderConfigModal.vue @@ -118,7 +118,21 @@ {{ advancedOpen ? '−' : '+' }}
- + + + +
{{ t('settings.model.fallbackPriorityHint') }}
+ +
{{ t('settings.model.advancedHint') }}
@@ -155,6 +169,7 @@ defineProps<{ generateKwargsText: string enableSearch: boolean searchStrategy: string + fallbackPriority: number } advancedOpen: boolean protocolOptions: Array<{ value: string; label: string }> diff --git a/mateclaw-ui/src/views/Settings/Models/useProviders.ts b/mateclaw-ui/src/views/Settings/Models/useProviders.ts index 605483d5..0b9cc5d0 100644 --- a/mateclaw-ui/src/views/Settings/Models/useProviders.ts +++ b/mateclaw-ui/src/views/Settings/Models/useProviders.ts @@ -36,6 +36,9 @@ export function useProviders() { generateKwargsText: '{}', enableSearch: false, searchStrategy: '', + // RFC-009 P3.5: position in the multi-model failover chain. + // 0 = excluded; positive int = ascending try-order. + fallbackPriority: 0, }) const providerModelForm = reactive({ @@ -81,6 +84,7 @@ export function useProviders() { generateKwargsText: '{}', enableSearch: false, searchStrategy: '', + fallbackPriority: 0, }) showProviderModal.value = true } @@ -104,6 +108,7 @@ export function useProviders() { generateKwargsText: JSON.stringify(kwargs, null, 2), enableSearch: searchDefault, searchStrategy: (kwargs.searchStrategy as string) || '', + fallbackPriority: provider.fallbackPriority ?? 0, }) showProviderModal.value = true } @@ -128,6 +133,8 @@ export function useProviders() { delete kwargs.enableSearch delete kwargs.searchStrategy } + // RFC-009 P3.5: clamp to non-negative, coerce string input back to integer. + const fallbackPriority = Math.max(0, Math.floor(Number(providerForm.fallbackPriority) || 0)) if (editingProvider.value) { await modelApi.updateProviderConfig(editingProvider.value.id, { apiKey: providerForm.apiKey, @@ -135,6 +142,7 @@ export function useProviders() { protocol: providerForm.protocol, chatModel: protocolToChatModel(providerForm.protocol), generateKwargs: kwargs, + fallbackPriority, }) } else { await modelApi.createCustomProvider({ @@ -146,13 +154,14 @@ export function useProviders() { chatModel: protocolToChatModel(providerForm.protocol), models: [], }) - if (providerForm.apiKey || providerForm.generateKwargsText) { + if (providerForm.apiKey || providerForm.generateKwargsText || fallbackPriority > 0) { await modelApi.updateProviderConfig(providerForm.id, { apiKey: providerForm.apiKey, baseUrl: providerForm.baseUrl, protocol: providerForm.protocol, chatModel: protocolToChatModel(providerForm.protocol), generateKwargs: kwargs, + fallbackPriority, }) } }