{@code MODEL_NOT_FOUND} is deliberately excluded: it means the + * provider rejected one specific model id, not that the provider is + * unusable. Evicting the whole provider would needlessly take its other + * models offline. SOFT errors are absorbed by + * {@link vip.mate.llm.failover.ProviderHealthTracker}'s cooldown instead.
*/ private static vip.mate.llm.failover.AvailableProviderPool.RemovalSource hardRemovalSource(ErrorType type) { if (type == null) return null; return switch (type) { case AUTH_ERROR -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.AUTH_ERROR; case BILLING -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.BILLING; - case MODEL_NOT_FOUND -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.MODEL_NOT_FOUND; default -> null; }; } + /** + * True when {@code type} reflects the provider's own health (auth, + * billing, rate limit, server error, empty response) rather than something + * specific to the requested model or prompt. Only provider-level failures + * should feed pool eviction and the consecutive-failure cooldown tracker — + * a {@code MODEL_NOT_FOUND} / {@code CLIENT_ERROR} / {@code PROMPT_TOO_LONG} + * says nothing about whether the provider's other models still work. + */ + private static boolean isProviderLevelFailure(ErrorType type) { + if (type == null) return false; + return switch (type) { + case NONE, PROMPT_TOO_LONG, CLIENT_ERROR, THINKING_BLOCK_ERROR, MODEL_NOT_FOUND -> false; + default -> true; + }; + } + /** Convenience: pool-aware membership check. Null pool means fail-open (everyone in). */ private boolean inPool(String providerId) { return providerPool == null || providerId == null || providerPool.contains(providerId); @@ -521,15 +541,23 @@ public class NodeStreamingChatHelper { removeFromPool(primaryProviderId, ErrorType.AUTH_ERROR, lastResult.errorMessage()); break; } - // RFC-009 P3.2: BILLING / MODEL_NOT_FOUND — provider-side hard failures - // that won't change on retry. Skip to fallback chain (a different - // provider may have credits, or the model name may be valid there). - if (lastResult.errorType() == ErrorType.BILLING - || lastResult.errorType() == ErrorType.MODEL_NOT_FOUND) { - log.warn("[{}] Primary error={} — skipping same-model retries, handing off to fallback chain", - phase, lastResult.errorType()); + // BILLING — provider-side hard failure (out of credit). Won't change + // on retry and affects every model on the provider, so evict it and + // hand off to the fallback chain (a different provider may have credits). + if (lastResult.errorType() == ErrorType.BILLING) { + log.warn("[{}] Primary billing failure — skipping same-model retries, handing off to fallback chain", phase); recordPrimary(false); - removeFromPool(primaryProviderId, lastResult.errorType(), lastResult.errorMessage()); + removeFromPool(primaryProviderId, ErrorType.BILLING, lastResult.errorMessage()); + break; + } + // MODEL_NOT_FOUND — the provider rejected this specific model id. The + // provider itself is healthy, so do NOT evict it from the pool or + // record a provider-level failure: that would take its sibling models + // down too. Just skip same-model retries and hand off to the fallback + // chain — a different provider may recognize the model name. + if (lastResult.errorType() == ErrorType.MODEL_NOT_FOUND) { + log.warn("[{}] Primary model not found — handing off to fallback chain " + + "(provider kept available for its other models)", phase); break; } // CLIENT_ERROR (400 Bad Request): 不重试（参数/格式错误重试也不会变） @@ -570,8 +598,10 @@ public class NodeStreamingChatHelper { } // lastResult == null 表示需要重试 } - // If we exhausted the retry loop without a verdict, primary effectively failed. - if (!primarySkipped && lastResult != null && lastResult.errorType() != ErrorType.NONE) { + // If we exhausted the retry loop without a verdict, primary effectively + // failed. Only count it against provider health for provider-level errors — + // a MODEL_NOT_FOUND break above must not nudge the provider toward cooldown. + if (!primarySkipped && lastResult != null && isProviderLevelFailure(lastResult.errorType())) { recordPrimary(false); } @@ -621,11 +651,17 @@ public class NodeStreamingChatHelper { logPerfSummary(phase, conversationId, callStartMs, llmCallCount, retryCount, failoverCount); return fallbackResult; } - if (healthTracker != null) healthTracker.recordFailure(entry.providerId()); + // Only provider-level failures count toward the cooldown tracker. A + // null result is a retryable soft failure; a MODEL_NOT_FOUND result is + // model-scoped and must not penalise an otherwise-healthy provider. + if (healthTracker != null + && (fallbackResult == null || isProviderLevelFailure(fallbackResult.errorType()))) { + healthTracker.recordFailure(entry.providerId()); + } if (fallbackResult != null) { - // RFC-009 Phase 4: HARD errors evict from the pool so later - // walks skip this provider outright. SOFT errors keep it - // in-pool and let the tracker's cooldown absorb the blip. + // HARD errors (auth / billing) evict the provider from the pool so + // later walks skip it outright. SOFT and model-scoped errors keep it + // in-pool — absorbed by the tracker's cooldown or simply retried. removeFromPool(entry.providerId(), fallbackResult.errorType(), fallbackResult.errorMessage()); lastResult = fallbackResult; // remember most recent to report if the whole chain fails } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java index 76e17425..0d7c4ae9 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java @@ -18,12 +18,14 @@ import java.util.concurrent.ConcurrentHashMap; *State is process-local; a restart re-runs the init probe. That's @@ -107,15 +109,17 @@ public class AvailableProviderPool { public record RemovalReason(RemovalSource source, String message, long removedAtMs) {} /** - * Categorical source of a pool removal. Mirrors the HARD error types from - * {@code NodeStreamingChatHelper.ErrorType} plus {@link #INIT_PROBE} for - * startup probe failures. SOFT errors (RATE_LIMIT / SERVER_ERROR) never - * appear here — they're handled by {@link ProviderHealthTracker} cooldown. + * Categorical source of a pool removal. Covers the provider-wide HARD + * error types from {@code NodeStreamingChatHelper.ErrorType} plus + * {@link #INIT_PROBE} for startup probe failures. SOFT errors (RATE_LIMIT / + * SERVER_ERROR) never appear here — they're handled by + * {@link ProviderHealthTracker} cooldown. A rejected model id is + * model-scoped and never removes a whole provider, so there is no + * {@code MODEL_NOT_FOUND} source. */ public enum RemovalSource { AUTH_ERROR, BILLING, - MODEL_NOT_FOUND, INIT_PROBE, MANUAL } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java index 05b7a8ea..85a76813 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java @@ -164,10 +164,12 @@ public class WikiProcessingJobService { } private static boolean isHardError(String errorCode) { + // Only provider-wide failures evict a provider from the shared pool. A + // MODEL_NOT_FOUND is model-scoped — it must not take the provider's other + // models offline for every other consumer of the pool. return errorCode != null && ( errorCode.equals("AUTH_ERROR") || - errorCode.equals("BILLING") || - errorCode.equals("MODEL_NOT_FOUND")); + errorCode.equals("BILLING")); } private void notifyPoolHardError(Long modelId, String errorCode) { diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java index f5ef3903..b195081c 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java @@ -179,19 +179,26 @@ class NodeStreamingChatHelperPoolTest { } @Test - @DisplayName("Primary MODEL_NOT_FOUND HARD-removes with MODEL_NOT_FOUND source") - void primaryModelNotFoundEvictsWithCorrectSource() { + @DisplayName("Primary MODEL_NOT_FOUND keeps the provider in the pool (model-scoped, not provider-wide)") + void primaryModelNotFoundKeepsProviderInPool() { pool.add("openai"); pool.add("dashscope"); + // One model id is rejected — the provider's other models are still fine, + // so the provider must stay usable for them. ChatModel primary = errorModel(new RuntimeException("404 model_not_found: gpt-99")); ChatModel fallback = successModel("ok"); var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); - helper.streamCall(primary, smallPrompt(), "conv-h3c", "reasoning"); + var result = helper.streamCall(primary, smallPrompt(), "conv-h3c", "reasoning"); - assertFalse(pool.contains("openai")); - assertEquals(RemovalSource.MODEL_NOT_FOUND, pool.snapshot().get("openai").source()); + assertEquals("ok", result.text(), "request still succeeds via the fallback chain"); + assertTrue(pool.contains("openai"), + "MODEL_NOT_FOUND rejects one model id — the provider's sibling models stay usable"); + assertNull(pool.snapshot().get("openai"), + "a model-scoped error must not record a provider removal reason"); + assertNull(healthTracker.snapshot().get("openai"), + "MODEL_NOT_FOUND must not nudge the provider toward cooldown"); } // ============================================================ diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java index 33f4d7f3..f000f0ee 100644 --- a/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java @@ -91,14 +91,14 @@ class AvailableProviderPoolTest { void snapshotMixedView() { pool.add("openai"); pool.add("dashscope"); - pool.remove("anthropic", RemovalSource.MODEL_NOT_FOUND, "model claude-99 not found"); + pool.remove("anthropic", RemovalSource.BILLING, "402 insufficient credit"); var snap = pool.snapshot(); assertEquals(3, snap.size()); assertNull(snap.get("openai"), "in-pool members appear with null value"); assertNull(snap.get("dashscope")); assertNotNull(snap.get("anthropic")); - assertEquals(RemovalSource.MODEL_NOT_FOUND, snap.get("anthropic").source()); + assertEquals(RemovalSource.BILLING, snap.get("anthropic").source()); } @Test