fix(llm): stop MODEL_NOT_FOUND from evicting the whole provider (#150)

mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java

@@ -179,23 +179,43 @@ public class NodeStreamingChatHelper {
     }
 
     /**
-     * RFC-009 Phase 4 — map an {@link ErrorType} to the matching pool
+     * Map an {@link ErrorType} to the matching pool
      * {@link vip.mate.llm.failover.AvailableProviderPool.RemovalSource} for
-     * HARD failures (AUTH / BILLING / MODEL_NOT_FOUND). Returns {@code null}
+     * provider-wide HARD failures (AUTH / BILLING). Returns {@code null} for
-     * for SOFT errors and benign types — those keep the provider in-pool and
+     * SOFT errors, benign types, and model-scoped errors — those keep the
-     * are handled by {@link vip.mate.llm.failover.ProviderHealthTracker}'s
+     * provider in-pool.
-     * cooldown instead.
+     *
+     * <p>{@code MODEL_NOT_FOUND} is deliberately excluded: it means the
+     * provider rejected one specific model id, not that the provider is
+     * unusable. Evicting the whole provider would needlessly take its other
+     * models offline. SOFT errors are absorbed by
+     * {@link vip.mate.llm.failover.ProviderHealthTracker}'s cooldown instead.</p>
      */
     private static vip.mate.llm.failover.AvailableProviderPool.RemovalSource hardRemovalSource(ErrorType type) {
         if (type == null) return null;
         return switch (type) {
             case AUTH_ERROR -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.AUTH_ERROR;
             case BILLING -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.BILLING;
-            case MODEL_NOT_FOUND -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.MODEL_NOT_FOUND;
             default -> null;
         };
     }
 
+    /**
+     * True when {@code type} reflects the <i>provider's own health</i> (auth,
+     * billing, rate limit, server error, empty response) rather than something
+     * specific to the requested model or prompt. Only provider-level failures
+     * should feed pool eviction and the consecutive-failure cooldown tracker —
+     * a {@code MODEL_NOT_FOUND} / {@code CLIENT_ERROR} / {@code PROMPT_TOO_LONG}
+     * says nothing about whether the provider's other models still work.
+     */
+    private static boolean isProviderLevelFailure(ErrorType type) {
+        if (type == null) return false;
+        return switch (type) {
+            case NONE, PROMPT_TOO_LONG, CLIENT_ERROR, THINKING_BLOCK_ERROR, MODEL_NOT_FOUND -> false;
+            default -> true;
+        };
+    }
+
     /** Convenience: pool-aware membership check. Null pool means fail-open (everyone in). */
     private boolean inPool(String providerId) {
         return providerPool == null || providerId == null || providerPool.contains(providerId);
@@ -521,15 +541,23 @@ public class NodeStreamingChatHelper {
                     removeFromPool(primaryProviderId, ErrorType.AUTH_ERROR, lastResult.errorMessage());
                     break;
                 }
-                // RFC-009 P3.2: BILLING / MODEL_NOT_FOUND — provider-side hard failures
+                // BILLING — provider-side hard failure (out of credit). Won't change
-                // that won't change on retry. Skip to fallback chain (a different
+                // on retry and affects every model on the provider, so evict it and
-                // provider may have credits, or the model name may be valid there).
+                // hand off to the fallback chain (a different provider may have credits).
-                if (lastResult.errorType() == ErrorType.BILLING
+                if (lastResult.errorType() == ErrorType.BILLING) {
-                        || lastResult.errorType() == ErrorType.MODEL_NOT_FOUND) {
+                    log.warn("[{}] Primary billing failure — skipping same-model retries, handing off to fallback chain", phase);
-                    log.warn("[{}] Primary error={} — skipping same-model retries, handing off to fallback chain",
-                            phase, lastResult.errorType());
                     recordPrimary(false);
-                    removeFromPool(primaryProviderId, lastResult.errorType(), lastResult.errorMessage());
+                    removeFromPool(primaryProviderId, ErrorType.BILLING, lastResult.errorMessage());
+                    break;
+                }
+                // MODEL_NOT_FOUND — the provider rejected this specific model id. The
+                // provider itself is healthy, so do NOT evict it from the pool or
+                // record a provider-level failure: that would take its sibling models
+                // down too. Just skip same-model retries and hand off to the fallback
+                // chain — a different provider may recognize the model name.
+                if (lastResult.errorType() == ErrorType.MODEL_NOT_FOUND) {
+                    log.warn("[{}] Primary model not found — handing off to fallback chain "
+                            + "(provider kept available for its other models)", phase);
                     break;
                 }
                 // CLIENT_ERROR (400 Bad Request): 不重试（参数/格式错误重试也不会变）
@@ -570,8 +598,10 @@ public class NodeStreamingChatHelper {
             }
             // lastResult == null 表示需要重试
         }
-        // If we exhausted the retry loop without a verdict, primary effectively failed.
+        // If we exhausted the retry loop without a verdict, primary effectively
-        if (!primarySkipped && lastResult != null && lastResult.errorType() != ErrorType.NONE) {
+        // failed. Only count it against provider health for provider-level errors —
+        // a MODEL_NOT_FOUND break above must not nudge the provider toward cooldown.
+        if (!primarySkipped && lastResult != null && isProviderLevelFailure(lastResult.errorType())) {
             recordPrimary(false);
         }
 
@@ -621,11 +651,17 @@ public class NodeStreamingChatHelper {
                 logPerfSummary(phase, conversationId, callStartMs, llmCallCount, retryCount, failoverCount);
                 return fallbackResult;
             }
-            if (healthTracker != null) healthTracker.recordFailure(entry.providerId());
+            // Only provider-level failures count toward the cooldown tracker. A
+            // null result is a retryable soft failure; a MODEL_NOT_FOUND result is
+            // model-scoped and must not penalise an otherwise-healthy provider.
+            if (healthTracker != null
+                    && (fallbackResult == null || isProviderLevelFailure(fallbackResult.errorType()))) {
+                healthTracker.recordFailure(entry.providerId());
+            }
             if (fallbackResult != null) {
-                // RFC-009 Phase 4: HARD errors evict from the pool so later
+                // HARD errors (auth / billing) evict the provider from the pool so
-                // walks skip this provider outright. SOFT errors keep it
+                // later walks skip it outright. SOFT and model-scoped errors keep it
-                // in-pool and let the tracker's cooldown absorb the blip.
+                // in-pool — absorbed by the tracker's cooldown or simply retried.
                 removeFromPool(entry.providerId(), fallbackResult.errorType(), fallbackResult.errorMessage());
                 lastResult = fallbackResult; // remember most recent to report if the whole chain fails
             }

mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java

@@ -18,12 +18,14 @@ import java.util.concurrent.ConcurrentHashMap;
  * <ul>
  *   <li><b>Add</b> — at startup ({@code ProviderInitProbe}), on user-triggered
  *       reprobe, or after a {@code ModelConfigChangedEvent}.</li>
- *   <li><b>Remove</b> — when a request hits a HARD error (AUTH_ERROR /
+ *   <li><b>Remove</b> — when a request hits a provider-wide HARD error
- *       BILLING / MODEL_NOT_FOUND) — these don't self-heal, so retrying on
+ *       (AUTH_ERROR / BILLING) — these don't self-heal, so retrying on
  *       every subsequent call wastes the user's time. SOFT errors
  *       (RATE_LIMIT / SERVER_ERROR / EMPTY_RESPONSE) keep the provider in
  *       the pool and are handled by {@link ProviderHealthTracker}'s short
- *       cooldown instead.</li>
+ *       cooldown instead. A rejected model id (MODEL_NOT_FOUND) is
+ *       model-scoped, not provider-scoped, and never evicts the provider —
+ *       its sibling models stay usable.</li>
  * </ul>
  *
  * <p>State is process-local; a restart re-runs the init probe. That's
@@ -107,15 +109,17 @@ public class AvailableProviderPool {
     public record RemovalReason(RemovalSource source, String message, long removedAtMs) {}
 
     /**
-     * Categorical source of a pool removal. Mirrors the HARD error types from
+     * Categorical source of a pool removal. Covers the provider-wide HARD
-     * {@code NodeStreamingChatHelper.ErrorType} plus {@link #INIT_PROBE} for
+     * error types from {@code NodeStreamingChatHelper.ErrorType} plus
-     * startup probe failures. SOFT errors (RATE_LIMIT / SERVER_ERROR) never
+     * {@link #INIT_PROBE} for startup probe failures. SOFT errors (RATE_LIMIT /
-     * appear here — they're handled by {@link ProviderHealthTracker} cooldown.
+     * SERVER_ERROR) never appear here — they're handled by
+     * {@link ProviderHealthTracker} cooldown. A rejected model id is
+     * model-scoped and never removes a whole provider, so there is no
+     * {@code MODEL_NOT_FOUND} source.
      */
     public enum RemovalSource {
         AUTH_ERROR,
         BILLING,
-        MODEL_NOT_FOUND,
         INIT_PROBE,
         MANUAL
     }

mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java

@@ -164,10 +164,12 @@ public class WikiProcessingJobService {
     }
 
     private static boolean isHardError(String errorCode) {
+        // Only provider-wide failures evict a provider from the shared pool. A
+        // MODEL_NOT_FOUND is model-scoped — it must not take the provider's other
+        // models offline for every other consumer of the pool.
         return errorCode != null && (
             errorCode.equals("AUTH_ERROR") ||
-            errorCode.equals("BILLING") ||
+            errorCode.equals("BILLING"));
-            errorCode.equals("MODEL_NOT_FOUND"));
     }
 
     private void notifyPoolHardError(Long modelId, String errorCode) {

mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java

@@ -179,19 +179,26 @@ class NodeStreamingChatHelperPoolTest {
     }
 
     @Test
-    @DisplayName("Primary MODEL_NOT_FOUND HARD-removes with MODEL_NOT_FOUND source")
+    @DisplayName("Primary MODEL_NOT_FOUND keeps the provider in the pool (model-scoped, not provider-wide)")
-    void primaryModelNotFoundEvictsWithCorrectSource() {
+    void primaryModelNotFoundKeepsProviderInPool() {
         pool.add("openai");
         pool.add("dashscope");
 
+        // One model id is rejected — the provider's other models are still fine,
+        // so the provider must stay usable for them.
         ChatModel primary = errorModel(new RuntimeException("404 model_not_found: gpt-99"));
         ChatModel fallback = successModel("ok");
         var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai");
 
-        helper.streamCall(primary, smallPrompt(), "conv-h3c", "reasoning");
+        var result = helper.streamCall(primary, smallPrompt(), "conv-h3c", "reasoning");
 
-        assertFalse(pool.contains("openai"));
+        assertEquals("ok", result.text(), "request still succeeds via the fallback chain");
-        assertEquals(RemovalSource.MODEL_NOT_FOUND, pool.snapshot().get("openai").source());
+        assertTrue(pool.contains("openai"),
+                "MODEL_NOT_FOUND rejects one model id — the provider's sibling models stay usable");
+        assertNull(pool.snapshot().get("openai"),
+                "a model-scoped error must not record a provider removal reason");
+        assertNull(healthTracker.snapshot().get("openai"),
+                "MODEL_NOT_FOUND must not nudge the provider toward cooldown");
     }
 
     // ============================================================

mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java

@@ -91,14 +91,14 @@ class AvailableProviderPoolTest {
     void snapshotMixedView() {
         pool.add("openai");
         pool.add("dashscope");
-        pool.remove("anthropic", RemovalSource.MODEL_NOT_FOUND, "model claude-99 not found");
+        pool.remove("anthropic", RemovalSource.BILLING, "402 insufficient credit");
 
         var snap = pool.snapshot();
         assertEquals(3, snap.size());
         assertNull(snap.get("openai"), "in-pool members appear with null value");
         assertNull(snap.get("dashscope"));
         assertNotNull(snap.get("anthropic"));
-        assertEquals(RemovalSource.MODEL_NOT_FOUND, snap.get("anthropic").source());
+        assertEquals(RemovalSource.BILLING, snap.get("anthropic").source());
     }
 
     @Test