diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java
index 9456986c..7499369c 100644
--- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java
+++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java
@@ -58,6 +58,25 @@ public class WikiProcessingService {
private final WikiCitationService citationService;
private final org.springframework.context.ApplicationEventPublisher eventPublisher;
+ /**
+ * Read-the-failover-chain handle. Optional so the existing constructors and
+ * lazy-mode tests don't have to thread a new dependency. When null, the
+ * fallback hop iterates {@code listEnabledModels} in DB order — same
+ * behavior as before this PR.
+ */
+ @org.springframework.beans.factory.annotation.Autowired(required = false)
+ private vip.mate.llm.service.ModelProviderService modelProviderService;
+
+ /**
+ * Per-provider failure counter / cooldown bookkeeping. Optional for the
+ * same reason. When wired, fatal errors mark the provider down so other
+ * code paths (chat agent, fallback chain) skip it during cooldown; on a
+ * successful call we clear the failure counter for the provider that
+ * actually responded.
+ */
+ @org.springframework.beans.factory.annotation.Autowired(required = false)
+ private vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker;
+
@org.springframework.beans.factory.annotation.Autowired(required = false)
@org.springframework.context.annotation.Lazy
private vip.mate.wiki.job.WikiProcessingJobService wikiJobService;
@@ -1562,6 +1581,12 @@ public class WikiProcessingService {
ChatModel chatModel = resolved.chatModel;
Long currentModelId = resolved.modelId;
boolean alreadyFellBack = false;
+ // When we fail over after a primary fatal, hold onto the primary's
+ // exception so we can surface BOTH errors if the fallback also dies.
+ // Operators reading logs need to see "the GLM 1113 we tried first" —
+ // not just "DashScope timed out" with the original cause lost.
+ Throwable primaryFatalError = null;
+ String primaryRootInfo = null;
int attempt = 0;
while (true) {
@@ -1577,6 +1602,11 @@ public class WikiProcessingService {
if (attempt > 1) {
log.info("[Wiki] LLM call for {} succeeded on attempt {}", ctx, attempt);
}
+ // The provider that just answered is healthy: clear any pending
+ // cooldown so subsequent calls can pick it freely. No-op on the
+ // happy path (counter is already 0); cleanup after a fallback
+ // success.
+ recordProviderSuccess(currentModelId);
// P0 telemetry: log token usage per LLM call so we can baseline costs before optimizing
long durationMs = (System.nanoTime() - startNanos) / 1_000_000L;
try {
@@ -1593,17 +1623,23 @@ public class WikiProcessingService {
}
String rootInfo = summarizeRoot(t);
if (isFatalModelError(t)) {
+ // Mark the wedged provider down so the chat-agent path,
+ // any future fallback walks, and the admin diagnostics
+ // know to skip it during cooldown.
+ recordProviderFailure(currentModelId);
// One-hop fallback: when the primary provider is wedged
// (auth / quota / 余额不足 / model-not-found), try the next
// configured chat model with a different provider before
// giving up. The hop is one-shot — if the fallback also
- // fatals, we surface the error rather than walking the
+ // fatals, we surface BOTH errors rather than walking the
// whole chain to avoid pathological loops.
if (!alreadyFellBack) {
ResolvedChatModel next = pickFallbackChatModel(currentModelId);
if (next != null) {
log.warn("[Wiki] LLM fatal on primary model={} for {}; failing over to model={} (rootCause={}): {}",
currentModelId, ctx, next.modelId, rootInfo, t.getMessage());
+ primaryFatalError = t;
+ primaryRootInfo = rootInfo;
chatModel = next.chatModel;
currentModelId = next.modelId;
alreadyFellBack = true;
@@ -1614,6 +1650,17 @@ public class WikiProcessingService {
continue;
}
}
+ if (alreadyFellBack && primaryFatalError != null) {
+ log.error("[Wiki] LLM unavailable (fatal on BOTH primary + fallback) for {}: primary rootCause={}: {} | fallback rootCause={}: {}",
+ ctx, primaryRootInfo, primaryFatalError.getMessage(), rootInfo, t.getMessage());
+ RuntimeException re = new RuntimeException(
+ "LLM unavailable on both primary and fallback. Primary (rootCause="
+ + primaryRootInfo + "): " + primaryFatalError.getMessage()
+ + " | Fallback (rootCause=" + rootInfo + "): " + t.getMessage(),
+ t);
+ re.addSuppressed(primaryFatalError);
+ throw re;
+ }
log.error("[Wiki] LLM unavailable (fatal) for {} after {} attempts on model={} (rootCause={}): {}",
ctx, attempt, currentModelId, rootInfo, t.getMessage());
throw new RuntimeException("LLM unavailable (rootCause=" + rootInfo + "): " + t.getMessage(), t);
@@ -1639,6 +1686,26 @@ public class WikiProcessingService {
}
}
+ private void recordProviderFailure(Long modelId) {
+ if (providerHealthTracker == null || modelId == null) return;
+ try {
+ ModelConfigEntity m = modelConfigService.getModel(modelId);
+ if (m != null && m.getProvider() != null) {
+ providerHealthTracker.recordFailure(m.getProvider());
+ }
+ } catch (Exception ignored) {}
+ }
+
+ private void recordProviderSuccess(Long modelId) {
+ if (providerHealthTracker == null || modelId == null) return;
+ try {
+ ModelConfigEntity m = modelConfigService.getModel(modelId);
+ if (m != null && m.getProvider() != null) {
+ providerHealthTracker.recordSuccess(m.getProvider());
+ }
+ } catch (Exception ignored) {}
+ }
+
/** Pair of modelId + built ChatModel — null modelId means we used the system default. */
private record ResolvedChatModel(Long modelId, ChatModel chatModel) {}
@@ -1663,9 +1730,19 @@ public class WikiProcessingService {
/**
* Pick the next enabled chat model whose provider differs from the failed
- * model's provider, so we cycle to a fresh credential / billing account
- * rather than retrying a wedged one. Returns null when no alternative
- * exists — caller surfaces the original failure.
+ * one, so we cycle to a fresh credential / billing account rather than
+ * retrying a wedged one. Returns null when no alternative exists.
+ *
+ * Provider order is deterministic — driven by
+ * {@code mate_model_provider.fallback_priority} (asc, lowest first) when
+ * {@link #modelProviderService} is wired. Falls back to
+ * {@code listEnabledModels} in DB order only when the provider service
+ * isn't available (older test harnesses), so behavior is at-least
+ * stable, never random.
+ *
+ *
Skips providers currently in cooldown ({@link
+ * vip.mate.llm.failover.ProviderHealthTracker}) so a flapping provider
+ * doesn't keep getting tried while we wait for it to recover.
*/
private ResolvedChatModel pickFallbackChatModel(Long failedModelId) {
try {
@@ -1674,25 +1751,68 @@ public class WikiProcessingService {
ModelConfigEntity failed = modelConfigService.getModel(failedModelId);
if (failed != null) failedProviderId = failed.getProvider();
}
- for (ModelConfigEntity candidate : modelConfigService.listEnabledModels()) {
- if (candidate.getId() == null) continue;
- if (candidate.getId().equals(failedModelId)) continue;
- String mt = candidate.getModelType();
- if (mt != null && !mt.isBlank() && !"chat".equalsIgnoreCase(mt)) continue;
- if (failedProviderId != null && failedProviderId.equals(candidate.getProvider())) continue;
- try {
- ChatModel built = agentGraphBuilder.buildRuntimeChatModel(candidate, WIKI_NO_RETRY);
- return new ResolvedChatModel(candidate.getId(), built);
- } catch (Exception e) {
- log.debug("[Wiki] fallback candidate model={} unbuildable: {}", candidate.getId(), e.getMessage());
+
+ if (modelProviderService != null) {
+ for (var provider : modelProviderService.listFallbackChain()) {
+ String pid = provider.getProviderId();
+ if (pid == null) continue;
+ if (pid.equals(failedProviderId)) continue;
+ if (providerHealthTracker != null && providerHealthTracker.isInCooldown(pid)) continue;
+ ResolvedChatModel built = firstChatModelForProvider(pid, failedModelId);
+ if (built != null) return built;
}
}
+
+ // Defensive fallback path — only triggers in tests / minimal
+ // environments without ModelProviderService wired. Mirrors the
+ // pre-PR behavior so legacy tests don't regress.
+ for (ModelConfigEntity candidate : modelConfigService.listEnabledModels()) {
+ if (!isUsableChatFallback(candidate, failedModelId, failedProviderId)) continue;
+ ResolvedChatModel built = tryBuild(candidate);
+ if (built != null) return built;
+ }
} catch (Exception e) {
log.debug("[Wiki] fallback lookup failed: {}", e.getMessage());
}
return null;
}
+ private ResolvedChatModel firstChatModelForProvider(String providerId, Long failedModelId) {
+ List rows;
+ try {
+ rows = modelConfigService.listModelsByProvider(providerId);
+ } catch (Exception e) {
+ log.debug("[Wiki] listModelsByProvider({}) failed: {}", providerId, e.getMessage());
+ return null;
+ }
+ for (ModelConfigEntity candidate : rows) {
+ if (!isUsableChatFallback(candidate, failedModelId, null)) continue;
+ ResolvedChatModel built = tryBuild(candidate);
+ if (built != null) return built;
+ }
+ return null;
+ }
+
+ private static boolean isUsableChatFallback(ModelConfigEntity candidate, Long failedModelId, String failedProviderId) {
+ if (candidate == null || candidate.getId() == null) return false;
+ if (candidate.getId().equals(failedModelId)) return false;
+ if (!Boolean.TRUE.equals(candidate.getEnabled())) return false;
+ String mt = candidate.getModelType();
+ if (mt != null && !mt.isBlank() && !"chat".equalsIgnoreCase(mt)) return false;
+ if (failedProviderId != null && failedProviderId.equals(candidate.getProvider())) return false;
+ return true;
+ }
+
+ private ResolvedChatModel tryBuild(ModelConfigEntity candidate) {
+ try {
+ ChatModel built = agentGraphBuilder.buildRuntimeChatModel(candidate, WIKI_NO_RETRY);
+ return new ResolvedChatModel(candidate.getId(), built);
+ } catch (Exception e) {
+ log.debug("[Wiki] fallback candidate model={} unbuildable: {}", candidate.getId(), e.getMessage());
+ return null;
+ }
+ }
+
/**
* 判断是否为"模型不可用"级别的致命错误（不重试，立即终止）。
*