mateclaw/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java
matevip 7b12c5f0c9 feat(llm): provider health tracker + UI editor for failover priority
UI — Failover priority editor
- ProviderConfigRequest + ProviderInfoDTO carry fallbackPriority
- ModelProviderService.updateProviderConfig persists it (null = unchanged);
  toProviderInfo exposes the current value to the UI (defaults to 0)
- ProviderConfigModal advanced panel exposes a number input with hint
- ProviderCard shows a "Fallback #N" badge for chain members so the
  priority order is visible at a glance without opening the modal
- 5 new i18n keys (zh + en) — verified to resolve at runtime via i18n.global.t

Backend — Per-provider health tracker
- ProviderHealthTracker: ConcurrentHashMap-backed counters; N consecutive
  failures (default 3) push the provider into a cooldown window (default
  5 min) during which the chain walker skips it. Success resets both
  counter and cooldown atomically. Lazy expiry on lookup so dead entries
  do not accumulate.
- ProviderHealthProperties exposed under mateclaw.llm.failover.health.*
  with sane production defaults
- New FallbackEntry record (providerId + ChatModel) replaces raw
  List<ChatModel> in the chain so the walker can correlate cooldown
  state to entries; AgentGraphBuilder.buildFallbackChain returns the
  new type
- NodeStreamingChatHelper takes the tracker through a new 4-arg
  constructor and consults it before each fallback call; records
  success/failure on each chain attempt. Legacy 2/3-arg constructors
  preserved as @Deprecated wrappers (synthetic providerId means no
  health tracking on the legacy path — that path is opt-out anyway)

Tests
- ProviderHealthTrackerTest (9 tests): below/at threshold, success
  reset, cooldown expiry (via reflection on the min-clamp setter),
  disabled-tracker no-op, null-providerId safety, per-provider
  isolation, snapshot output
- NodeStreamingChatHelperFallbackChainTest updated to FallbackEntry
  field type — verifies providerId + ChatModel survive the chain
- 168 tests pass (was 159 + 9 new)

Verification
- mvn test green; vue-tsc clean; live UI confirms i18n resolution
2026-04-19 16:57:03 +08:00

118 lines
4.2 KiB
Java

package vip.mate.llm.failover;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/**
* per-provider failure-count + cooldown logic.
*/
class ProviderHealthTrackerTest {
private ProviderHealthProperties props;
private ProviderHealthTracker tracker;
@BeforeEach
void setUp() {
props = new ProviderHealthProperties();
props.setFailureThreshold(3);
props.setCooldownMs(60_000L);
tracker = new ProviderHealthTracker(props);
}
@Test
@DisplayName("New provider is not in cooldown")
void newProviderNotInCooldown() {
assertFalse(tracker.isInCooldown("openai"));
}
@Test
@DisplayName("Failures below threshold do not trigger cooldown")
void belowThresholdNoCooldown() {
tracker.recordFailure("openai");
tracker.recordFailure("openai");
assertFalse(tracker.isInCooldown("openai"),
"two failures < threshold of 3 must not enter cooldown");
}
@Test
@DisplayName("Failures hitting threshold enter cooldown")
void thresholdReachedTriggersCooldown() {
tracker.recordFailure("openai");
tracker.recordFailure("openai");
tracker.recordFailure("openai");
assertTrue(tracker.isInCooldown("openai"),
"third failure must enter cooldown");
}
@Test
@DisplayName("Success resets failure counter and clears cooldown")
void successResetsCounterAndCooldown() {
tracker.recordFailure("openai");
tracker.recordFailure("openai");
tracker.recordFailure("openai");
assertTrue(tracker.isInCooldown("openai"));
tracker.recordSuccess("openai");
assertFalse(tracker.isInCooldown("openai"),
"success must clear cooldown so the provider becomes eligible again");
}
@Test
@DisplayName("After cooldown expires, provider becomes eligible again")
void cooldownExpires() throws Exception {
// Bypass the min-1000ms clamp in setCooldownMs via reflection — the
// clamp is there to prevent prod misconfiguration, but for this test
// we want a fast-expiring window to avoid sleeping 1+ seconds.
java.lang.reflect.Field f = ProviderHealthProperties.class.getDeclaredField("cooldownMs");
f.setAccessible(true);
f.setLong(props, 50L);
for (int i = 0; i < 3; i++) tracker.recordFailure("openai");
assertTrue(tracker.isInCooldown("openai"), "sanity: still in cooldown right after trigger");
Thread.sleep(120);
assertFalse(tracker.isInCooldown("openai"),
"cooldown should expire once the window has passed");
}
@Test
@DisplayName("Disabled tracker never reports cooldown")
void disabledTrackerInert() {
props.setEnabled(false);
for (int i = 0; i < 10; i++) tracker.recordFailure("openai");
assertFalse(tracker.isInCooldown("openai"),
"disabled tracker must report no cooldown regardless of failures");
}
@Test
@DisplayName("Null providerId is a safe no-op")
void nullProviderIdSafe() {
tracker.recordFailure(null);
tracker.recordSuccess(null);
assertFalse(tracker.isInCooldown(null),
"null providerId must not crash and must report no cooldown");
}
@Test
@DisplayName("Per-provider isolation: cooldown on A does not affect B")
void perProviderIsolation() {
for (int i = 0; i < 3; i++) tracker.recordFailure("openai");
assertTrue(tracker.isInCooldown("openai"));
assertFalse(tracker.isInCooldown("dashscope"),
"cooldown must be scoped per provider id");
}
@Test
@DisplayName("Snapshot reports both failure count and remaining cooldown")
void snapshotReportsState() {
for (int i = 0; i < 3; i++) tracker.recordFailure("openai");
var snap = tracker.snapshot();
assertNotNull(snap.get("openai"));
assertEquals(3L, snap.get("openai").consecutiveFailures());
assertTrue(snap.get("openai").cooldownRemainingMs() > 0,
"cooldown remaining ms must be positive while active");
}
}