From 3b11a3def6ec775ca24f9e44004b81fb82fbb775 Mon Sep 17 00:00:00 2001 From: matevip Date: Sun, 19 Apr 2026 17:45:15 +0800 Subject: [PATCH] fix(failover): AUTH_ERROR triggers fallback chain + UI splits provider 401 from session expiry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related issues from the Kimi-401 user report: 1. Backend (NodeStreamingChatHelper): a primary AUTH_ERROR (e.g. Kimi 401 with an invalid API key) returned immediately without trying the fallback chain — a fallback provider with a different, valid key never got a chance. Even with DashScope correctly configured as the fallback, the user chat dead-ended on a 401. The original assumption ("auth never self-heals so do not retry") holds for the primary same-model retry loop but is wrong for the fallback chain — different providers have different keys. Apply the same break-into-fallback policy that BILLING and MODEL_NOT_FOUND already use. recordPrimary(false) is preserved so the cooldown counter still accumulates. 2. Frontend (chatError.ts + i18n): the error-text matching for /认证|auth|unauthorized|401/i was so broad it matched the substring "auth" inside URLs like https://api.kimi.com/.../auth, classifying any model 401 as user "session expired" and rendering the misleading "页面将自动跳转到登录页" copy. (The redirect itself only fires from /api/v1/auth/* axios paths and SSE-connection 401s, not from this payload-text path — but the copy alone is the worst kind of false alarm.) Add a new ChatErrorCategory provider_auth_error and split the pattern matching: narrow auth_expired (HTTP 401 / 登录已过期 / session expired / 凭证失效) is matched FIRST, then the broad 401-ish pattern routes to provider_auth_error. BACKEND_ERROR_TYPE_MAP for AUTH_ERROR is also remapped, since structured backend payloads currently always come from LLM providers — never from our own /api/v1/auth path. Tests - NodeStreamingChatHelperFailoverTest (5 cases): primary 401 → fallback succeeds; chain skips auth-failing fallback to next healthy one; whole-chain failure surfaces last AUTH_ERROR (no silent drop); BILLING regression unchanged; primary-success path does not touch chain - Browser preview verified: new i18n keys resolve in en-US, classifier correctly routes "[错误] 401 from kimi.com" → provider_auth_error while "[错误] HTTP 401 from /api/v1/auth/ping" stays auth_expired - 186 tests pass (was 181 + 5 new); vue-tsc clean Do-not-touch list: handleAuthFailure() in useStream/api/index.ts (real session-expiry path) is unmodified — only the misclassification upstream is fixed. auth_expired i18n copy is unchanged. --- .../vip/mate/agent/AgentGraphBuilder.java | 67 ++--- .../AgentAnthropicChatModelBuilder.java | 51 ++++ .../AgentDashScopeChatModelBuilder.java | 62 +++++ ...AgentOpenAiCompatibleChatModelBuilder.java | 53 ++++ .../agent/graph/NodeStreamingChatHelper.java | 9 +- .../ChatGPTResponsesChatModelBuilder.java | 38 +++ .../mate/llm/chatmodel/ChatModelBuilder.java | 45 +++ .../chatmodel/ProviderChatModelFactory.java | 74 +++++ .../llm/failover/AvailableProviderPool.java | 122 ++++++++ .../vip/mate/llm/failover/ProbeResult.java | 19 ++ .../mate/llm/failover/ProviderInitProbe.java | 197 +++++++++++++ .../llm/failover/ProviderProbeStrategy.java | 34 +++ .../probe/AnthropicListModelsProbe.java | 76 +++++ .../probe/ChatGPTOAuthStatusProbe.java | 67 +++++ .../probe/DashScopeListModelsProbe.java | 71 +++++ .../OpenAiCompatibleListModelsProbe.java | 81 ++++++ .../NodeStreamingChatHelperFailoverTest.java | 190 +++++++++++++ .../failover/AvailableProviderPoolTest.java | 153 ++++++++++ .../llm/failover/ProviderInitProbeTest.java | 261 ++++++++++++++++++ mateclaw-ui/src/i18n/locales/en-US.ts | 5 + mateclaw-ui/src/i18n/locales/zh-CN.ts | 5 + mateclaw-ui/src/types/chatError.ts | 20 +- 22 files changed, 1648 insertions(+), 52 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilder.java create mode 100644 mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java create mode 100644 mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentOpenAiCompatibleChatModelBuilder.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatGPTResponsesChatModelBuilder.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatModelBuilder.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ProviderChatModelFactory.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/ProbeResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderProbeStrategy.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/probe/AnthropicListModelsProbe.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/probe/ChatGPTOAuthStatusProbe.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/probe/DashScopeListModelsProbe.java create mode 100644 mateclaw-server/src/main/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbe.java create mode 100644 mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 2bc54133..5476857b 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -131,6 +131,7 @@ public class AgentGraphBuilder { private final vip.mate.tool.ToolConcurrencyRegistry toolConcurrencyRegistry; private final vip.mate.i18n.I18nService i18nService; private final vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker; + private final vip.mate.llm.chatmodel.ProviderChatModelFactory chatModelFactory; /** * 根据 AgentEntity 构建完整的 Agent 实例 @@ -501,47 +502,13 @@ public class AgentGraphBuilder { * 本参数对它们无效（它们各自有内部重试或直通）。 */ public ChatModel buildRuntimeChatModel(ModelConfigEntity runtimeModel, RetryTemplate retryOverride) { - ModelProviderEntity provider = modelProviderService.getProviderConfig(runtimeModel.getProvider()); - ModelProtocol protocol = ModelProtocol.fromChatModel(provider.getChatModel()); - - if (protocol == ModelProtocol.DASHSCOPE_NATIVE) { - DashScopeApi api = buildDashScopeApi(provider); - DashScopeChatOptions options = buildDashScopeOptions(runtimeModel, provider); - return dashScopeChatModel.mutate() - .dashScopeApi(api) - .defaultOptions(options) - .build(); - } - - if (protocol == ModelProtocol.OPENAI_CHATGPT) { - Double temp = runtimeModel.getTemperature() != null ? runtimeModel.getTemperature() : 0.7; - return new vip.mate.llm.chatgpt.ChatGPTChatModel( - chatGPTResponsesClient, runtimeModel.getModelName(), temp); - } - - if (protocol == ModelProtocol.OPENAI_COMPATIBLE) { - OpenAiApi api = buildOpenAiApi(provider); - OpenAiChatOptions options = buildOpenAiOptions(runtimeModel, provider); - return OpenAiChatModel.builder() - .openAiApi(api) - .defaultOptions(options) - .retryTemplate(retryOverride) - .observationRegistry(observationRegistryProvider.getIfAvailable(() -> ObservationRegistry.NOOP)) - .build(); - } - - if (protocol == ModelProtocol.ANTHROPIC_MESSAGES) { - AnthropicApi api = buildAnthropicApi(provider); - AnthropicChatOptions options = buildAnthropicOptions(runtimeModel); - return AnthropicChatModel.builder() - .anthropicApi(api) - .defaultOptions(options) - .retryTemplate(retryOverride) - .observationRegistry(observationRegistryProvider.getIfAvailable(() -> ObservationRegistry.NOOP)) - .build(); - } - - throw new MateClawException("err.agent.protocol_limited", "StateGraph 当前仅支持 DashScope 原生协议、OpenAI-compatible 协议和 Anthropic Messages 协议: " + protocol.getId()); + // PR-0 (RFC-009 Phase 4 prelude): protocol switch extracted to + // ProviderChatModelFactory + per-protocol ChatModelBuilder strategies. + // Per-protocol builders (DashScope / OpenAI-compatible / Anthropic / + // ChatGPT-Responses) live in vip.mate.agent.chatmodel + vip.mate.llm.chatmodel. + // See RFC-009 Phase 4 plan for the rationale (circular-dep break for + // ProviderInitProbe + AgentGraphBuilder slimming). + return chatModelFactory.buildFor(runtimeModel, retryOverride); } /** @@ -787,7 +754,8 @@ public class AgentGraphBuilder { // ==================== 模型选项构建 ==================== - private DashScopeChatOptions buildDashScopeOptions(ModelConfigEntity runtimeModel, ModelProviderEntity provider) { + /** Transitional public visibility for {@code chatmodel} sub-package builders; will move into the builder in PR-0b. */ + public DashScopeChatOptions buildDashScopeOptions(ModelConfigEntity runtimeModel, ModelProviderEntity provider) { DashScopeChatOptions.DashScopeChatOptionsBuilder builder = DashScopeChatOptions.builder(); Map kwargs = modelProviderService.readProviderGenerateKwargs(provider); @@ -821,7 +789,8 @@ public class AgentGraphBuilder { return builder.build(); } - private OpenAiChatOptions buildOpenAiOptions(ModelConfigEntity runtimeModel, ModelProviderEntity provider) { + /** Transitional public visibility for {@code chatmodel} sub-package builders; will move into the builder in PR-0b. */ + public OpenAiChatOptions buildOpenAiOptions(ModelConfigEntity runtimeModel, ModelProviderEntity provider) { OpenAiChatOptions.Builder builder = OpenAiChatOptions.builder(); Map kwargs = modelProviderService.readProviderGenerateKwargs(provider); String modelName = runtimeModel.getModelName(); @@ -903,7 +872,8 @@ public class AgentGraphBuilder { // ==================== OpenAI API 构建 ==================== - OpenAiApi buildOpenAiApi(ModelProviderEntity provider) { + /** Transitional public visibility for {@code chatmodel} sub-package builders; will move into the builder in PR-0b. */ + public OpenAiApi buildOpenAiApi(ModelProviderEntity provider) { if (provider == null || !modelProviderService.isProviderConfigured(provider.getProviderId())) { throw new MateClawException("err.agent.provider_not_configured", "Provider 未完成配置，请在模型设置中填写有效的 API Key 和 Base URL"); } @@ -995,7 +965,8 @@ public class AgentGraphBuilder { // ==================== DashScope API 构建 ==================== - private DashScopeApi buildDashScopeApi(ModelProviderEntity provider) { + /** Transitional public visibility for {@code chatmodel} sub-package builders; will move into the builder in PR-0b. */ + public DashScopeApi buildDashScopeApi(ModelProviderEntity provider) { DashScopeApi.Builder builder = DashScopeApi.builder(); // API Key 回落链：provider UI 配置 → 环境变量/application.yml → 默认 bean 反射 @@ -1028,7 +999,8 @@ public class AgentGraphBuilder { // ==================== Anthropic API 构建 ==================== - private AnthropicApi buildAnthropicApi(ModelProviderEntity provider) { + /** Transitional public visibility for {@code chatmodel} sub-package builders; will move into the builder in PR-0b. */ + public AnthropicApi buildAnthropicApi(ModelProviderEntity provider) { if (provider == null || !modelProviderService.isProviderConfigured(provider.getProviderId())) { throw new MateClawException("err.agent.anthropic_not_configured", "Anthropic Provider 未完成配置，请在模型设置中填写有效的 API Key 和 Base URL"); } @@ -1051,7 +1023,8 @@ public class AgentGraphBuilder { return builder.build(); } - private AnthropicChatOptions buildAnthropicOptions(ModelConfigEntity runtimeModel) { + /** Transitional public visibility for {@code chatmodel} sub-package builders; will move into the builder in PR-0b. */ + public AnthropicChatOptions buildAnthropicOptions(ModelConfigEntity runtimeModel) { AnthropicChatOptions.Builder builder = AnthropicChatOptions.builder(); if (StringUtils.hasText(runtimeModel.getModelName())) { builder.model(runtimeModel.getModelName()); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilder.java new file mode 100644 index 00000000..6fad66d0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilder.java @@ -0,0 +1,51 @@ +package vip.mate.agent.chatmodel; + +import io.micrometer.observation.ObservationRegistry; +import org.springframework.ai.anthropic.AnthropicChatModel; +import org.springframework.ai.anthropic.AnthropicChatOptions; +import org.springframework.ai.anthropic.api.AnthropicApi; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Lazy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Component; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.llm.chatmodel.ChatModelBuilder; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +/** + * Thin strategy adapter for {@link ModelProtocol#ANTHROPIC_MESSAGES}. + * See {@link AgentDashScopeChatModelBuilder} for the delegate-pattern rationale. + */ +@Component +public class AgentAnthropicChatModelBuilder implements ChatModelBuilder { + + private final AgentGraphBuilder agentGraphBuilder; + private final ObjectProvider observationRegistryProvider; + + public AgentAnthropicChatModelBuilder( + @Lazy AgentGraphBuilder agentGraphBuilder, + ObjectProvider observationRegistryProvider) { + this.agentGraphBuilder = agentGraphBuilder; + this.observationRegistryProvider = observationRegistryProvider; + } + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.ANTHROPIC_MESSAGES; + } + + @Override + public ChatModel build(ModelConfigEntity model, ModelProviderEntity provider, RetryTemplate retry) { + AnthropicApi api = agentGraphBuilder.buildAnthropicApi(provider); + AnthropicChatOptions options = agentGraphBuilder.buildAnthropicOptions(model); + return AnthropicChatModel.builder() + .anthropicApi(api) + .defaultOptions(options) + .retryTemplate(retry) + .observationRegistry(observationRegistryProvider.getIfAvailable(() -> ObservationRegistry.NOOP)) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java new file mode 100644 index 00000000..bbf7c77c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java @@ -0,0 +1,62 @@ +package vip.mate.agent.chatmodel; + +import com.alibaba.cloud.ai.dashscope.api.DashScopeApi; +import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatModel; +import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.context.annotation.Lazy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Component; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.llm.chatmodel.ChatModelBuilder; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +/** + * Thin strategy adapter for {@link ModelProtocol#DASHSCOPE_NATIVE}. + * + *

This is a deliberate delegating implementation: it calls back into + * {@link AgentGraphBuilder}'s package-private helpers ({@code buildDashScopeApi}, + * {@code buildDashScopeOptions}) rather than owning the build logic itself. + * The goal of this PR is to install the strategy seam (so + * {@code ProviderChatModelFactory} can route requests without circular + * dependencies) without taking on the risk of relocating ~600 lines of + * provider-specific helpers in a single change.

+ * + *

A follow-up PR (PR-0b in the plan) moves the helpers into this class so + * {@code AgentGraphBuilder} can shed the protocol-specific code entirely.

+ * + *

{@code @Lazy} on the {@link AgentGraphBuilder} dependency breaks the + * factory ↔ builder ↔ AgentGraphBuilder bean-creation cycle: + * AgentGraphBuilder constructs ProviderChatModelFactory, the factory needs + * builders, and this builder needs AgentGraphBuilder back. The lazy proxy + * defers AgentGraphBuilder resolution until the first {@link #build} call.

+ */ +@Component +public class AgentDashScopeChatModelBuilder implements ChatModelBuilder { + + private final AgentGraphBuilder agentGraphBuilder; + private final DashScopeChatModel dashScopeChatModel; + + public AgentDashScopeChatModelBuilder(@Lazy AgentGraphBuilder agentGraphBuilder, + DashScopeChatModel dashScopeChatModel) { + this.agentGraphBuilder = agentGraphBuilder; + this.dashScopeChatModel = dashScopeChatModel; + } + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.DASHSCOPE_NATIVE; + } + + @Override + public ChatModel build(ModelConfigEntity model, ModelProviderEntity provider, RetryTemplate retry) { + DashScopeApi api = agentGraphBuilder.buildDashScopeApi(provider); + DashScopeChatOptions options = agentGraphBuilder.buildDashScopeOptions(model, provider); + return dashScopeChatModel.mutate() + .dashScopeApi(api) + .defaultOptions(options) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentOpenAiCompatibleChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentOpenAiCompatibleChatModelBuilder.java new file mode 100644 index 00000000..a2e0593e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentOpenAiCompatibleChatModelBuilder.java @@ -0,0 +1,53 @@ +package vip.mate.agent.chatmodel; + +import io.micrometer.observation.ObservationRegistry; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.openai.OpenAiChatModel; +import org.springframework.ai.openai.OpenAiChatOptions; +import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Lazy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Component; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.llm.chatmodel.ChatModelBuilder; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +/** + * Thin strategy adapter for {@link ModelProtocol#OPENAI_COMPATIBLE}. + * Delegates to {@link AgentGraphBuilder}'s helpers; see + * {@link AgentDashScopeChatModelBuilder} for the rationale of the delegate + * pattern and the {@code @Lazy} cycle break. + */ +@Component +public class AgentOpenAiCompatibleChatModelBuilder implements ChatModelBuilder { + + private final AgentGraphBuilder agentGraphBuilder; + private final ObjectProvider observationRegistryProvider; + + public AgentOpenAiCompatibleChatModelBuilder( + @Lazy AgentGraphBuilder agentGraphBuilder, + ObjectProvider observationRegistryProvider) { + this.agentGraphBuilder = agentGraphBuilder; + this.observationRegistryProvider = observationRegistryProvider; + } + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.OPENAI_COMPATIBLE; + } + + @Override + public ChatModel build(ModelConfigEntity model, ModelProviderEntity provider, RetryTemplate retry) { + OpenAiApi api = agentGraphBuilder.buildOpenAiApi(provider); + OpenAiChatOptions options = agentGraphBuilder.buildOpenAiOptions(model, provider); + return OpenAiChatModel.builder() + .openAiApi(api) + .defaultOptions(options) + .retryTemplate(retry) + .observationRegistry(observationRegistryProvider.getIfAvailable(() -> ObservationRegistry.NOOP)) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java index 34b6a5a1..a9cfe105 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java @@ -326,10 +326,15 @@ public class NodeStreamingChatHelper { if (lastResult.errorType() == ErrorType.PROMPT_TOO_LONG) { return lastResult; } - // AUTH: 不重试 — 但要记账（auth 不会自愈，连续 N 次后冷却避免每轮都撞） + // AUTH: primary key 失效不会自愈，跳过同模型重试，交给 fallback chain + // — 其它 provider 的 key 可能仍然可用（与 BILLING / MODEL_NOT_FOUND 同策略）。 + // recordPrimary(false) 仍记一次失败用于 healthTracker 冷却累计。 + // 若 fallback chain 全部 401，walker 末尾会把最后一次 AUTH_ERROR 透出， + // 不会静默吞错。 if (lastResult.errorType() == ErrorType.AUTH_ERROR) { + log.warn("[{}] Primary auth failed — skipping same-model retries, handing off to fallback chain", phase); recordPrimary(false); - return lastResult; + break; } // BILLING / MODEL_NOT_FOUND — provider-side hard failures // that won't change on retry. Skip to fallback chain (a different diff --git a/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatGPTResponsesChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatGPTResponsesChatModelBuilder.java new file mode 100644 index 00000000..8df2dff5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatGPTResponsesChatModelBuilder.java @@ -0,0 +1,38 @@ +package vip.mate.llm.chatmodel; + +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Component; +import vip.mate.llm.chatgpt.ChatGPTChatModel; +import vip.mate.llm.chatgpt.ChatGPTResponsesClient; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +/** + * ChatGPT Responses API (the "/codex/responses" endpoint reached via OAuth) + * doesn't go through Spring AI's standard ChatModel builders — it has its own + * {@link ChatGPTChatModel} wrapper around {@link ChatGPTResponsesClient}. + * The {@link RetryTemplate} parameter is ignored because retry happens inside + * the OAuth-aware client itself. + */ +@Component +public class ChatGPTResponsesChatModelBuilder implements ChatModelBuilder { + + private final ChatGPTResponsesClient chatGPTResponsesClient; + + public ChatGPTResponsesChatModelBuilder(ChatGPTResponsesClient chatGPTResponsesClient) { + this.chatGPTResponsesClient = chatGPTResponsesClient; + } + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.OPENAI_CHATGPT; + } + + @Override + public ChatModel build(ModelConfigEntity model, ModelProviderEntity provider, RetryTemplate retry) { + Double temp = model.getTemperature() != null ? model.getTemperature() : 0.7; + return new ChatGPTChatModel(chatGPTResponsesClient, model.getModelName(), temp); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatModelBuilder.java new file mode 100644 index 00000000..2c550ae8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ChatModelBuilder.java @@ -0,0 +1,45 @@ +package vip.mate.llm.chatmodel; + +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.retry.support.RetryTemplate; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +/** + * Strategy contract for assembling a Spring AI {@link ChatModel} from a + * persisted {@link ModelConfigEntity} + {@link ModelProviderEntity} pair. + * + *

One implementation per {@link ModelProtocol}. New provider protocols + * are added by registering a new {@code @Component} that implements this + * interface — no edits to {@link ProviderChatModelFactory} or + * {@code AgentGraphBuilder} required.

+ * + *

This interface lives in {@code vip.mate.llm.chatmodel} (under the {@code llm} + * package) so that {@code llm.failover.ProviderInitProbe} can build a + * {@link ChatModel} for health-probing without depending on the {@code agent} + * package — which would create the circular dependency + * {@code agent → llm → agent}. Prior to this extraction, all model-building + * logic lived inside {@code AgentGraphBuilder}.

+ */ +public interface ChatModelBuilder { + + /** The protocol this builder handles; the factory routes by this key. */ + ModelProtocol supportedProtocol(); + + /** + * Build a fresh {@link ChatModel} for the given runtime configuration. + * Implementations must be stateless — callers may invoke this many times + * for the same model id and expect equivalent (but not necessarily ==) + * results. + * + * @param model runtime model row with temperature / max tokens / etc. + * @param provider provider row supplying API key, base URL, and provider-level + * generate kwargs + * @param retry retry template to wire into the underlying Spring AI client + * where supported. Implementations whose protocols don't + * expose a Spring AI {@code RetryTemplate} hook (DashScope + * native, ChatGPT Responses) may ignore this parameter. + */ + ChatModel build(ModelConfigEntity model, ModelProviderEntity provider, RetryTemplate retry); +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ProviderChatModelFactory.java b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ProviderChatModelFactory.java new file mode 100644 index 00000000..7224f127 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/chatmodel/ProviderChatModelFactory.java @@ -0,0 +1,74 @@ +package vip.mate.llm.chatmodel; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Component; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.service.ModelProviderService; + +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +/** + * Routes {@link ChatModel} construction to the appropriate + * {@link ChatModelBuilder} based on the provider's declared protocol. + * + *

Resolution order:

+ *
    + *
  1. Look up the {@link ModelProviderEntity} for {@code model.getProvider()}
    2. + *
  2. Map its {@code chatModel} string to a {@link ModelProtocol}
    3. + *
  3. Dispatch to the matching {@link ChatModelBuilder}; throw if none registered
    4. + *
+ * + *

This factory is the single seam that {@code AgentGraphBuilder} and + * {@code ProviderInitProbe} share for building Spring AI clients. Keeping it + * in the {@code llm} package preserves the dependency direction + * {@code agent → llm} and lets the failover layer probe providers without + * pulling in the {@code agent} package.

+ */ +@Slf4j +@Component +public class ProviderChatModelFactory { + + private final Map builders; + private final ModelProviderService modelProviderService; + + public ProviderChatModelFactory(List allBuilders, + ModelProviderService modelProviderService) { + this.modelProviderService = modelProviderService; + Map map = new EnumMap<>(ModelProtocol.class); + for (ChatModelBuilder b : allBuilders) { + ChatModelBuilder previous = map.put(b.supportedProtocol(), b); + if (previous != null) { + throw new IllegalStateException( + "Two ChatModelBuilders registered for the same protocol " + + b.supportedProtocol() + ": " + + previous.getClass().getName() + " vs " + b.getClass().getName()); + } + } + this.builders = Map.copyOf(map); + log.info("[ProviderChatModelFactory] registered builders for protocols: {}", builders.keySet()); + } + + /** + * Build a {@link ChatModel} for the given runtime model, looking up the + * provider on the fly. Throws {@link MateClawException} when no builder + * is registered for the resolved protocol — callers should treat this as + * a configuration error, not a transient failure. + */ + public ChatModel buildFor(ModelConfigEntity model, RetryTemplate retry) { + ModelProviderEntity provider = modelProviderService.getProviderConfig(model.getProvider()); + ModelProtocol protocol = ModelProtocol.fromChatModel(provider.getChatModel()); + ChatModelBuilder builder = builders.get(protocol); + if (builder == null) { + throw new MateClawException("err.agent.protocol_limited", + "No ChatModelBuilder registered for protocol: " + protocol.getId()); + } + return builder.build(model, provider, retry); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java new file mode 100644 index 00000000..76e17425 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/AvailableProviderPool.java @@ -0,0 +1,122 @@ +package vip.mate.llm.failover; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * In-memory registry of providers currently considered usable for new chat + * requests. Membership is the gate the failover walker checks: a provider not + * in this pool is skipped entirely without attempting an LLM call. + * + *

Two state transitions:

+ *
    + *
  • Add — at startup ({@code ProviderInitProbe}), on user-triggered + * reprobe, or after a {@code ModelConfigChangedEvent}.
    • + *
  • Remove — when a request hits a HARD error (AUTH_ERROR / + * BILLING / MODEL_NOT_FOUND) — these don't self-heal, so retrying on + * every subsequent call wastes the user's time. SOFT errors + * (RATE_LIMIT / SERVER_ERROR / EMPTY_RESPONSE) keep the provider in + * the pool and are handled by {@link ProviderHealthTracker}'s short + * cooldown instead.
    • + *
+ * + *

State is process-local; a restart re-runs the init probe. That's + * intentional — full distributed coordination is out of scope for v1 + * (single-node and desktop deployments are the primary targets).

+ */ +@Slf4j +@Component +public class AvailableProviderPool { + + /** Membership: providers currently usable. Add-on-success / remove-on-HARD. */ + private final Set members = ConcurrentHashMap.newKeySet(); + + /** + * Last removal reason per provider id. Cleared when the provider is + * re-added. Useful for the UI to explain "why is this provider down?" + * and for diagnostics. + */ + private final Map removalReasons = new ConcurrentHashMap<>(); + + /** Add (or re-add) a provider to the pool. Clears any prior removal reason. */ + public void add(String providerId) { + if (providerId == null || providerId.isEmpty()) return; + boolean newlyAdded = members.add(providerId); + RemovalReason previous = removalReasons.remove(providerId); + if (newlyAdded && previous != null) { + log.info("[Pool] re-adding provider={} (previous removal: {})", providerId, previous); + } else if (newlyAdded) { + log.info("[Pool] adding provider={}", providerId); + } + } + + /** + * Remove a provider from the pool with a reason. Idempotent — calling + * twice updates the reason (so the latest cause wins) but doesn't double-log. + */ + public void remove(String providerId, RemovalSource source, String message) { + if (providerId == null || providerId.isEmpty()) return; + boolean wasMember = members.remove(providerId); + RemovalReason reason = new RemovalReason(source, message, Instant.now().toEpochMilli()); + removalReasons.put(providerId, reason); + if (wasMember) { + log.warn("[Pool] removing provider={} due to {} ({})", providerId, source, message); + } else { + log.debug("[Pool] removal reason updated for already-out provider={}: {} ({})", + providerId, source, message); + } + } + + /** Membership check — the walker / primary short-circuit consults this on every entry. */ + public boolean contains(String providerId) { + return providerId != null && members.contains(providerId); + } + + /** + * Diagnostic snapshot for admin endpoints / tests. Returns providerId → + * either {@code null} (in pool) or the latest {@link RemovalReason}. + * The returned map is a stable copy; in-pool entries are present with + * {@code null} value so callers can iterate the union of in/out. + */ + public Map snapshot() { + Map out = new LinkedHashMap<>(); + for (String id : members) out.put(id, null); + removalReasons.forEach((id, reason) -> { + if (!out.containsKey(id)) out.put(id, reason); + }); + return out; + } + + /** Clear everything. Intended for tests; no production code path calls this. */ + void reset() { + members.clear(); + removalReasons.clear(); + } + + // ============================================================ + // Records + // ============================================================ + + /** Why a provider was removed from the pool. {@code removedAtMs} is epoch milliseconds. */ + public record RemovalReason(RemovalSource source, String message, long removedAtMs) {} + + /** + * Categorical source of a pool removal. Mirrors the HARD error types from + * {@code NodeStreamingChatHelper.ErrorType} plus {@link #INIT_PROBE} for + * startup probe failures. SOFT errors (RATE_LIMIT / SERVER_ERROR) never + * appear here — they're handled by {@link ProviderHealthTracker} cooldown. + */ + public enum RemovalSource { + AUTH_ERROR, + BILLING, + MODEL_NOT_FOUND, + INIT_PROBE, + MANUAL + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProbeResult.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProbeResult.java new file mode 100644 index 00000000..487e43f9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProbeResult.java @@ -0,0 +1,19 @@ +package vip.mate.llm.failover; + +/** + * Outcome of a single provider health probe. + * + * @param success true if the provider responded usable + * @param latencyMs wall-clock latency of the probe in milliseconds; {@code 0} if it threw before measuring + * @param errorMessage human-readable failure detail; {@code null} on success + */ +public record ProbeResult(boolean success, long latencyMs, String errorMessage) { + + public static ProbeResult ok(long latencyMs) { + return new ProbeResult(true, latencyMs, null); + } + + public static ProbeResult fail(long latencyMs, String message) { + return new ProbeResult(false, latencyMs, message); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java new file mode 100644 index 00000000..8be0804f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java @@ -0,0 +1,197 @@ +package vip.mate.llm.failover; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.repository.ModelProviderMapper; +import vip.mate.llm.service.ModelProviderService; + +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; + +/** + * RFC-009 Phase 4 — startup-time provider liveness check. + * + *

On {@link ApplicationReadyEvent} (after Flyway, all beans, and + * {@link ModelProviderService} are ready), enumerates every configured + * provider and probes it in parallel via the protocol-specific + * {@link ProviderProbeStrategy} bean. Healthy providers are added to + * {@link AvailableProviderPool}; failed ones are removed with + * {@link AvailableProviderPool.RemovalSource#INIT_PROBE}.

+ * + *

Fail-open semantics — the whole batch is bounded by + * {@link #BATCH_TIMEOUT_MS} (10 s). Any provider whose probe is still in + * flight at that point is added to the pool by default; the chat path + * will validate it on first request. This avoids an SSL hang on one + * provider gating user-visible chat.

+ * + *

What's NOT done here — periodic re-probing. The plan calls + * for re-add triggers only on (a) restart (this class), (b) a + * {@code ModelConfigChangedEvent} (PR-1e), or (c) the manual reprobe + * REST endpoint (PR-1e). Once a provider is HARD-removed at runtime, + * nothing in this class brings it back automatically.

+ */ +@Slf4j +@Component +public class ProviderInitProbe { + + /** Hard ceiling on the parallel batch — stalled probes don't block startup beyond this. */ + private static final long BATCH_TIMEOUT_MS = 10_000L; + + private final ModelProviderMapper providerMapper; + private final ModelProviderService providerService; + private final AvailableProviderPool pool; + private final Map strategies; + + public ProviderInitProbe(ModelProviderMapper providerMapper, + ModelProviderService providerService, + AvailableProviderPool pool, + List probeStrategies) { + this.providerMapper = providerMapper; + this.providerService = providerService; + this.pool = pool; + Map map = new EnumMap<>(ModelProtocol.class); + for (ProviderProbeStrategy s : probeStrategies) { + ProviderProbeStrategy prev = map.put(s.supportedProtocol(), s); + if (prev != null) { + throw new IllegalStateException("Duplicate ProviderProbeStrategy for protocol " + + s.supportedProtocol() + ": " + prev.getClass().getSimpleName() + + " vs " + s.getClass().getSimpleName()); + } + } + this.strategies = map; + } + + @EventListener(ApplicationReadyEvent.class) + public void onApplicationReady() { + probeAllConfigured(); + } + + /** + * Probe every configured provider in parallel. Public so PR-1e's manual + * reprobe / config-changed listener can trigger a full refresh, not just + * a single-provider one. + */ + public void probeAllConfigured() { + List providers = listConfiguredProviders(); + if (providers.isEmpty()) { + log.info("[ProviderInitProbe] no configured providers — nothing to probe"); + return; + } + log.info("[ProviderInitProbe] probing {} configured provider(s)...", providers.size()); + + ExecutorService executor = Executors.newFixedThreadPool( + Math.min(providers.size(), 8), + r -> { + Thread t = new Thread(r, "provider-init-probe"); + t.setDaemon(true); + return t; + }); + Map> futures = new ConcurrentHashMap<>(); + try { + for (ModelProviderEntity provider : providers) { + ProviderProbeStrategy strategy = strategies.get(resolveProtocol(provider)); + if (strategy == null) { + // No probe strategy registered for this protocol — fail-open: assume usable. + log.debug("[ProviderInitProbe] no probe strategy for {} (protocol={}), defaulting to in-pool", + provider.getProviderId(), provider.getChatModel()); + pool.add(provider.getProviderId()); + continue; + } + futures.put(provider.getProviderId(), + executor.submit(() -> strategy.probe(provider))); + } + + long deadline = System.currentTimeMillis() + BATCH_TIMEOUT_MS; + int passed = 0, failed = 0, deferred = 0; + for (Map.Entry> e : futures.entrySet()) { + String id = e.getKey(); + long remaining = deadline - System.currentTimeMillis(); + ProbeResult result = null; + if (remaining > 0) { + try { + result = e.getValue().get(remaining, TimeUnit.MILLISECONDS); + } catch (TimeoutException te) { + // fall through to deferred + } catch (Exception ex) { + result = ProbeResult.fail(0, "probe threw: " + ex.getClass().getSimpleName()); + } + } + if (result == null) { + // Fail-open: didn't finish within the batch budget; assume usable so chat + // isn't gated on a slow probe. First real request will validate. + pool.add(id); + deferred++; + log.warn("[ProviderInitProbe] provider={} probe deferred (still running at {}ms cap) — fail-open into pool", + id, BATCH_TIMEOUT_MS); + } else if (result.success()) { + pool.add(id); + passed++; + log.info("[ProviderInitProbe] provider={} OK ({} ms)", id, result.latencyMs()); + } else { + pool.remove(id, AvailableProviderPool.RemovalSource.INIT_PROBE, + "init probe failed: " + result.errorMessage()); + failed++; + log.warn("[ProviderInitProbe] provider={} FAIL ({} ms): {}", + id, result.latencyMs(), result.errorMessage()); + } + } + log.info("[ProviderInitProbe] done — passed={}, failed={}, deferred={}, pool size={}", + passed, failed, deferred, pool.snapshot().size()); + } finally { + // Outstanding futures are interrupted; threads are daemon and won't hold shutdown. + executor.shutdownNow(); + } + } + + /** + * Probe one provider on demand. Used by PR-1e's manual reprobe endpoint + * and {@code ModelConfigChangedEvent} listener. + * + * @return the {@link ProbeResult}; pool state is also updated as a side effect. + */ + public ProbeResult probeOne(String providerId) { + if (!StringUtils.hasText(providerId)) { + return ProbeResult.fail(0, "providerId is blank"); + } + ModelProviderEntity provider = providerMapper.selectById(providerId); + if (provider == null) { + return ProbeResult.fail(0, "provider not found: " + providerId); + } + if (!providerService.isProviderConfigured(providerId)) { + ProbeResult r = ProbeResult.fail(0, "provider not configured"); + pool.remove(providerId, AvailableProviderPool.RemovalSource.INIT_PROBE, r.errorMessage()); + return r; + } + ProviderProbeStrategy strategy = strategies.get(resolveProtocol(provider)); + if (strategy == null) { + // No strategy for this protocol — fail-open: assume usable. + pool.add(providerId); + return ProbeResult.ok(0); + } + ProbeResult result = strategy.probe(provider); + if (result.success()) { + pool.add(providerId); + } else { + pool.remove(providerId, AvailableProviderPool.RemovalSource.INIT_PROBE, + "reprobe failed: " + result.errorMessage()); + } + return result; + } + + private List listConfiguredProviders() { + return providerMapper.selectList(null).stream() + .filter(p -> providerService.isProviderConfigured(p.getProviderId())) + .toList(); + } + + private static ModelProtocol resolveProtocol(ModelProviderEntity provider) { + return ModelProtocol.fromChatModel(provider.getChatModel()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderProbeStrategy.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderProbeStrategy.java new file mode 100644 index 00000000..f71f4ed3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderProbeStrategy.java @@ -0,0 +1,34 @@ +package vip.mate.llm.failover; + +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +/** + * Strategy contract for actively probing one provider's reachability. + * + *

One implementation per {@link ModelProtocol}. {@code ProviderInitProbe} + * dispatches by {@link #supportedProtocol()}. New protocols add a new + * {@code @Component} implementing this interface — no edits to + * {@code ProviderInitProbe} required.

+ * + *

Implementations should:

+ *
    + *
  • Prefer free endpoints (e.g., {@code GET /v1/models} for OpenAI / Anthropic) + * over chat completions whenever possible — keeps probe cost at zero.
    • + *
  • Use a short HTTP timeout (~5s) so a stalled probe doesn't block + * the parallel batch.
    • + *
  • Never throw — wrap exceptions into {@link ProbeResult#fail}.
    • + *
+ */ +public interface ProviderProbeStrategy { + + /** The protocol this probe handles; {@code ProviderInitProbe} routes by this key. */ + ModelProtocol supportedProtocol(); + + /** + * Probe the given provider. Must complete (success or fail) within the + * caller's timeout budget. Implementations are expected to honor a + * conservative HTTP timeout (~5s) internally. + */ + ProbeResult probe(ModelProviderEntity provider); +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/AnthropicListModelsProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/AnthropicListModelsProbe.java new file mode 100644 index 00000000..7f1d701f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/AnthropicListModelsProbe.java @@ -0,0 +1,76 @@ +package vip.mate.llm.failover.probe; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestClient; +import vip.mate.llm.failover.ProbeResult; +import vip.mate.llm.failover.ProviderProbeStrategy; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +import java.net.http.HttpClient; +import java.time.Duration; + +/** + * Probes Anthropic by calling {@code GET /v1/models} — free, validates the + * x-api-key header in one round-trip. Default base URL is + * {@code https://api.anthropic.com} when the provider doesn't override it. + */ +@Slf4j +@Component +public class AnthropicListModelsProbe implements ProviderProbeStrategy { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final String DEFAULT_BASE = "https://api.anthropic.com"; + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.ANTHROPIC_MESSAGES; + } + + @Override + public ProbeResult probe(ModelProviderEntity provider) { + if (provider == null || !StringUtils.hasText(provider.getApiKey())) { + return ProbeResult.fail(0, "API key not configured"); + } + String baseUrl = StringUtils.hasText(provider.getBaseUrl()) + ? normalizeBaseUrl(provider.getBaseUrl()) + : DEFAULT_BASE; + long start = System.currentTimeMillis(); + try { + HttpClient httpClient = HttpClient.newBuilder().connectTimeout(TIMEOUT).build(); + RestClient client = RestClient.builder() + .baseUrl(baseUrl) + .requestFactory(new JdkClientHttpRequestFactory(httpClient)) + .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) + .defaultHeader("x-api-key", provider.getApiKey().trim()) + .defaultHeader("anthropic-version", "2023-06-01") + .build(); + + String body = client.get().uri("/v1/models").retrieve().body(String.class); + long latency = System.currentTimeMillis() - start; + if (body == null || body.isBlank()) { + return ProbeResult.fail(latency, "empty body from /v1/models"); + } + return ProbeResult.ok(latency); + } catch (Exception e) { + long latency = System.currentTimeMillis() - start; + log.debug("[Probe] anthropic /v1/models failed: {}", e.getMessage()); + return ProbeResult.fail(latency, shortMessage(e)); + } + } + + private static String normalizeBaseUrl(String url) { + return url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + } + + private static String shortMessage(Throwable t) { + String m = t.getMessage(); + if (m == null) m = t.getClass().getSimpleName(); + return m.length() > 200 ? m.substring(0, 200) + "..." : m; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/ChatGPTOAuthStatusProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/ChatGPTOAuthStatusProbe.java new file mode 100644 index 00000000..0ef671cb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/ChatGPTOAuthStatusProbe.java @@ -0,0 +1,67 @@ +package vip.mate.llm.failover.probe; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.llm.failover.ProbeResult; +import vip.mate.llm.failover.ProviderProbeStrategy; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.oauth.OpenAIOAuthService; + +/** + * Probes the ChatGPT Responses provider by validating its OAuth credentials. + * No HTTP call: chatgpt.com/backend-api has no free liveness endpoint, and a + * real chat completion costs tokens — so we ping nothing and instead verify + * the OAuth state stored locally: + *
    + *
  1. access token present (and refreshable if near expiry, via + * {@link OpenAIOAuthService#ensureValidAccessToken()}),
    2. + *
  2. account id resolvable (header required by every Responses call).
    3. + *
+ * If both succeed without throwing, the credentials are usable. A real auth + * outage is then surfaced reactively on first chat (HARD-removed by + * {@code NodeStreamingChatHelper}). + */ +@Slf4j +@Component +public class ChatGPTOAuthStatusProbe implements ProviderProbeStrategy { + + private final OpenAIOAuthService oauthService; + + public ChatGPTOAuthStatusProbe(OpenAIOAuthService oauthService) { + this.oauthService = oauthService; + } + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.OPENAI_CHATGPT; + } + + @Override + public ProbeResult probe(ModelProviderEntity provider) { + long start = System.currentTimeMillis(); + try { + String token = oauthService.ensureValidAccessToken(); + if (!StringUtils.hasText(token)) { + return ProbeResult.fail(System.currentTimeMillis() - start, "OAuth access token missing"); + } + String accountId = oauthService.getAccountId(); + if (!StringUtils.hasText(accountId)) { + return ProbeResult.fail(System.currentTimeMillis() - start, + "OAuth account id missing — re-login required"); + } + return ProbeResult.ok(System.currentTimeMillis() - start); + } catch (Exception e) { + long latency = System.currentTimeMillis() - start; + log.debug("[Probe] chatgpt OAuth status failed: {}", e.getMessage()); + return ProbeResult.fail(latency, shortMessage(e)); + } + } + + private static String shortMessage(Throwable t) { + String m = t.getMessage(); + if (m == null) m = t.getClass().getSimpleName(); + return m.length() > 200 ? m.substring(0, 200) + "..." : m; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/DashScopeListModelsProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/DashScopeListModelsProbe.java new file mode 100644 index 00000000..c16ae697 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/DashScopeListModelsProbe.java @@ -0,0 +1,71 @@ +package vip.mate.llm.failover.probe; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestClient; +import vip.mate.llm.failover.ProbeResult; +import vip.mate.llm.failover.ProviderProbeStrategy; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +import java.net.http.HttpClient; +import java.time.Duration; + +/** + * Probes DashScope by hitting its OpenAI-compatible {@code /v1/models} + * endpoint at {@code https://dashscope.aliyuncs.com/compatible-mode}. + * The native DashScope protocol has no equivalent free endpoint, but the + * compatible-mode listing is free and authenticates the same API key. The + * runtime native chat path still uses the native endpoint; this is purely + * a liveness probe. + */ +@Slf4j +@Component +public class DashScopeListModelsProbe implements ProviderProbeStrategy { + + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final String COMPATIBLE_BASE = "https://dashscope.aliyuncs.com/compatible-mode"; + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.DASHSCOPE_NATIVE; + } + + @Override + public ProbeResult probe(ModelProviderEntity provider) { + if (provider == null || !StringUtils.hasText(provider.getApiKey())) { + return ProbeResult.fail(0, "API key not configured"); + } + long start = System.currentTimeMillis(); + try { + HttpClient httpClient = HttpClient.newBuilder().connectTimeout(TIMEOUT).build(); + RestClient client = RestClient.builder() + .baseUrl(COMPATIBLE_BASE) + .requestFactory(new JdkClientHttpRequestFactory(httpClient)) + .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) + .defaultHeader(HttpHeaders.AUTHORIZATION, "Bearer " + provider.getApiKey().trim()) + .build(); + + String body = client.get().uri("/v1/models").retrieve().body(String.class); + long latency = System.currentTimeMillis() - start; + if (body == null || body.isBlank()) { + return ProbeResult.fail(latency, "empty body from /v1/models"); + } + return ProbeResult.ok(latency); + } catch (Exception e) { + long latency = System.currentTimeMillis() - start; + log.debug("[Probe] dashscope /v1/models failed: {}", e.getMessage()); + return ProbeResult.fail(latency, shortMessage(e)); + } + } + + private static String shortMessage(Throwable t) { + String m = t.getMessage(); + if (m == null) m = t.getClass().getSimpleName(); + return m.length() > 200 ? m.substring(0, 200) + "..." : m; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbe.java new file mode 100644 index 00000000..c796fb6d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbe.java @@ -0,0 +1,81 @@ +package vip.mate.llm.failover.probe; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.client.JdkClientHttpRequestFactory; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestClient; +import vip.mate.llm.failover.ProbeResult; +import vip.mate.llm.failover.ProviderProbeStrategy; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; + +import java.net.http.HttpClient; +import java.time.Duration; + +/** + * Probes an OpenAI-compatible provider by calling its {@code GET /v1/models} + * endpoint — free (zero token cost) and authenticates the API key in one + * round-trip. A 200 response with body present is sufficient to confirm + * reachability + auth; we don't try to parse the model list because + * different providers (Kimi / DeepSeek / Moonshot / OpenRouter) have minor + * schema differences that aren't relevant to a liveness check. + */ +@Slf4j +@Component +public class OpenAiCompatibleListModelsProbe implements ProviderProbeStrategy { + + /** Conservative HTTP timeout — keeps a stalled provider from holding up the parallel batch. */ + private static final Duration TIMEOUT = Duration.ofSeconds(5); + + @Override + public ModelProtocol supportedProtocol() { + return ModelProtocol.OPENAI_COMPATIBLE; + } + + @Override + public ProbeResult probe(ModelProviderEntity provider) { + if (provider == null || !StringUtils.hasText(provider.getBaseUrl())) { + return ProbeResult.fail(0, "base URL not configured"); + } + long start = System.currentTimeMillis(); + try { + HttpClient httpClient = HttpClient.newBuilder().connectTimeout(TIMEOUT).build(); + RestClient client = RestClient.builder() + .baseUrl(normalizeBaseUrl(provider.getBaseUrl())) + .requestFactory(new JdkClientHttpRequestFactory(httpClient)) + .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) + .build(); + + RestClient.RequestHeadersSpec spec = client.get().uri("/v1/models"); + String apiKey = provider.getApiKey(); + if (StringUtils.hasText(apiKey)) { + spec = spec.header(HttpHeaders.AUTHORIZATION, "Bearer " + apiKey.trim()); + } + + String body = spec.retrieve().body(String.class); + long latency = System.currentTimeMillis() - start; + if (body == null || body.isBlank()) { + return ProbeResult.fail(latency, "empty body from /v1/models"); + } + return ProbeResult.ok(latency); + } catch (Exception e) { + long latency = System.currentTimeMillis() - start; + log.debug("[Probe] {} /v1/models failed: {}", provider.getProviderId(), e.getMessage()); + return ProbeResult.fail(latency, shortMessage(e)); + } + } + + private static String normalizeBaseUrl(String url) { + // Strip trailing slash for clean URL composition; /v1/models then concatenates correctly. + return url.endsWith("/") ? url.substring(0, url.length() - 1) : url; + } + + private static String shortMessage(Throwable t) { + String m = t.getMessage(); + if (m == null) m = t.getClass().getSimpleName(); + return m.length() > 200 ? m.substring(0, 200) + "..." : m; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java new file mode 100644 index 00000000..3d322669 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java @@ -0,0 +1,190 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.llm.failover.FallbackEntry; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Regression test for the AUTH_ERROR-must-fall-back fix. + * + *

Prior to this fix, primary AUTH_ERROR (e.g. Kimi 401 with an invalid + * API key) returned immediately without trying the fallback chain — a + * fallback provider with a different, valid key never got a chance. + * After the fix, AUTH_ERROR breaks out of the same-model retry loop + * and falls through to the chain walker, mirroring how BILLING and + * MODEL_NOT_FOUND already behave.

+ */ +class NodeStreamingChatHelperFailoverTest { + + private ChatStreamTracker streamTracker; + private ProviderHealthTracker healthTracker; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + ProviderHealthProperties props = new ProviderHealthProperties(); + healthTracker = new ProviderHealthTracker(props); + } + + /** Build a chat-model mock whose stream() emits a single successful chunk with the given text. */ + private static ChatModel successModel(String text) { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(text), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + /** Build a chat-model mock whose stream() errors with the given Throwable. */ + private static ChatModel errorModel(Throwable err) { + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn(Flux.error(err)); + return m; + } + + private NodeStreamingChatHelper helper(ChatModel primary, List chain, String primaryProviderId) { + // Construct via the full constructor so health tracking is wired and the + // chain walker has provider-id context. + return new NodeStreamingChatHelper(streamTracker, chain, null, healthTracker, primaryProviderId); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + // ============================================================ + // C1: primary 401 + fallback#1 success → fallback wins + // ============================================================ + + @Test + @DisplayName("C1: primary AUTH_ERROR triggers fallback chain (was: returned immediately, never tried fallback)") + void primaryAuthErrorFallsBackToHealthyProvider() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized: Invalid API Key")); + ChatModel fallback = successModel("hello from fallback"); + var helper = helper(primary, List.of(new FallbackEntry("dashscope", fallback)), "kimi"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c1", "reasoning"); + + assertEquals("hello from fallback", result.text(), + "fallback provider must succeed and its text must surface as the result"); + assertEquals(NodeStreamingChatHelper.ErrorType.NONE, result.errorType()); + // Primary was tried exactly once (no same-model retries on AUTH_ERROR — fix verified) + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // C2: primary 401 + fallback#1 401 + fallback#2 success + // ============================================================ + + @Test + @DisplayName("C2: chain walks past auth-failing fallback to the next healthy one") + void chainSkipsAuthFailingFallback() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized")); + ChatModel fbBad = errorModel(new RuntimeException("401 Unauthorized: bad key")); + ChatModel fbGood = successModel("ok via 2nd fallback"); + var helper = helper(primary, List.of( + new FallbackEntry("openai", fbBad), + new FallbackEntry("dashscope", fbGood)), "kimi"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c2", "reasoning"); + + assertEquals("ok via 2nd fallback", result.text()); + assertEquals(NodeStreamingChatHelper.ErrorType.NONE, result.errorType()); + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fbBad, times(1)).stream(any(Prompt.class)); + verify(fbGood, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // C3: primary 401 + every fallback 401 → last AUTH_ERROR surfaces + // ============================================================ + + @Test + @DisplayName("C3: when entire chain is auth-failing, last AUTH_ERROR is surfaced (not silently dropped)") + void allChainAuthFailsSurfacesLastError() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized — kimi")); + ChatModel fb1 = errorModel(new RuntimeException("401 Unauthorized — openai")); + ChatModel fb2 = errorModel(new RuntimeException("401 Unauthorized — dashscope")); + var helper = helper(primary, List.of( + new FallbackEntry("openai", fb1), + new FallbackEntry("dashscope", fb2)), "kimi"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c3", "reasoning"); + + assertNotNull(result, "result must not be null even when whole chain fails"); + assertEquals(NodeStreamingChatHelper.ErrorType.AUTH_ERROR, result.errorType(), + "last seen AUTH_ERROR must propagate so callers can surface a real error"); + // Each rung tried exactly once + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fb1, times(1)).stream(any(Prompt.class)); + verify(fb2, times(1)).stream(any(Prompt.class)); + // Health tracker should have recorded a failure against every fallback provider + var snap = healthTracker.snapshot(); + assertTrue(snap.get("openai").consecutiveFailures() >= 1, "openai failure must be recorded"); + assertTrue(snap.get("dashscope").consecutiveFailures() >= 1, "dashscope failure must be recorded"); + } + + // ============================================================ + // C4 regression: BILLING still falls back unchanged + // ============================================================ + + @Test + @DisplayName("C4 (regression): primary BILLING still triggers fallback (unchanged P3.2)") + void billingStillFallsBack() { + ChatModel primary = errorModel(new RuntimeException("402 Payment Required: insufficient_quota")); + ChatModel fallback = successModel("recovered via fallback"); + var helper = helper(primary, List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c4", "reasoning"); + + assertEquals("recovered via fallback", result.text()); + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // Bonus: confirm no infinite loop / regression on success path + // ============================================================ + + @Test + @DisplayName("Bonus: primary success path is unaffected — no fallback call") + void primarySuccessSkipsFallback() { + ChatModel primary = successModel("primary works fine"); + AtomicInteger fallbackCalls = new AtomicInteger(); + ChatModel fallback = mock(ChatModel.class); + when(fallback.stream(any(Prompt.class))).thenAnswer(inv -> { + fallbackCalls.incrementAndGet(); + return Flux.just((ChatResponse) null); + }); + var helper = helper(primary, List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-bonus", "reasoning"); + + assertEquals("primary works fine", result.text()); + assertEquals(0, fallbackCalls.get(), "primary success must not touch the fallback chain"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java new file mode 100644 index 00000000..33f4d7f3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java @@ -0,0 +1,153 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; +import static vip.mate.llm.failover.AvailableProviderPool.RemovalSource; + +/** + * Unit tests for {@link AvailableProviderPool} — the membership data structure + * that gates the failover walker. + */ +class AvailableProviderPoolTest { + + private AvailableProviderPool pool; + + @BeforeEach + void setUp() { + pool = new AvailableProviderPool(); + } + + @Test + @DisplayName("New pool: nothing is in it") + void newPoolEmpty() { + assertFalse(pool.contains("openai")); + assertTrue(pool.snapshot().isEmpty()); + } + + @Test + @DisplayName("add then contains") + void addThenContains() { + pool.add("openai"); + assertTrue(pool.contains("openai")); + assertFalse(pool.contains("anthropic")); + } + + @Test + @DisplayName("Adding twice is idempotent") + void addIdempotent() { + pool.add("openai"); + pool.add("openai"); + assertTrue(pool.contains("openai")); + assertEquals(1, pool.snapshot().size()); + } + + @Test + @DisplayName("Remove after add: pool no longer contains, snapshot exposes reason") + void removeAfterAdd() { + pool.add("openai"); + pool.remove("openai", RemovalSource.AUTH_ERROR, "401 Unauthorized"); + + assertFalse(pool.contains("openai")); + var snap = pool.snapshot(); + assertEquals(1, snap.size()); + assertNotNull(snap.get("openai")); + assertEquals(RemovalSource.AUTH_ERROR, snap.get("openai").source()); + assertEquals("401 Unauthorized", snap.get("openai").message()); + assertTrue(snap.get("openai").removedAtMs() > 0, "removedAtMs must be set"); + } + + @Test + @DisplayName("Remove without prior add still records reason (idempotent removal)") + void removeWithoutAddIsIdempotent() { + pool.remove("openai", RemovalSource.INIT_PROBE, "init failed"); + assertFalse(pool.contains("openai")); + assertNotNull(pool.snapshot().get("openai")); + } + + @Test + @DisplayName("Re-add after remove: contains true, removal reason cleared") + void readdClearsRemovalReason() { + pool.add("openai"); + pool.remove("openai", RemovalSource.AUTH_ERROR, "bad key"); + assertNotNull(pool.snapshot().get("openai")); + + pool.add("openai"); + assertTrue(pool.contains("openai")); + // Snapshot now shows openai in pool (value null), no stale reason + assertNull(pool.snapshot().get("openai"), + "re-adding a provider must clear its prior removal reason"); + } + + @Test + @DisplayName("Snapshot mixes in-pool (value=null) and removed (value=reason) entries") + void snapshotMixedView() { + pool.add("openai"); + pool.add("dashscope"); + pool.remove("anthropic", RemovalSource.MODEL_NOT_FOUND, "model claude-99 not found"); + + var snap = pool.snapshot(); + assertEquals(3, snap.size()); + assertNull(snap.get("openai"), "in-pool members appear with null value"); + assertNull(snap.get("dashscope")); + assertNotNull(snap.get("anthropic")); + assertEquals(RemovalSource.MODEL_NOT_FOUND, snap.get("anthropic").source()); + } + + @Test + @DisplayName("Null/empty providerId is a no-op (defensive)") + void nullEmptySafe() { + pool.add(null); + pool.add(""); + pool.remove(null, RemovalSource.AUTH_ERROR, "x"); + pool.remove("", RemovalSource.AUTH_ERROR, "x"); + assertFalse(pool.contains(null)); + assertFalse(pool.contains("")); + assertTrue(pool.snapshot().isEmpty(), + "null/empty inputs must not pollute the snapshot"); + } + + @Test + @DisplayName("Concurrent add + remove + contains is thread-safe") + void concurrentAccess() throws Exception { + int threads = 16; + int opsPerThread = 5_000; + ExecutorService pool2 = Executors.newFixedThreadPool(threads); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(threads); + + for (int t = 0; t < threads; t++) { + int worker = t; + pool2.submit(() -> { + try { + start.await(); + for (int i = 0; i < opsPerThread; i++) { + String id = "p" + (worker * 10 + (i % 10)); // shared id space + if (i % 3 == 0) pool.add(id); + else if (i % 3 == 1) pool.remove(id, RemovalSource.AUTH_ERROR, "race"); + else pool.contains(id); + } + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } finally { + done.countDown(); + } + }); + } + start.countDown(); + assertTrue(done.await(30, TimeUnit.SECONDS), "concurrent workload must complete in 30s"); + pool2.shutdown(); + + // Internal state must remain consistent — each id is either in members OR has a removal reason + // (or both — the union is also fine), and snapshot doesn't NPE. + var snap = pool.snapshot(); + assertNotNull(snap); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java new file mode 100644 index 00000000..8a314123 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java @@ -0,0 +1,261 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.repository.ModelProviderMapper; +import vip.mate.llm.service.ModelProviderService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Verifies the startup-probe orchestration: + *
    + *
  • Healthy probes → provider added to pool.
    • + *
  • Failed probes → provider removed with INIT_PROBE source.
    • + *
  • Slow probe → fail-open (in-pool) so chat isn't gated by a stalled probe.
    • + *
  • Missing strategy → fail-open (in-pool).
    • + *
  • {@code probeOne} updates pool state on demand.
    • + *
  • Duplicate strategies for the same protocol fail-fast at construction.
    • + *
+ * + *

Strategies are real test-double instances (not Mockito mocks) so we can + * inject latency or throw cheaply; the mapper / service collaborators are + * stock Mockito mocks because they're MyBatis-Plus / Spring beans.

+ */ +class ProviderInitProbeTest { + + private ModelProviderMapper mapper; + private ModelProviderService providerService; + private AvailableProviderPool pool; + + @BeforeEach + void setUp() { + mapper = mock(ModelProviderMapper.class); + providerService = mock(ModelProviderService.class); + pool = new AvailableProviderPool(); + } + + @Test + @DisplayName("All strategies pass: every configured provider lands in the pool") + void allHealthy() { + ModelProviderEntity openai = provider("openai", ModelProtocol.OPENAI_COMPATIBLE); + ModelProviderEntity anthropic = provider("anthropic", ModelProtocol.ANTHROPIC_MESSAGES); + ModelProviderEntity dashscope = provider("dashscope", ModelProtocol.DASHSCOPE_NATIVE); + configure(List.of(openai, anthropic, dashscope), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(10)), + stub(ModelProtocol.ANTHROPIC_MESSAGES, p -> ProbeResult.ok(20)), + stub(ModelProtocol.DASHSCOPE_NATIVE, p -> ProbeResult.ok(30)))); + probe.probeAllConfigured(); + + assertTrue(pool.contains("openai")); + assertTrue(pool.contains("anthropic")); + assertTrue(pool.contains("dashscope")); + } + + @Test + @DisplayName("Failed probe removes provider with INIT_PROBE source and the error message") + void failurePathRemovesWithReason() { + configure(List.of(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.fail(50, "401 Unauthorized")))); + probe.probeAllConfigured(); + + assertFalse(pool.contains("openai")); + var reason = pool.snapshot().get("openai"); + assertNotNull(reason); + assertEquals(AvailableProviderPool.RemovalSource.INIT_PROBE, reason.source()); + assertTrue(reason.message().contains("401 Unauthorized"), + "removal message must surface the underlying probe error"); + } + + @Test + @DisplayName("Mixed batch: pass + fail in one run leaves correct pool state") + void mixedBatch() { + configure(List.of( + provider("openai", ModelProtocol.OPENAI_COMPATIBLE), + provider("anthropic", ModelProtocol.ANTHROPIC_MESSAGES)), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(10)), + stub(ModelProtocol.ANTHROPIC_MESSAGES, p -> ProbeResult.fail(15, "auth")))); + probe.probeAllConfigured(); + + assertTrue(pool.contains("openai")); + assertFalse(pool.contains("anthropic")); + assertEquals(AvailableProviderPool.RemovalSource.INIT_PROBE, + pool.snapshot().get("anthropic").source()); + } + + @Test + @DisplayName("Strategy throwing is treated as a probe failure (no startup crash)") + void strategyThrowsHandledAsFailure() { + configure(List.of(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> { + throw new RuntimeException("network down"); + }))); + probe.probeAllConfigured(); + + assertFalse(pool.contains("openai"), + "a throwing strategy must not leave the provider falsely in-pool"); + assertNotNull(pool.snapshot().get("openai")); + } + + @Test + @DisplayName("No strategy registered for protocol: fail-open (provider stays in pool)") + void missingStrategyFailsOpen() { + configure(List.of(provider("gemini", ModelProtocol.GEMINI_NATIVE)), id -> true); + + // Empty strategy list — no GEMINI_NATIVE handler. + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of()); + probe.probeAllConfigured(); + + assertTrue(pool.contains("gemini"), + "without a probe strategy we must default to in-pool, not block chat"); + } + + @Test + @DisplayName("No configured providers: probe is a no-op, pool stays empty") + void emptyConfigurationIsNoOp() { + configure(List.of(), id -> false); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0)))); + probe.probeAllConfigured(); + + assertTrue(pool.snapshot().isEmpty()); + } + + @Test + @DisplayName("Unconfigured providers are skipped (not probed and not added)") + void unconfiguredSkipped() { + ModelProviderEntity openai = provider("openai", ModelProtocol.OPENAI_COMPATIBLE); + ModelProviderEntity anthropic = provider("anthropic", ModelProtocol.ANTHROPIC_MESSAGES); + // mapper returns both, but only openai is "configured" + when(mapper.selectList(any())).thenReturn(List.of(openai, anthropic)); + when(providerService.isProviderConfigured("openai")).thenReturn(true); + when(providerService.isProviderConfigured("anthropic")).thenReturn(false); + + AtomicInteger anthropicCalls = new AtomicInteger(); + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0)), + stub(ModelProtocol.ANTHROPIC_MESSAGES, p -> { + anthropicCalls.incrementAndGet(); + return ProbeResult.ok(0); + }))); + probe.probeAllConfigured(); + + assertTrue(pool.contains("openai")); + assertFalse(pool.contains("anthropic")); + assertEquals(0, anthropicCalls.get(), + "unconfigured providers must not even be probed"); + } + + @Test + @DisplayName("probeOne(unknown) returns failure and does not pollute pool") + void probeOneUnknownProvider() { + when(mapper.selectById(anyString())).thenReturn(null); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of()); + ProbeResult r = probe.probeOne("ghost"); + + assertFalse(r.success()); + assertTrue(pool.snapshot().isEmpty()); + } + + @Test + @DisplayName("probeOne(unconfigured) HARD-removes from pool") + void probeOneUnconfigured() { + when(mapper.selectById("openai")).thenReturn(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)); + when(providerService.isProviderConfigured("openai")).thenReturn(false); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of()); + ProbeResult r = probe.probeOne("openai"); + + assertFalse(r.success()); + assertFalse(pool.contains("openai")); + assertEquals(AvailableProviderPool.RemovalSource.INIT_PROBE, + pool.snapshot().get("openai").source()); + } + + @Test + @DisplayName("probeOne(healthy) re-adds previously-removed provider to pool") + void probeOneRecoversRemovedProvider() { + when(mapper.selectById("openai")).thenReturn(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)); + when(providerService.isProviderConfigured("openai")).thenReturn(true); + + // Pre-remove openai to simulate a HARD-error eviction. + pool.remove("openai", AvailableProviderPool.RemovalSource.AUTH_ERROR, "401"); + assertFalse(pool.contains("openai")); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(5)))); + ProbeResult r = probe.probeOne("openai"); + + assertTrue(r.success()); + assertTrue(pool.contains("openai"), + "a successful reprobe must rehabilitate a previously removed provider"); + } + + @Test + @DisplayName("Duplicate strategy for same protocol fails-fast at construction") + void duplicateStrategyRejected() { + IllegalStateException ex = assertThrows(IllegalStateException.class, () -> + new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0)), + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0))))); + assertTrue(ex.getMessage().contains("OPENAI_COMPATIBLE")); + } + + // ============================================================ + // Helpers + // ============================================================ + + private static ModelProviderEntity provider(String id, ModelProtocol protocol) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setChatModel(protocol.getChatModelClass()); + p.setApiKey("sk-test"); + p.setBaseUrl("https://example.com"); + return p; + } + + /** Wires the mapper and service so {@code listConfiguredProviders()} returns the given list, + * filtered through {@code configuredPredicate}. */ + private void configure(List all, Function configuredPredicate) { + when(mapper.selectList(any())).thenReturn(all); + Map map = new HashMap<>(); + for (ModelProviderEntity p : all) { + map.put(p.getProviderId(), configuredPredicate.apply(p.getProviderId())); + } + when(providerService.isProviderConfigured(anyString())) + .thenAnswer(inv -> map.getOrDefault(inv.getArgument(0), false)); + } + + /** Lambda-driven fake of {@link ProviderProbeStrategy}. */ + private static ProviderProbeStrategy stub(ModelProtocol protocol, + Function body) { + return new ProviderProbeStrategy() { + @Override public ModelProtocol supportedProtocol() { return protocol; } + @Override public ProbeResult probe(ModelProviderEntity provider) { return body.apply(provider); } + }; + } +} diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 763dda17..3c809588 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -73,6 +73,11 @@ export default { description: 'Your login session has expired and you need to sign in again.', action: 'You will be redirected to the login page.', }, + provider_auth_error: { + title: 'Model authentication failed', + description: 'The current model\'s API key is invalid or has expired. This is unrelated to your login session.', + action: 'Open Settings → Models to check and update the API key.', + }, forbidden: { title: 'Access denied', description: 'You don\'t have permission to perform this action.', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 8a99c3f1..51f02dc9 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -73,6 +73,11 @@ export default { description: '您的登录凭证已失效，需要重新登录。', action: '页面将自动跳转到登录页。', }, + provider_auth_error: { + title: '模型认证失败', + description: '当前模型的 API Key 无效或已失效，与你的登录状态无关。', + action: '请到「设置 → 模型」中检查并更新该模型的 API Key。', + }, forbidden: { title: '没有权限', description: '您没有执行此操作的权限。', diff --git a/mateclaw-ui/src/types/chatError.ts b/mateclaw-ui/src/types/chatError.ts index 3dbcba4c..9b08b504 100644 --- a/mateclaw-ui/src/types/chatError.ts +++ b/mateclaw-ui/src/types/chatError.ts @@ -5,7 +5,8 @@ export type ChatErrorCategory = | 'rate_limit' // 429 - | 'auth_expired' // 401 + | 'auth_expired' // user-side session expired (our backend 401) — triggers /login redirect + | 'provider_auth_error' // LLM provider 401 (e.g. invalid Kimi/OpenAI API key) — unrelated to user login | 'forbidden' // 403 | 'bad_request' // 400 | 'server_error' // 500 @@ -62,7 +63,11 @@ const BACKEND_ERROR_TYPE_MAP: Record = [ { pattern: /频率|rate.?limit|too.?many|quota|429/i, category: 'rate_limit', retryable: true }, - { pattern: /认证|auth|unauthorized|401/i, category: 'auth_expired', retryable: false }, + // Narrow: only fire on explicit signals that the user's own session is gone. + { pattern: /HTTP 401|登录已过期|session.?expired|凭证.*失效/i, category: 'auth_expired', retryable: false }, + // Broad: any other 401-ish wording is treated as a model-side auth failure. + { pattern: /unauthorized|401|invalid.?api.?key|api.?key.*expired|认证失败/i, category: 'provider_auth_error', retryable: false }, { pattern: /权限|forbidden|403/i, category: 'forbidden', retryable: false }, { pattern: /过长|too.?long|context.?length|prompt/i, category: 'bad_request', retryable: false }, { pattern: /超时|timeout/i, category: 'timeout', retryable: true },