From a369e8055d07bab23e95f30c4adfb7ad4e40e367 Mon Sep 17 00:00:00 2001 From: matevip Date: Tue, 14 Apr 2026 16:05:37 +0800 Subject: [PATCH] fix(wiki): bound LLM retry + http read timeout (RFC-012 M1 follow-up) --- .../vip/mate/agent/AgentGraphBuilder.java | 21 ++++++++++++++-- .../java/vip/mate/wiki/WikiProperties.java | 24 +++++++++++++++++++ .../wiki/service/WikiProcessingService.java | 23 ++++++++++++++---- .../src/main/resources/application.yml | 4 +++- 4 files changed, 64 insertions(+), 8 deletions(-) diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 3ba895ae..095d6852 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -33,7 +33,9 @@ import org.springframework.http.HttpHeaders; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.util.StringUtils; +import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.RestClient; +import java.time.Duration; import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientResponseException; import reactor.core.publisher.Flux; @@ -805,7 +807,8 @@ public class AgentGraphBuilder { Map kwargs = modelProviderService.readProviderGenerateKwargs(provider); MultiValueMap headers = buildOpenAiHeaders(kwargs); String completionsPath = resolveOpenAiCompletionsPath(baseUrl, kwargs); - RestClient.Builder restClientBuilder = restClientBuilderProvider.getIfAvailable(RestClient::builder); + RestClient.Builder restClientBuilder = applyHttpTimeouts( + restClientBuilderProvider.getIfAvailable(RestClient::builder)); WebClient.Builder webClientBuilder = webClientBuilderProvider.getIfAvailable(WebClient::builder); // Spring AI OpenAiApi 构造函数会先 set User-Agent 为 "spring-ai"，再 addAll 我们的 headers， @@ -923,7 +926,8 @@ public class AgentGraphBuilder { throw new MateClawException("err.agent.anthropic_key_invalid", "Anthropic API Key 未配置或无效: " + provider.getProviderId()); } String baseUrl = provider.getBaseUrl(); - RestClient.Builder restClientBuilder = restClientBuilderProvider.getIfAvailable(RestClient::builder); + RestClient.Builder restClientBuilder = applyHttpTimeouts( + restClientBuilderProvider.getIfAvailable(RestClient::builder)); WebClient.Builder webClientBuilder = webClientBuilderProvider.getIfAvailable(WebClient::builder); AnthropicApi.Builder builder = AnthropicApi.builder() @@ -1275,6 +1279,19 @@ public class AgentGraphBuilder { return headers; } + /** + * RFC-012 M1：给 LLM 调用走的 RestClient 显式配置超时，避免 socket 永久挂起等待。 + *

+ * connectTimeout=10s（任何 LLM 提供方都不该超过这个建立连接时间）； + * readTimeout=180s（覆盖 nginx 60s 网关超时 + 留足真实长响应余量；超时后由上层 retry 接管）。 + */ + private RestClient.Builder applyHttpTimeouts(RestClient.Builder builder) { + SimpleClientHttpRequestFactory rf = new SimpleClientHttpRequestFactory(); + rf.setConnectTimeout(Duration.ofSeconds(10)); + rf.setReadTimeout(Duration.ofSeconds(180)); + return builder.requestFactory(rf); + } + /** * 从 generateKwargs.headers 中提取需要强制覆盖的 headers。 * 用于通过 RestClient/WebClient 拦截器绕过 Spring AI OpenAiApi 的默认 User-Agent。 diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java index 7b23dd04..3ce58c20 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java @@ -57,4 +57,28 @@ public class WikiProperties { /** 扫描时跳过大于此大小的文件（字节），默认 50MB */ private long maxScanFileSize = 50 * 1024 * 1024; + + /** + * Wiki LLM 重试最大尝试次数（含首次）。 + *

+ * RFC-012 M1：旧实现无最大次数，遇到 nginx 504 这种"反复瞬时"错误会永远重试。 + * 设为 5 后单 chunk 最多走 5 轮，配合 llmMaxTotalDurationMs 共同保证有界停止。 + */ + private int llmMaxAttempts = 5; + + /** + * Wiki LLM 重试总耗时上限（毫秒），从首次调用开始计时。 + *

+ * RFC-012 M1：单 chunk LLM 调用 + 重试的硬封顶，超过即放弃，让该 chunk 进入 failed 计数。 + * 默认 4 分钟。 + */ + private long llmMaxTotalDurationMs = 240_000; + + /** + * 是否启用两阶段消化（路由 → 逐页 merge）。 + *

+ * RFC-012 M2：true 时单 chunk 的 LLM 输出量大幅缩减，避免 nginx 60s 网关超时。 + * 默认 false 保持向后兼容；M2 实现完成后切到 true。 + */ + private boolean useTwoPhaseDigest = false; } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index 41a0ccd0..f97f5b89 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -454,16 +454,22 @@ public class WikiProcessingService { * 立即终止（模型不可用）：401/403 认证失败、模型不存在、quota 用尽、非法 API key、 * InterruptedException（优雅关停）。 *

- * 使用指数退避（1s → 2s → 4s → ... → 封顶 60s），无最大尝试次数。 + * 使用指数退避（1s → 2s → 4s → ... → 封顶 60s）。 + *

+ * RFC-012 M1：加入 maxAttempts 与 maxTotalDurationMs 双重上限，避免 nginx 504 这种 + * 反复瞬时错误把单 chunk 卡到永远；buildChatModel 提到循环外，所有重试复用同一实例。 */ private String callLlmWithResilientRetry(Prompt prompt, String ctx) { long backoffMs = 1000; final long maxBackoffMs = 60_000; + final int maxAttempts = Math.max(1, properties.getLlmMaxAttempts()); + final long maxTotalDurationMs = Math.max(1_000L, properties.getLlmMaxTotalDurationMs()); + final long startNanos = System.nanoTime(); + final ChatModel chatModel = buildChatModel(); int attempt = 0; while (true) { attempt++; try { - ChatModel chatModel = buildChatModel(); ChatResponse response = chatModel.call(prompt); if (response == null || response.getResult() == null || response.getResult().getOutput() == null @@ -484,10 +490,17 @@ public class WikiProcessingService { ctx, attempt, t.getMessage()); throw new RuntimeException("LLM unavailable: " + t.getMessage(), t); } - log.warn("[Wiki] LLM transient failure for {} attempt={}, retrying in {}ms: {}", - ctx, attempt, backoffMs, t.getMessage()); + long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000L; + if (attempt >= maxAttempts || elapsedMs >= maxTotalDurationMs) { + log.error("[Wiki] LLM exhausted for {} after {} attempts in {}ms (limits: maxAttempts={}, maxTotalDurationMs={}): {}", + ctx, attempt, elapsedMs, maxAttempts, maxTotalDurationMs, t.getMessage()); + throw new RuntimeException("LLM exhausted: " + t.getMessage(), t); + } + long sleepMs = Math.min(backoffMs, Math.max(0L, maxTotalDurationMs - elapsedMs)); + log.warn("[Wiki] LLM transient failure for {} attempt={}/{} elapsed={}ms, retrying in {}ms: {}", + ctx, attempt, maxAttempts, elapsedMs, sleepMs, t.getMessage()); try { - Thread.sleep(backoffMs); + Thread.sleep(sleepMs); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new RuntimeException("LLM retry interrupted for " + ctx, ie); diff --git a/mateclaw-server/src/main/resources/application.yml b/mateclaw-server/src/main/resources/application.yml index 3faaf4b1..44f21709 100644 --- a/mateclaw-server/src/main/resources/application.yml +++ b/mateclaw-server/src/main/resources/application.yml @@ -59,8 +59,10 @@ spring: jdbc: initialize-schema: embedded # Spring AI Retry 配置：429/503/529 归为可重试（TransientAiException），启用指数退避 + # RFC-012 M1：max-attempts 从 5 调到 2，避免与 wiki 层 callLlmWithResilientRetry 的 5×N 嵌套放大； + # 真正的重试主控权交给业务层（wiki / agent），Spring AI 仅负责"一次性瞬时抖动"的快速重试 retry: - max-attempts: 5 + max-attempts: 2 on-http-codes: 429, 503, 529 backoff: initial-interval: 3000