fix(llm): validate completed model probe responses (#658)

Contributed by RobinZhiBin <robinzhi@gmail.com> (@RobinZhiBin). Validate completion status, nonempty content, and complete JSON input; include both original commits. Verified by related backend regression tests.

mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java

@@ -1,5 +1,7 @@
 package vip.mate.llm.service;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.RequiredArgsConstructor;
@@ -683,7 +685,9 @@ public class ModelDiscoveryService {
      * Build the smoke-test request body for the OpenAI-compatible test-prompt path.
      * Probe limits follow the same ModelFamily constraints as runtime chat.
      * Reasoning models need a completion budget that includes reasoning tokens;
-     * the standard ten-token probe is insufficient for those models. Any
+     * use 4096 for recognized reasoning families and 512 otherwise, since even
+     * an unrecognized family may spend tokens reasoning before producing text.
+     * These are upper bounds, not a guarantee that every model can finish. Any
      * unrecognized top-level {@code generateKwargs} key (e.g. vLLM's
      * {@code chat_template_kwargs} used to disable Qwen thinking mode) is forwarded
      * verbatim, same as the runtime chat path in
@@ -701,7 +705,7 @@ public class ModelDiscoveryService {
             // Let OpenAI/Azure use model defaults: reasoning deployments may
             // reject explicit sampling options even in a connectivity probe.
         } else {
-            requestBody.put("max_tokens", 10);
+            requestBody.put("max_tokens", family.isThinking() ? 4096 : 512);
             Object probeTemperature;
             if (family.fixedTemperatureOne()) {
                 probeTemperature = 1.0d;
@@ -900,13 +904,45 @@ public class ModelDiscoveryService {
         }
     }
 
+    /**
+     * A reachable endpoint is not proof that a chat probe completed. Require a
+     * final textual answer and the OpenAI-compatible completion marker. In
+     * particular, reasoning-only and token-limited responses must fail visibly.
+     * Never turn malformed JSON into canned success or expose the raw payload
+     * (which may contain provider-internal details) in a validation error.
+     */
     private String extractOpenAiChatContent(String body) {
-        try {
+        if (!StringUtils.hasText(body)) {
-            JsonNode root = objectMapper.readTree(body);
+            throw new IllegalStateException("模型探针返回空响应，未取得完整回复");
-            return root.path("choices").path(0).path("message").path("content").asText("连接正常");
-        } catch (Exception e) {
-            return "连接正常（响应解析异常）";
         }
+        JsonNode root;
+        try {
+            root = objectMapper.reader()
+                    .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
+                    .readTree(body);
+        } catch (JsonProcessingException e) {
+            throw new IllegalStateException("模型探针响应不是有效 JSON，未取得完整回复");
+        }
+        if (root == null || !root.isObject() || root.hasNonNull("error")) {
+            throw new IllegalStateException("模型探针返回错误或无效响应，未取得完整回复");
+        }
+        JsonNode choices = root.path("choices");
+        if (!choices.isArray() || choices.isEmpty()) {
+            throw new IllegalStateException("模型探针响应缺少 choices，未取得完整回复");
+        }
+        JsonNode choice = choices.get(0);
+        String finishReason = choice.path("finish_reason").asText("");
+        if ("length".equals(finishReason)) {
+            throw new IllegalStateException("模型探针达到输出令牌上限，未取得完整回复（不代表模型不可用）");
+        }
+        if (!"stop".equals(finishReason)) {
+            throw new IllegalStateException("模型探针未正常完成（finish_reason 必须为 stop），请检查供应商响应");
+        }
+        JsonNode content = choice.path("message").path("content");
+        if (!content.isTextual() || !StringUtils.hasText(content.textValue())) {
+            throw new IllegalStateException("模型探针未返回有效正文（仅推理内容不算完整回复）");
+        }
+        return content.textValue();
     }
 
     private String extractGeminiContent(String body) {

mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceProbeValidationTest.java

@@ -0,0 +1,147 @@
+package vip.mate.llm.service;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.sun.net.httpserver.HttpServer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import vip.mate.llm.model.ModelConfigEntity;
+import vip.mate.llm.model.ModelProviderEntity;
+import vip.mate.llm.oauth.OpenAIOAuthService;
+
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
+/** Real loopback HTTP exercises the public test and discovery paths without provider credentials. */
+class ModelDiscoveryServiceProbeValidationTest {
+    private final ObjectMapper mapper = new ObjectMapper();
+    private final AtomicReference<String> response = new AtomicReference<>();
+    private final AtomicReference<JsonNode> request = new AtomicReference<>();
+    private HttpServer server;
+    private ModelDiscoveryService service;
+    private ModelProviderEntity provider;
+    private ModelConfigService configService;
+
+    @BeforeEach
+    void setUp() throws Exception {
+        server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+        server.createContext("/v1/chat/completions", exchange -> {
+            request.set(mapper.readTree(exchange.getRequestBody()));
+            byte[] bytes = response.get().getBytes(StandardCharsets.UTF_8);
+            exchange.getResponseHeaders().set("Content-Type", "application/json");
+            exchange.sendResponseHeaders(200, bytes.length);
+            try (var out = exchange.getResponseBody()) { out.write(bytes); }
+        });
+        server.createContext("/v1/models", exchange -> {
+            byte[] bytes = "{\"data\":[{\"id\":\"probe-model\"}]}".getBytes(StandardCharsets.UTF_8);
+            exchange.getResponseHeaders().set("Content-Type", "application/json");
+            exchange.sendResponseHeaders(200, bytes.length);
+            try (var out = exchange.getResponseBody()) { out.write(bytes); }
+        });
+        server.start();
+        provider = new ModelProviderEntity();
+        provider.setProviderId("loopback");
+        provider.setChatModel("OpenAIChatModel");
+        provider.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/v1");
+        provider.setSupportModelDiscovery(false);
+        var providers = mock(ModelProviderService.class);
+        when(providers.getProviderConfig("loopback")).thenReturn(provider);
+        when(providers.readProviderGenerateKwargs(provider)).thenReturn(Map.of());
+        configService = mock(ModelConfigService.class);
+        when(configService.listModelsByProvider("loopback")).thenReturn(List.of());
+        service = new ModelDiscoveryService(providers, configService, mapper, mock(OpenAIOAuthService.class));
+    }
+
+    @AfterEach
+    void tearDown() { if (server != null) server.stop(0); }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "not-json", "", "null", "[]", "{}", "{\"choices\":[]}",
+            "{\"error\":{\"message\":\"private-provider-detail\"}}",
+            "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{}}]}",
+            "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":null}}]}",
+            "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":\"   \"}}]}",
+            "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":42}}]}",
+            "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"reasoning_content\":\"private-provider-detail\",\"content\":\"\"}}]}",
+            "{\"choices\":[{\"message\":{\"content\":\"partial\"}}]}"
+    })
+    void rejectsMalformedOrEmptySuccessfulHttpResponses(String body) {
+        response.set(body);
+        var result = service.testModel("loopback", "probe-model");
+        assertFalse(result.isSuccess(), body);
+        assertNotNull(result.getErrorMessage());
+        assertFalse(result.getErrorMessage().contains("private-provider-detail"));
+        assertNull(result.getMessage(), "Never manufacture a successful reply");
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"length", "content_filter", "tool_calls", "function_call", "unknown"})
+    void rejectsIncompleteRepliesEvenWithText(String reason) {
+        response.set(completion("partial", reason));
+        var result = service.testModel("loopback", "probe-model");
+        assertFalse(result.isSuccess(), reason);
+        assertNotNull(result.getErrorMessage());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {"NOT_JSON", "{\"error\":\"failed\"}"})
+    void rejectsTrailingContentAfterAValidCompletion(String suffix) {
+        response.set(completion("连接正常", "stop") + suffix);
+        assertFalse(service.testModel("loopback", "probe-model").isSuccess());
+    }
+
+    @Test
+    void acceptsCompletedTextAndSendsBoundedBudget() {
+        response.set(completion("连接正常", "stop"));
+        var result = service.testModel("loopback", "probe-model");
+        assertTrue(result.isSuccess(), result.getErrorMessage());
+        assertEquals("连接正常", result.getMessage());
+        assertEquals(512, request.get().path("max_tokens").asInt());
+        assertFalse(request.get().has("max_completion_tokens"));
+    }
+
+    @Test
+    void rejectsErrorEnvelopeEvenWhenItContainsAChoice() {
+        response.set("{\"error\":{\"message\":\"private-provider-detail\"}," + completion("ok", "stop").substring(1));
+        var result = service.testModel("loopback", "probe-model");
+        assertFalse(result.isSuccess());
+        assertFalse(result.getErrorMessage().contains("private-provider-detail"));
+    }
+
+    @Test
+    void connectionFallbackAlsoRejectsReasoningOnly() {
+        var model = new ModelConfigEntity();
+        model.setModelName("probe-model");
+        when(configService.listModelsByProvider("loopback")).thenReturn(List.of(model));
+        response.set(completion("", "length"));
+        assertFalse(service.testConnection("loopback").isSuccess());
+    }
+
+    @Test
+    void discoveryRetainsFailedProbeButDoesNotRecommendIt() {
+        provider.setSupportModelDiscovery(true);
+        response.set(completion("partial", "length"));
+        var result = service.discoverModels("loopback");
+        assertEquals(1, result.getTotalDiscovered());
+        assertEquals(Boolean.FALSE, result.getDiscoveredModels().getFirst().getProbeOk());
+        assertNotNull(result.getDiscoveredModels().getFirst().getProbeError());
+        assertTrue(result.getNewModels().isEmpty());
+    }
+
+    private String completion(String content, String reason) {
+        try {
+            return mapper.writeValueAsString(Map.of("choices", List.of(Map.of(
+                    "finish_reason", reason, "message", Map.of("role", "assistant", "content", content)))));
+        } catch (Exception e) { throw new AssertionError(e); }
+    }
+}

mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceTestPromptTest.java

@@ -37,7 +37,7 @@ class ModelDiscoveryServiceTestPromptTest {
         assertEquals(chatTemplateKwargs, requestBody.get("chat_template_kwargs"),
                 "the nested map must be forwarded verbatim, not flattened or re-wrapped");
         assertEquals("qwen3-32b", requestBody.get("model"));
-        assertEquals(10, requestBody.get("max_tokens"));
+        assertEquals(512, requestBody.get("max_tokens"));
         assertEquals(0, requestBody.get("temperature"));
     }
 
@@ -53,8 +53,8 @@ class ModelDiscoveryServiceTestPromptTest {
 
         assertEquals(0, requestBody.get("temperature"),
                 "the probe's fixed temperature=0 must win over a reserved generateKwargs key");
-        assertEquals(10, requestBody.get("max_tokens"),
+        assertEquals(512, requestBody.get("max_tokens"),
-                "the probe's fixed max_tokens=10 must win over a reserved generateKwargs key");
+                "the probe's fixed max_tokens=512 must win over a reserved generateKwargs key");
         assertFalse(requestBody.containsKey("maxTokens"),
                 "reserved keys (even in their original casing) must not leak into the body verbatim");
         assertTrue(requestBody.containsKey("chat_template_kwargs"),
@@ -94,7 +94,7 @@ class ModelDiscoveryServiceTestPromptTest {
 
         assertEquals(Set.of("model", "messages", "max_tokens", "temperature"), requestBody.keySet());
         assertEquals("gpt-4-turbo", requestBody.get("model"));
-        assertEquals(10, requestBody.get("max_tokens"));
+        assertEquals(512, requestBody.get("max_tokens"));
         assertEquals(0, requestBody.get("temperature"));
 
         Map<String, Object> requestBodyFromNull = ModelDiscoveryService.buildTestPromptRequestBody("gpt-4-turbo", null);
@@ -108,6 +108,7 @@ class ModelDiscoveryServiceTestPromptTest {
                 "kimi-for-coding", Map.of("temperature", 0.2));
 
         assertEquals(1.0d, requestBody.get("temperature"));
+        assertEquals(4096, requestBody.get("max_tokens"));
     }
     @Test
     @DisplayName("OpenAI reasoning probes use completion-token budgets and omit unsupported sampling parameters (#640)")
@@ -126,7 +127,7 @@ class ModelDiscoveryServiceTestPromptTest {
     @DisplayName("GPT-4o probes retain their supported standard parameters")
     void gpt4oProbeRemainsStandard() {
         Map<String, Object> body = ModelDiscoveryService.buildTestPromptRequestBody("gpt-4o", Map.of());
-        assertEquals(10, body.get("max_tokens"));
+        assertEquals(512, body.get("max_tokens"));
         assertEquals(0, body.get("temperature"));
         assertFalse(body.containsKey("max_completion_tokens"));
     }