diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java index 17c431b8..ec7c7ab8 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java @@ -1,5 +1,7 @@ package vip.mate.llm.service; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; @@ -683,7 +685,9 @@ public class ModelDiscoveryService { * Build the smoke-test request body for the OpenAI-compatible test-prompt path. * Probe limits follow the same ModelFamily constraints as runtime chat. * Reasoning models need a completion budget that includes reasoning tokens; - * the standard ten-token probe is insufficient for those models. Any + * use 4096 for recognized reasoning families and 512 otherwise, since even + * an unrecognized family may spend tokens reasoning before producing text. + * These are upper bounds, not a guarantee that every model can finish. Any * unrecognized top-level {@code generateKwargs} key (e.g. vLLM's * {@code chat_template_kwargs} used to disable Qwen thinking mode) is forwarded * verbatim, same as the runtime chat path in @@ -701,7 +705,7 @@ public class ModelDiscoveryService { // Let OpenAI/Azure use model defaults: reasoning deployments may // reject explicit sampling options even in a connectivity probe. } else { - requestBody.put("max_tokens", 10); + requestBody.put("max_tokens", family.isThinking() ? 4096 : 512); Object probeTemperature; if (family.fixedTemperatureOne()) { probeTemperature = 1.0d; @@ -900,13 +904,45 @@ public class ModelDiscoveryService { } } + /** + * A reachable endpoint is not proof that a chat probe completed. Require a + * final textual answer and the OpenAI-compatible completion marker. In + * particular, reasoning-only and token-limited responses must fail visibly. + * Never turn malformed JSON into canned success or expose the raw payload + * (which may contain provider-internal details) in a validation error. + */ private String extractOpenAiChatContent(String body) { - try { - JsonNode root = objectMapper.readTree(body); - return root.path("choices").path(0).path("message").path("content").asText("连接正常"); - } catch (Exception e) { - return "连接正常（响应解析异常）"; + if (!StringUtils.hasText(body)) { + throw new IllegalStateException("模型探针返回空响应，未取得完整回复"); } + JsonNode root; + try { + root = objectMapper.reader() + .with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .readTree(body); + } catch (JsonProcessingException e) { + throw new IllegalStateException("模型探针响应不是有效 JSON，未取得完整回复"); + } + if (root == null || !root.isObject() || root.hasNonNull("error")) { + throw new IllegalStateException("模型探针返回错误或无效响应，未取得完整回复"); + } + JsonNode choices = root.path("choices"); + if (!choices.isArray() || choices.isEmpty()) { + throw new IllegalStateException("模型探针响应缺少 choices，未取得完整回复"); + } + JsonNode choice = choices.get(0); + String finishReason = choice.path("finish_reason").asText(""); + if ("length".equals(finishReason)) { + throw new IllegalStateException("模型探针达到输出令牌上限，未取得完整回复（不代表模型不可用）"); + } + if (!"stop".equals(finishReason)) { + throw new IllegalStateException("模型探针未正常完成（finish_reason 必须为 stop），请检查供应商响应"); + } + JsonNode content = choice.path("message").path("content"); + if (!content.isTextual() || !StringUtils.hasText(content.textValue())) { + throw new IllegalStateException("模型探针未返回有效正文（仅推理内容不算完整回复）"); + } + return content.textValue(); } private String extractGeminiContent(String body) { diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceProbeValidationTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceProbeValidationTest.java new file mode 100644 index 00000000..5eca43a8 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceProbeValidationTest.java @@ -0,0 +1,147 @@ +package vip.mate.llm.service; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.oauth.OpenAIOAuthService; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** Real loopback HTTP exercises the public test and discovery paths without provider credentials. */ +class ModelDiscoveryServiceProbeValidationTest { + private final ObjectMapper mapper = new ObjectMapper(); + private final AtomicReference response = new AtomicReference<>(); + private final AtomicReference request = new AtomicReference<>(); + private HttpServer server; + private ModelDiscoveryService service; + private ModelProviderEntity provider; + private ModelConfigService configService; + + @BeforeEach + void setUp() throws Exception { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/chat/completions", exchange -> { + request.set(mapper.readTree(exchange.getRequestBody())); + byte[] bytes = response.get().getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, bytes.length); + try (var out = exchange.getResponseBody()) { out.write(bytes); } + }); + server.createContext("/v1/models", exchange -> { + byte[] bytes = "{\"data\":[{\"id\":\"probe-model\"}]}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, bytes.length); + try (var out = exchange.getResponseBody()) { out.write(bytes); } + }); + server.start(); + provider = new ModelProviderEntity(); + provider.setProviderId("loopback"); + provider.setChatModel("OpenAIChatModel"); + provider.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/v1"); + provider.setSupportModelDiscovery(false); + var providers = mock(ModelProviderService.class); + when(providers.getProviderConfig("loopback")).thenReturn(provider); + when(providers.readProviderGenerateKwargs(provider)).thenReturn(Map.of()); + configService = mock(ModelConfigService.class); + when(configService.listModelsByProvider("loopback")).thenReturn(List.of()); + service = new ModelDiscoveryService(providers, configService, mapper, mock(OpenAIOAuthService.class)); + } + + @AfterEach + void tearDown() { if (server != null) server.stop(0); } + + @ParameterizedTest + @ValueSource(strings = { + "not-json", "", "null", "[]", "{}", "{\"choices\":[]}", + "{\"error\":{\"message\":\"private-provider-detail\"}}", + "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{}}]}", + "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":null}}]}", + "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":\" \"}}]}", + "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":42}}]}", + "{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"reasoning_content\":\"private-provider-detail\",\"content\":\"\"}}]}", + "{\"choices\":[{\"message\":{\"content\":\"partial\"}}]}" + }) + void rejectsMalformedOrEmptySuccessfulHttpResponses(String body) { + response.set(body); + var result = service.testModel("loopback", "probe-model"); + assertFalse(result.isSuccess(), body); + assertNotNull(result.getErrorMessage()); + assertFalse(result.getErrorMessage().contains("private-provider-detail")); + assertNull(result.getMessage(), "Never manufacture a successful reply"); + } + + @ParameterizedTest + @ValueSource(strings = {"length", "content_filter", "tool_calls", "function_call", "unknown"}) + void rejectsIncompleteRepliesEvenWithText(String reason) { + response.set(completion("partial", reason)); + var result = service.testModel("loopback", "probe-model"); + assertFalse(result.isSuccess(), reason); + assertNotNull(result.getErrorMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"NOT_JSON", "{\"error\":\"failed\"}"}) + void rejectsTrailingContentAfterAValidCompletion(String suffix) { + response.set(completion("连接正常", "stop") + suffix); + assertFalse(service.testModel("loopback", "probe-model").isSuccess()); + } + + @Test + void acceptsCompletedTextAndSendsBoundedBudget() { + response.set(completion("连接正常", "stop")); + var result = service.testModel("loopback", "probe-model"); + assertTrue(result.isSuccess(), result.getErrorMessage()); + assertEquals("连接正常", result.getMessage()); + assertEquals(512, request.get().path("max_tokens").asInt()); + assertFalse(request.get().has("max_completion_tokens")); + } + + @Test + void rejectsErrorEnvelopeEvenWhenItContainsAChoice() { + response.set("{\"error\":{\"message\":\"private-provider-detail\"}," + completion("ok", "stop").substring(1)); + var result = service.testModel("loopback", "probe-model"); + assertFalse(result.isSuccess()); + assertFalse(result.getErrorMessage().contains("private-provider-detail")); + } + + @Test + void connectionFallbackAlsoRejectsReasoningOnly() { + var model = new ModelConfigEntity(); + model.setModelName("probe-model"); + when(configService.listModelsByProvider("loopback")).thenReturn(List.of(model)); + response.set(completion("", "length")); + assertFalse(service.testConnection("loopback").isSuccess()); + } + + @Test + void discoveryRetainsFailedProbeButDoesNotRecommendIt() { + provider.setSupportModelDiscovery(true); + response.set(completion("partial", "length")); + var result = service.discoverModels("loopback"); + assertEquals(1, result.getTotalDiscovered()); + assertEquals(Boolean.FALSE, result.getDiscoveredModels().getFirst().getProbeOk()); + assertNotNull(result.getDiscoveredModels().getFirst().getProbeError()); + assertTrue(result.getNewModels().isEmpty()); + } + + private String completion(String content, String reason) { + try { + return mapper.writeValueAsString(Map.of("choices", List.of(Map.of( + "finish_reason", reason, "message", Map.of("role", "assistant", "content", content))))); + } catch (Exception e) { throw new AssertionError(e); } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceTestPromptTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceTestPromptTest.java index d3e22c59..88b81535 100644 --- a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceTestPromptTest.java +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceTestPromptTest.java @@ -37,7 +37,7 @@ class ModelDiscoveryServiceTestPromptTest { assertEquals(chatTemplateKwargs, requestBody.get("chat_template_kwargs"), "the nested map must be forwarded verbatim, not flattened or re-wrapped"); assertEquals("qwen3-32b", requestBody.get("model")); - assertEquals(10, requestBody.get("max_tokens")); + assertEquals(512, requestBody.get("max_tokens")); assertEquals(0, requestBody.get("temperature")); } @@ -53,8 +53,8 @@ class ModelDiscoveryServiceTestPromptTest { assertEquals(0, requestBody.get("temperature"), "the probe's fixed temperature=0 must win over a reserved generateKwargs key"); - assertEquals(10, requestBody.get("max_tokens"), - "the probe's fixed max_tokens=10 must win over a reserved generateKwargs key"); + assertEquals(512, requestBody.get("max_tokens"), + "the probe's fixed max_tokens=512 must win over a reserved generateKwargs key"); assertFalse(requestBody.containsKey("maxTokens"), "reserved keys (even in their original casing) must not leak into the body verbatim"); assertTrue(requestBody.containsKey("chat_template_kwargs"), @@ -94,7 +94,7 @@ class ModelDiscoveryServiceTestPromptTest { assertEquals(Set.of("model", "messages", "max_tokens", "temperature"), requestBody.keySet()); assertEquals("gpt-4-turbo", requestBody.get("model")); - assertEquals(10, requestBody.get("max_tokens")); + assertEquals(512, requestBody.get("max_tokens")); assertEquals(0, requestBody.get("temperature")); Map requestBodyFromNull = ModelDiscoveryService.buildTestPromptRequestBody("gpt-4-turbo", null); @@ -108,6 +108,7 @@ class ModelDiscoveryServiceTestPromptTest { "kimi-for-coding", Map.of("temperature", 0.2)); assertEquals(1.0d, requestBody.get("temperature")); + assertEquals(4096, requestBody.get("max_tokens")); } @Test @DisplayName("OpenAI reasoning probes use completion-token budgets and omit unsupported sampling parameters (#640)") @@ -126,7 +127,7 @@ class ModelDiscoveryServiceTestPromptTest { @DisplayName("GPT-4o probes retain their supported standard parameters") void gpt4oProbeRemainsStandard() { Map body = ModelDiscoveryService.buildTestPromptRequestBody("gpt-4o", Map.of()); - assertEquals(10, body.get("max_tokens")); + assertEquals(512, body.get("max_tokens")); assertEquals(0, body.get("temperature")); assertFalse(body.containsKey("max_completion_tokens")); }