mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-11-02 14:11:49 +08:00
fix(llm): validate completed model probe responses (#658)
Contributed by RobinZhiBin <robinzhi@gmail.com> (@RobinZhiBin). Validate completion status, nonempty content, and complete JSON input; include both original commits. Verified by related backend regression tests.
This commit is contained in:
RobinZhiBin
GitHub
parent
0394ba2803
commit
6365f0af8a
@ -1,5 +1,7 @@
|
||||
package vip.mate.llm.service;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -683,7 +685,9 @@ public class ModelDiscoveryService {
|
||||
* Build the smoke-test request body for the OpenAI-compatible test-prompt path.
|
||||
* Probe limits follow the same ModelFamily constraints as runtime chat.
|
||||
* Reasoning models need a completion budget that includes reasoning tokens;
|
||||
* the standard ten-token probe is insufficient for those models. Any
|
||||
* use 4096 for recognized reasoning families and 512 otherwise, since even
|
||||
* an unrecognized family may spend tokens reasoning before producing text.
|
||||
* These are upper bounds, not a guarantee that every model can finish. Any
|
||||
* unrecognized top-level {@code generateKwargs} key (e.g. vLLM's
|
||||
* {@code chat_template_kwargs} used to disable Qwen thinking mode) is forwarded
|
||||
* verbatim, same as the runtime chat path in
|
||||
@ -701,7 +705,7 @@ public class ModelDiscoveryService {
|
||||
// Let OpenAI/Azure use model defaults: reasoning deployments may
|
||||
// reject explicit sampling options even in a connectivity probe.
|
||||
} else {
|
||||
requestBody.put("max_tokens", 10);
|
||||
requestBody.put("max_tokens", family.isThinking() ? 4096 : 512);
|
||||
Object probeTemperature;
|
||||
if (family.fixedTemperatureOne()) {
|
||||
probeTemperature = 1.0d;
|
||||
@ -900,13 +904,45 @@ public class ModelDiscoveryService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A reachable endpoint is not proof that a chat probe completed. Require a
|
||||
* final textual answer and the OpenAI-compatible completion marker. In
|
||||
* particular, reasoning-only and token-limited responses must fail visibly.
|
||||
* Never turn malformed JSON into canned success or expose the raw payload
|
||||
* (which may contain provider-internal details) in a validation error.
|
||||
*/
|
||||
private String extractOpenAiChatContent(String body) {
|
||||
try {
|
||||
JsonNode root = objectMapper.readTree(body);
|
||||
return root.path("choices").path(0).path("message").path("content").asText("连接正常");
|
||||
} catch (Exception e) {
|
||||
return "连接正常（响应解析异常）";
|
||||
if (!StringUtils.hasText(body)) {
|
||||
throw new IllegalStateException("模型探针返回空响应，未取得完整回复");
|
||||
}
|
||||
JsonNode root;
|
||||
try {
|
||||
root = objectMapper.reader()
|
||||
.with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
||||
.readTree(body);
|
||||
} catch (JsonProcessingException e) {
|
||||
throw new IllegalStateException("模型探针响应不是有效 JSON，未取得完整回复");
|
||||
}
|
||||
if (root == null || !root.isObject() || root.hasNonNull("error")) {
|
||||
throw new IllegalStateException("模型探针返回错误或无效响应，未取得完整回复");
|
||||
}
|
||||
JsonNode choices = root.path("choices");
|
||||
if (!choices.isArray() || choices.isEmpty()) {
|
||||
throw new IllegalStateException("模型探针响应缺少 choices，未取得完整回复");
|
||||
}
|
||||
JsonNode choice = choices.get(0);
|
||||
String finishReason = choice.path("finish_reason").asText("");
|
||||
if ("length".equals(finishReason)) {
|
||||
throw new IllegalStateException("模型探针达到输出令牌上限，未取得完整回复（不代表模型不可用）");
|
||||
}
|
||||
if (!"stop".equals(finishReason)) {
|
||||
throw new IllegalStateException("模型探针未正常完成（finish_reason 必须为 stop），请检查供应商响应");
|
||||
}
|
||||
JsonNode content = choice.path("message").path("content");
|
||||
if (!content.isTextual() || !StringUtils.hasText(content.textValue())) {
|
||||
throw new IllegalStateException("模型探针未返回有效正文（仅推理内容不算完整回复）");
|
||||
}
|
||||
return content.textValue();
|
||||
}
|
||||
|
||||
private String extractGeminiContent(String body) {
|
||||
|
||||
@ -0,0 +1,147 @@
|
||||
package vip.mate.llm.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.model.ModelProviderEntity;
|
||||
import vip.mate.llm.oauth.OpenAIOAuthService;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
/** Real loopback HTTP exercises the public test and discovery paths without provider credentials. */
|
||||
class ModelDiscoveryServiceProbeValidationTest {
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
private final AtomicReference<String> response = new AtomicReference<>();
|
||||
private final AtomicReference<JsonNode> request = new AtomicReference<>();
|
||||
private HttpServer server;
|
||||
private ModelDiscoveryService service;
|
||||
private ModelProviderEntity provider;
|
||||
private ModelConfigService configService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/v1/chat/completions", exchange -> {
|
||||
request.set(mapper.readTree(exchange.getRequestBody()));
|
||||
byte[] bytes = response.get().getBytes(StandardCharsets.UTF_8);
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
exchange.sendResponseHeaders(200, bytes.length);
|
||||
try (var out = exchange.getResponseBody()) { out.write(bytes); }
|
||||
});
|
||||
server.createContext("/v1/models", exchange -> {
|
||||
byte[] bytes = "{\"data\":[{\"id\":\"probe-model\"}]}".getBytes(StandardCharsets.UTF_8);
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
exchange.sendResponseHeaders(200, bytes.length);
|
||||
try (var out = exchange.getResponseBody()) { out.write(bytes); }
|
||||
});
|
||||
server.start();
|
||||
provider = new ModelProviderEntity();
|
||||
provider.setProviderId("loopback");
|
||||
provider.setChatModel("OpenAIChatModel");
|
||||
provider.setBaseUrl("http://127.0.0.1:" + server.getAddress().getPort() + "/v1");
|
||||
provider.setSupportModelDiscovery(false);
|
||||
var providers = mock(ModelProviderService.class);
|
||||
when(providers.getProviderConfig("loopback")).thenReturn(provider);
|
||||
when(providers.readProviderGenerateKwargs(provider)).thenReturn(Map.of());
|
||||
configService = mock(ModelConfigService.class);
|
||||
when(configService.listModelsByProvider("loopback")).thenReturn(List.of());
|
||||
service = new ModelDiscoveryService(providers, configService, mapper, mock(OpenAIOAuthService.class));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() { if (server != null) server.stop(0); }
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {
|
||||
"not-json", "", "null", "[]", "{}", "{\"choices\":[]}",
|
||||
"{\"error\":{\"message\":\"private-provider-detail\"}}",
|
||||
"{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{}}]}",
|
||||
"{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":null}}]}",
|
||||
"{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":\" \"}}]}",
|
||||
"{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"content\":42}}]}",
|
||||
"{\"choices\":[{\"finish_reason\":\"stop\",\"message\":{\"reasoning_content\":\"private-provider-detail\",\"content\":\"\"}}]}",
|
||||
"{\"choices\":[{\"message\":{\"content\":\"partial\"}}]}"
|
||||
})
|
||||
void rejectsMalformedOrEmptySuccessfulHttpResponses(String body) {
|
||||
response.set(body);
|
||||
var result = service.testModel("loopback", "probe-model");
|
||||
assertFalse(result.isSuccess(), body);
|
||||
assertNotNull(result.getErrorMessage());
|
||||
assertFalse(result.getErrorMessage().contains("private-provider-detail"));
|
||||
assertNull(result.getMessage(), "Never manufacture a successful reply");
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"length", "content_filter", "tool_calls", "function_call", "unknown"})
|
||||
void rejectsIncompleteRepliesEvenWithText(String reason) {
|
||||
response.set(completion("partial", reason));
|
||||
var result = service.testModel("loopback", "probe-model");
|
||||
assertFalse(result.isSuccess(), reason);
|
||||
assertNotNull(result.getErrorMessage());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"NOT_JSON", "{\"error\":\"failed\"}"})
|
||||
void rejectsTrailingContentAfterAValidCompletion(String suffix) {
|
||||
response.set(completion("连接正常", "stop") + suffix);
|
||||
assertFalse(service.testModel("loopback", "probe-model").isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
void acceptsCompletedTextAndSendsBoundedBudget() {
|
||||
response.set(completion("连接正常", "stop"));
|
||||
var result = service.testModel("loopback", "probe-model");
|
||||
assertTrue(result.isSuccess(), result.getErrorMessage());
|
||||
assertEquals("连接正常", result.getMessage());
|
||||
assertEquals(512, request.get().path("max_tokens").asInt());
|
||||
assertFalse(request.get().has("max_completion_tokens"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsErrorEnvelopeEvenWhenItContainsAChoice() {
|
||||
response.set("{\"error\":{\"message\":\"private-provider-detail\"}," + completion("ok", "stop").substring(1));
|
||||
var result = service.testModel("loopback", "probe-model");
|
||||
assertFalse(result.isSuccess());
|
||||
assertFalse(result.getErrorMessage().contains("private-provider-detail"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void connectionFallbackAlsoRejectsReasoningOnly() {
|
||||
var model = new ModelConfigEntity();
|
||||
model.setModelName("probe-model");
|
||||
when(configService.listModelsByProvider("loopback")).thenReturn(List.of(model));
|
||||
response.set(completion("", "length"));
|
||||
assertFalse(service.testConnection("loopback").isSuccess());
|
||||
}
|
||||
|
||||
@Test
|
||||
void discoveryRetainsFailedProbeButDoesNotRecommendIt() {
|
||||
provider.setSupportModelDiscovery(true);
|
||||
response.set(completion("partial", "length"));
|
||||
var result = service.discoverModels("loopback");
|
||||
assertEquals(1, result.getTotalDiscovered());
|
||||
assertEquals(Boolean.FALSE, result.getDiscoveredModels().getFirst().getProbeOk());
|
||||
assertNotNull(result.getDiscoveredModels().getFirst().getProbeError());
|
||||
assertTrue(result.getNewModels().isEmpty());
|
||||
}
|
||||
|
||||
private String completion(String content, String reason) {
|
||||
try {
|
||||
return mapper.writeValueAsString(Map.of("choices", List.of(Map.of(
|
||||
"finish_reason", reason, "message", Map.of("role", "assistant", "content", content)))));
|
||||
} catch (Exception e) { throw new AssertionError(e); }
|
||||
}
|
||||
}
|
||||
@ -37,7 +37,7 @@ class ModelDiscoveryServiceTestPromptTest {
|
||||
assertEquals(chatTemplateKwargs, requestBody.get("chat_template_kwargs"),
|
||||
"the nested map must be forwarded verbatim, not flattened or re-wrapped");
|
||||
assertEquals("qwen3-32b", requestBody.get("model"));
|
||||
assertEquals(10, requestBody.get("max_tokens"));
|
||||
assertEquals(512, requestBody.get("max_tokens"));
|
||||
assertEquals(0, requestBody.get("temperature"));
|
||||
}
|
||||
|
||||
@ -53,8 +53,8 @@ class ModelDiscoveryServiceTestPromptTest {
|
||||
|
||||
assertEquals(0, requestBody.get("temperature"),
|
||||
"the probe's fixed temperature=0 must win over a reserved generateKwargs key");
|
||||
assertEquals(10, requestBody.get("max_tokens"),
|
||||
"the probe's fixed max_tokens=10 must win over a reserved generateKwargs key");
|
||||
assertEquals(512, requestBody.get("max_tokens"),
|
||||
"the probe's fixed max_tokens=512 must win over a reserved generateKwargs key");
|
||||
assertFalse(requestBody.containsKey("maxTokens"),
|
||||
"reserved keys (even in their original casing) must not leak into the body verbatim");
|
||||
assertTrue(requestBody.containsKey("chat_template_kwargs"),
|
||||
@ -94,7 +94,7 @@ class ModelDiscoveryServiceTestPromptTest {
|
||||
|
||||
assertEquals(Set.of("model", "messages", "max_tokens", "temperature"), requestBody.keySet());
|
||||
assertEquals("gpt-4-turbo", requestBody.get("model"));
|
||||
assertEquals(10, requestBody.get("max_tokens"));
|
||||
assertEquals(512, requestBody.get("max_tokens"));
|
||||
assertEquals(0, requestBody.get("temperature"));
|
||||
|
||||
Map<String, Object> requestBodyFromNull = ModelDiscoveryService.buildTestPromptRequestBody("gpt-4-turbo", null);
|
||||
@ -108,6 +108,7 @@ class ModelDiscoveryServiceTestPromptTest {
|
||||
"kimi-for-coding", Map.of("temperature", 0.2));
|
||||
|
||||
assertEquals(1.0d, requestBody.get("temperature"));
|
||||
assertEquals(4096, requestBody.get("max_tokens"));
|
||||
}
|
||||
@Test
|
||||
@DisplayName("OpenAI reasoning probes use completion-token budgets and omit unsupported sampling parameters (#640)")
|
||||
@ -126,7 +127,7 @@ class ModelDiscoveryServiceTestPromptTest {
|
||||
@DisplayName("GPT-4o probes retain their supported standard parameters")
|
||||
void gpt4oProbeRemainsStandard() {
|
||||
Map<String, Object> body = ModelDiscoveryService.buildTestPromptRequestBody("gpt-4o", Map.of());
|
||||
assertEquals(10, body.get("max_tokens"));
|
||||
assertEquals(512, body.get("max_tokens"));
|
||||
assertEquals(0, body.get("temperature"));
|
||||
assertFalse(body.containsKey("max_completion_tokens"));
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user