mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-16 04:18:17 +08:00
test(evaluation): reject duplicate keys in offline suites
This commit is contained in:
parent
ebe4981476
commit
6efae9e71c
@ -23,7 +23,7 @@ import java.util.Objects;
|
|||||||
|
|
||||||
/** Runs allowlisted platform actions on temporary files. No model, shell command or user path is executed. */
|
/** Runs allowlisted platform actions on temporary files. No model, shell command or user path is executed. */
|
||||||
final class OfflineArtifactTaskReplay {
|
final class OfflineArtifactTaskReplay {
|
||||||
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
||||||
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
||||||
enum Operation { REGISTER, MUTATE_INPUT, MUTATE_DOWNLOAD, RESTART_MUTATE_DOWNLOAD,
|
enum Operation { REGISTER, MUTATE_INPUT, MUTATE_DOWNLOAD, RESTART_MUTATE_DOWNLOAD,
|
||||||
STORAGE_UNAVAILABLE, DIRECT_RETURN, MISSING_OWNER, REWRITE_DISK }
|
STORAGE_UNAVAILABLE, DIRECT_RETURN, MISSING_OWNER, REWRITE_DISK }
|
||||||
|
|||||||
@ -15,7 +15,7 @@ import java.util.*;
|
|||||||
|
|
||||||
/** Fixed service transitions on a real test database; evaluation results are fixtures, not model calls. */
|
/** Fixed service transitions on a real test database; evaluation results are fixtures, not model calls. */
|
||||||
final class OfflineGoalServiceTaskReplay {
|
final class OfflineGoalServiceTaskReplay {
|
||||||
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
||||||
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
||||||
enum Operation { CURRENT_REVISION_COMPLETION, APPEND_BEFORE_VERDICT, PAUSE_BEFORE_VERDICT,
|
enum Operation { CURRENT_REVISION_COMPLETION, APPEND_BEFORE_VERDICT, PAUSE_BEFORE_VERDICT,
|
||||||
APPEND_BEFORE_BOOTSTRAP, REPLACE_DEFINITION, ABA_DEFINITION }
|
APPEND_BEFORE_BOOTSTRAP, REPLACE_DEFINITION, ABA_DEFINITION }
|
||||||
|
|||||||
@ -35,7 +35,7 @@ import static org.mockito.Mockito.when;
|
|||||||
|
|
||||||
/** Offline policy replay. The model is a fixture: no agent task is actually executed. */
|
/** Offline policy replay. The model is a fixture: no agent task is actually executed. */
|
||||||
final class OfflineGoalTaskReplay {
|
final class OfflineGoalTaskReplay {
|
||||||
static final ObjectMapper JSON = new ObjectMapper()
|
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
|
||||||
.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
|
||||||
static final String MODE = "offline_synthetic_evaluator_replay";
|
static final String MODE = "offline_synthetic_evaluator_replay";
|
||||||
|
|
||||||
|
|||||||
@ -15,7 +15,7 @@ import java.util.*;
|
|||||||
|
|
||||||
/** Fixed platform IO/recipe replay. Does not call HTTP, a model, a shell or an Agent. */
|
/** Fixed platform IO/recipe replay. Does not call HTTP, a model, a shell or an Agent. */
|
||||||
final class OfflineJsonArtifactTaskReplay {
|
final class OfflineJsonArtifactTaskReplay {
|
||||||
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
||||||
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
||||||
enum Operation { CHECK, REWRITE_DISK, TOO_SMALL_BUDGET, FOREIGN_OWNER }
|
enum Operation { CHECK, REWRITE_DISK, TOO_SMALL_BUDGET, FOREIGN_OWNER }
|
||||||
enum Status { MATCH, MISSING_FIELDS, INVALID_JSON, UNKNOWN, STALE, UNAVAILABLE }
|
enum Status { MATCH, MISSING_FIELDS, INVALID_JSON, UNKNOWN, STALE, UNAVAILABLE }
|
||||||
|
|||||||
@ -0,0 +1,42 @@
|
|||||||
|
package vip.mate.evaluation;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import org.junit.jupiter.params.ParameterizedTest;
|
||||||
|
import org.junit.jupiter.params.provider.CsvSource;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
class OfflineSuiteValidationTest {
|
||||||
|
@ParameterizedTest
|
||||||
|
@CsvSource({"goal,goal-boundaries-v1.json", "artifact,artifact-boundaries-v1.json",
|
||||||
|
"json,json-artifact-boundaries-v1.json", "service,goal-service-boundaries-v1.json"})
|
||||||
|
void duplicateRootAndExpectedKeysAreRejectedBeforeReplay(String mode, String fixture) throws Exception {
|
||||||
|
String original;
|
||||||
|
try (var stream = getClass().getResourceAsStream("/agent-evaluation/" + fixture)) {
|
||||||
|
assertNotNull(stream);
|
||||||
|
original = new String(stream.readAllBytes(), StandardCharsets.UTF_8);
|
||||||
|
}
|
||||||
|
String duplicateRoot = original.replaceFirst("\\{", "{\"schemaVersion\":2,");
|
||||||
|
assertThrows(JsonProcessingException.class, () -> parse(mode, duplicateRoot));
|
||||||
|
|
||||||
|
var expected = OfflineGoalTaskReplay.JSON.readTree(original).path("tasks").get(0).path("expected");
|
||||||
|
String field = expected.fieldNames().next();
|
||||||
|
int start = original.indexOf('{', original.indexOf("\"expected\"")) + 1;
|
||||||
|
String duplicateExpected = original.substring(0, start) + "\"" + field + "\":" + expected.get(field)
|
||||||
|
+ "," + original.substring(start);
|
||||||
|
assertThrows(JsonProcessingException.class, () -> parse(mode, duplicateExpected));
|
||||||
|
}
|
||||||
|
|
||||||
|
private void parse(String mode, String json) throws Exception {
|
||||||
|
byte[] bytes = json.getBytes(StandardCharsets.UTF_8);
|
||||||
|
switch (mode) {
|
||||||
|
case "goal" -> OfflineGoalTaskReplay.parse(bytes);
|
||||||
|
case "artifact" -> OfflineArtifactTaskReplay.parse(bytes);
|
||||||
|
case "json" -> OfflineJsonArtifactTaskReplay.parse(bytes);
|
||||||
|
case "service" -> OfflineGoalServiceTaskReplay.parse(bytes);
|
||||||
|
default -> throw new IllegalArgumentException(mode);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -203,3 +203,10 @@ related regression boundaries; do not treat them as 30 independent online
|
|||||||
Agent task attempts. Online calls remain zero, and online cost/Agent success
|
Agent task attempts. Online calls remain zero, and online cost/Agent success
|
||||||
rate remain `not_measured`. HTTP authorization, browser flows and distributed
|
rate remain `not_measured`. HTTP authorization, browser flows and distributed
|
||||||
owner/scope fences are not exercised by the H2 replay.
|
owner/scope fences are not exercised by the H2 replay.
|
||||||
|
|
||||||
|
|
||||||
|
All four suite parsers reject duplicate JSON keys at every nesting level, before
|
||||||
|
running any case. This includes duplicate `schemaVersion` and keys inside
|
||||||
|
`expected`, even if the duplicate values agree. Correct the input rather than
|
||||||
|
relying on last-value-wins parsing. The four parser regression inputs are schema
|
||||||
|
checks, not additional task scenarios; the fixed task count remains 30.
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user