opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-14 03:33:43 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

test(evaluation): reject duplicate keys in offline suites

Browse Source
This commit is contained in:
mateaix 2026-09-14 00:33:41 +08:00
parent ebe4981476
commit 6efae9e71c
6 changed files with 53 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineArtifactTaskReplay.java
View File

@ -23,7 +23,7 @@ import java.util.Objects;
/** Runs allowlisted platform actions on temporary files. No model, shell command or user path is executed. */
final class OfflineArtifactTaskReplay {
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
enum Operation { REGISTER, MUTATE_INPUT, MUTATE_DOWNLOAD, RESTART_MUTATE_DOWNLOAD,
STORAGE_UNAVAILABLE, DIRECT_RETURN, MISSING_OWNER, REWRITE_DISK }

2
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplay.java
View File

@ -15,7 +15,7 @@ import java.util.*;
/** Fixed service transitions on a real test database; evaluation results are fixtures, not model calls. */
final class OfflineGoalServiceTaskReplay {
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
enum Operation { CURRENT_REVISION_COMPLETION, APPEND_BEFORE_VERDICT, PAUSE_BEFORE_VERDICT,
APPEND_BEFORE_BOOTSTRAP, REPLACE_DEFINITION, ABA_DEFINITION }

2
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplay.java
View File

@ -35,7 +35,7 @@ import static org.mockito.Mockito.when;
/** Offline policy replay. The model is a fixture: no agent task is actually executed. */
final class OfflineGoalTaskReplay {
static final ObjectMapper JSON = new ObjectMapper()
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION)
.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
static final String MODE = "offline_synthetic_evaluator_replay";

2
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplay.java
View File

@ -15,7 +15,7 @@ import java.util.*;
/** Fixed platform IO/recipe replay. Does not call HTTP, a model, a shell or an Agent. */
final class OfflineJsonArtifactTaskReplay {
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
static final ObjectMapper JSON = new ObjectMapper().enable(com.fasterxml.jackson.core.JsonParser.Feature.STRICT_DUPLICATE_DETECTION).enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
enum Operation { CHECK, REWRITE_DISK, TOO_SMALL_BUDGET, FOREIGN_OWNER }
enum Status { MATCH, MISSING_FIELDS, INVALID_JSON, UNKNOWN, STALE, UNAVAILABLE }

42
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineSuiteValidationTest.java Normal file
View File

@ -0,0 +1,42 @@
package vip.mate.evaluation;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.*;
class OfflineSuiteValidationTest {
@ParameterizedTest
@CsvSource({"goal,goal-boundaries-v1.json", "artifact,artifact-boundaries-v1.json",
"json,json-artifact-boundaries-v1.json", "service,goal-service-boundaries-v1.json"})
void duplicateRootAndExpectedKeysAreRejectedBeforeReplay(String mode, String fixture) throws Exception {
String original;
try (var stream = getClass().getResourceAsStream("/agent-evaluation/" + fixture)) {
assertNotNull(stream);
original = new String(stream.readAllBytes(), StandardCharsets.UTF_8);
}
String duplicateRoot = original.replaceFirst("\\{", "{\"schemaVersion\":2,");
assertThrows(JsonProcessingException.class, () -> parse(mode, duplicateRoot));
var expected = OfflineGoalTaskReplay.JSON.readTree(original).path("tasks").get(0).path("expected");
String field = expected.fieldNames().next();
int start = original.indexOf('{', original.indexOf("\"expected\"")) + 1;
String duplicateExpected = original.substring(0, start) + "\"" + field + "\":" + expected.get(field)
+ "," + original.substring(start);
assertThrows(JsonProcessingException.class, () -> parse(mode, duplicateExpected));
}
private void parse(String mode, String json) throws Exception {
byte[] bytes = json.getBytes(StandardCharsets.UTF_8);
switch (mode) {
case "goal" -> OfflineGoalTaskReplay.parse(bytes);
case "artifact" -> OfflineArtifactTaskReplay.parse(bytes);
case "json" -> OfflineJsonArtifactTaskReplay.parse(bytes);
case "service" -> OfflineGoalServiceTaskReplay.parse(bytes);
default -> throw new IllegalArgumentException(mode);
}
}
}

7
mateclaw-server/src/test/resources/agent-evaluation/README.md
View File

@ -203,3 +203,10 @@ related regression boundaries; do not treat them as 30 independent online
Agent task attempts. Online calls remain zero, and online cost/Agent success
rate remain `not_measured`. HTTP authorization, browser flows and distributed
owner/scope fences are not exercised by the H2 replay.
All four suite parsers reject duplicate JSON keys at every nesting level, before
running any case. This includes duplicate `schemaVersion` and keys inside
`expected`, even if the duplicate values agree. Correct the input rather than
relying on last-value-wins parsing. The four parser regression inputs are schema
checks, not additional task scenarios; the fixed task count remains 30.