From 5c91dcc234db6067fd0e2057d2416f9414f52e74 Mon Sep 17 00:00:00 2001 From: mateaix <7333791@qq.com> Date: Sun, 13 Sep 2026 22:20:49 +0800 Subject: [PATCH] test: add replayable JSON artifact check scenarios --- .../OfflineJsonArtifactTaskReplay.java | 105 +++++++++++++ .../OfflineJsonArtifactTaskReplayTest.java | 44 ++++++ .../test/resources/agent-evaluation/README.md | 46 ++++++ .../json-artifact-baseline-v1.json | 145 ++++++++++++++++++ .../json-artifact-boundaries-v1.json | 104 +++++++++++++ 5 files changed, 444 insertions(+) create mode 100644 mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplay.java create mode 100644 mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplayTest.java create mode 100644 mateclaw-server/src/test/resources/agent-evaluation/json-artifact-baseline-v1.json create mode 100644 mateclaw-server/src/test/resources/agent-evaluation/json-artifact-boundaries-v1.json diff --git a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplay.java b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplay.java new file mode 100644 index 00000000..c2fbd496 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplay.java @@ -0,0 +1,105 @@ +package vip.mate.evaluation; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import vip.mate.execution.evidence.service.JsonArtifactRecipe; +import vip.mate.tool.document.GeneratedFileCache; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; + +/** Fixed platform IO/recipe replay. Does not call HTTP, a model, a shell or an Agent. */ +final class OfflineJsonArtifactTaskReplay { + static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS); + enum Operation { CHECK, REWRITE_DISK, TOO_SMALL_BUDGET, FOREIGN_OWNER } + enum Status { MATCH, MISSING_FIELDS, INVALID_JSON, UNKNOWN, STALE, UNAVAILABLE } + record Suite(Integer schemaVersion, String suiteId, List tasks) { } + record Task(String id, String task, String source, Operation operation, String content, + List requiredFields, Expected expected) { } + record Expected(Status status, List missingFields, Boolean recipeInvoked, Boolean acceptanceEligible) { } + record Actual(String readStatus, Status status, List missingFields, boolean recipeInvoked, + String recipeId, int recipeRevision, boolean acceptanceEligible) { } + record Case(String id, String task, String source, Expected expected, Actual actual, boolean matched, String error) { } + record Report(int schemaVersion, String suiteId, String suiteSha256, String revisionLabel, + Map productionClassSha256, String executionMode, int onlineModelCalls, + String agentTaskSuccessRate, String onlineCost, int matchedCases, int mismatchedCases, List cases) { } + + static Suite parse(byte[] bytes) throws IOException { + Suite suite = JSON.readValue(bytes, Suite.class); + require(suite != null && Integer.valueOf(1).equals(suite.schemaVersion()), "schemaVersion must be 1"); + require(nonblank(suite.suiteId()) && suite.tasks() != null && !suite.tasks().isEmpty() + && suite.tasks().size() <= 100, "suiteId and 1..100 tasks required"); + Set ids = new HashSet<>(); + for (Task task : suite.tasks()) { + require(task != null && nonblank(task.id()) && ids.add(task.id()), "unique IDs required"); + require(nonblank(task.task()) && nonblank(task.source()) && task.operation() != null + && task.content() != null && task.content().length() <= 16_384, "invalid task inputs"); + JsonArtifactRecipe.validate(task.requiredFields()); + Expected e = task.expected(); + require(e != null && e.status() != null && e.missingFields() != null + && e.recipeInvoked() != null && e.acceptanceEligible() != null, "all expectations required"); + require(e.missingFields().stream().allMatch(task.requiredFields()::contains), "unknown missing field expectation"); + } + return suite; + } + + static Report run(byte[] bytes, Path root, String revisionLabel) throws IOException { + Suite suite = parse(bytes); // Validate the entire suite before any file mutation. + require(nonblank(revisionLabel), "revisionLabel required"); + List results = new ArrayList<>(); + for (Task task : suite.tasks()) { + try { + Actual actual = execute(task, Files.createTempDirectory(root, "json-artifact-case-")); + Expected e = task.expected(); + boolean matched = e.status() == actual.status() && e.missingFields().equals(actual.missingFields()) + && e.recipeInvoked() == actual.recipeInvoked() && e.acceptanceEligible() == actual.acceptanceEligible(); + results.add(new Case(task.id(), task.task(), task.source(), e, actual, matched, null)); + } catch (IOException | RuntimeException error) { + results.add(new Case(task.id(), task.task(), task.source(), task.expected(), null, false, + error.getClass().getSimpleName())); + } + } + Map classes = new LinkedHashMap<>(); + for (Class type : List.of(GeneratedFileCache.class, GeneratedFileCache.ArtifactRead.class, + JsonArtifactRecipe.class, JsonArtifactRecipe.Result.class)) { + try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) { + if (stream == null) throw new IOException("Missing tested class " + type.getName()); + classes.put(type.getName(), digest(stream.readAllBytes())); + } + } + int matched = (int) results.stream().filter(Case::matched).count(); + return new Report(1, suite.suiteId(), digest(bytes), revisionLabel, classes, + "offline_platform_fixture_jsoncheck", 0, "not_measured", "not_measured", matched, + results.size() - matched, List.copyOf(results)); + } + + private static Actual execute(Task task, Path root) throws IOException { + var cache = new GeneratedFileCache(root); + byte[] bytes = task.content().getBytes(StandardCharsets.UTF_8); + String id = cache.put(bytes, "report.json", "application/json", new GeneratedFileCache.Owner(1L, 1L, "fixture")); + if (task.operation() == Operation.REWRITE_DISK) Files.writeString(root.resolve(id), "{}"); + int budget = task.operation() == Operation.TOO_SMALL_BUDGET ? 1 : 1_048_576; + long owner = task.operation() == Operation.FOREIGN_OWNER ? 2L : 1L; + var read = cache.readDurableArtifactSnapshot(id, owner, "fixture", digest(bytes), budget); + boolean invoked = "READ".equals(read.status()); + var result = invoked ? JsonArtifactRecipe.check(read.bytes(), task.requiredFields()) + : JsonArtifactRecipe.outcome(read.status(), task.requiredFields(), List.of()); + return new Actual(read.status(), Status.valueOf(result.status()), result.missingFields(), invoked, + result.recipeId(), result.recipeRevision(), result.acceptanceEligible()); + } + + private static String digest(byte[] bytes) { + try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); } + catch (NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); } + } + private static boolean nonblank(String value) { return value != null && !value.isBlank(); } + private static void require(boolean condition, String message) { + if (!condition) throw new IllegalArgumentException(message); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplayTest.java b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplayTest.java new file mode 100644 index 00000000..1e75784e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineJsonArtifactTaskReplayTest.java @@ -0,0 +1,44 @@ +package vip.mate.evaluation; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import static org.junit.jupiter.api.Assertions.*; + +class OfflineJsonArtifactTaskReplayTest { + @TempDir Path root; + private byte[] fixture() throws Exception { + try (var stream = getClass().getResourceAsStream("/agent-evaluation/json-artifact-boundaries-v1.json")) { + assertNotNull(stream); + return stream.readAllBytes(); + } + } + @Test void executeTasksAndWriteReport() throws Exception { + String input = System.getProperty("json.artifact.eval.suite"); + var report = OfflineJsonArtifactTaskReplay.run(input == null ? fixture() : Files.readAllBytes(Path.of(input)), + root, System.getProperty("json.artifact.eval.revision", "unrecorded")); + Path output = Path.of(System.getProperty("json.artifact.eval.report", "target/agent-evaluation/json-artifact-baseline.json")); + Files.createDirectories(output.toAbsolutePath().getParent()); + OfflineJsonArtifactTaskReplay.JSON.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), report); + assertEquals(0, report.mismatchedCases(), () -> "JSON artifact mismatches: " + output.toAbsolutePath()); + assertEquals(0, report.onlineModelCalls()); + assertEquals(4, report.productionClassSha256().size()); + } + @Test void invalidLaterTaskFailsBeforeAnyFileIsCreated() throws Exception { + ObjectNode suite = (ObjectNode) OfflineJsonArtifactTaskReplay.JSON.readTree(fixture()); + ((ObjectNode) suite.withArray("tasks").get(1).get("expected")).remove("recipeInvoked"); + assertThrows(IllegalArgumentException.class, () -> OfflineJsonArtifactTaskReplay.run( + suite.toString().getBytes(StandardCharsets.UTF_8), root, "test")); + try (var files = Files.list(root)) { assertEquals(0, files.count()); } + } + @Test void wrongExpectationRecordsMismatchAndRunsEveryTask() throws Exception { + ObjectNode suite = (ObjectNode) OfflineJsonArtifactTaskReplay.JSON.readTree(fixture()); + ((ObjectNode) suite.withArray("tasks").get(0).get("expected")).put("acceptanceEligible", true); + var report = OfflineJsonArtifactTaskReplay.run(suite.toString().getBytes(StandardCharsets.UTF_8), root, "test"); + assertEquals(1, report.mismatchedCases()); + assertEquals(suite.withArray("tasks").size(), report.cases().size()); + } +} diff --git a/mateclaw-server/src/test/resources/agent-evaluation/README.md b/mateclaw-server/src/test/resources/agent-evaluation/README.md index a28c61e1..7eb609ae 100644 --- a/mateclaw-server/src/test/resources/agent-evaluation/README.md +++ b/mateclaw-server/src/test/resources/agent-evaluation/README.md @@ -105,3 +105,49 @@ storage, not an accepted strong-validation result. The report never emits a CHECK_RESULT/PASS. These eight tests plus ten evaluator replays are **18 fixed scenarios, not 18 Agent task executions**. Actual task-solving models, their latency, cost and success rate remain unmeasured. + +## JSON artifact check pilot + +The evidence details panel can explicitly check a registered JSON file with +`POST /api/v1/execution-evidence/{id}/json-check` and a body such as +`{"requiredFields":["report","appendix"]}`. The server uses the conversation and +file permissions of the authenticated caller. Field names are exact top-level +keys (1–16 unique keys, up to 128 characters each); values must be non-null. +This does not validate value types, business correctness, or document quality. + +The fixed `json-required-fields` recipe revision 1 reads at most the configured +artifact-version budget, capped at 1 MiB. Zero disables reading. Durable bytes +must match the registered digest before parsing. The result distinguishes +`MATCH`, `MISSING_FIELDS`, `INVALID_JSON`, `UNKNOWN`, `STALE`, `UNAVAILABLE` and +always has `acceptanceEligible=false`. Duplicate keys, multiple JSON documents, +and nesting deeper than 32 are rejected. Responses contain no file values or +parser excerpts. No Goal criterion, acceptance binding, or evidence PASS is +written. The result describes the captured bytes at the displayed time; shared +storage still has no managed generation/owner fence. + +Run the separate six-case IO/recipe replay: + +```sh +mvn -pl mateclaw-server -am -Dtest=OfflineJsonArtifactTaskReplayTest \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Djson.artifact.eval.revision="$(git rev-parse HEAD)" test +``` + +Default output: `mateclaw-server/target/agent-evaluation/json-artifact-baseline.json`. +Override `json.artifact.eval.suite` and `json.artifact.eval.report` with absolute +paths. `json-artifact-boundaries-v1.json` contains requirements and explicit +expected status, missing fields, whether the recipe actually ran, and whether +acceptance was granted. The harness validates all tasks before creating its own +temporary paths. Operations are allowlisted: `CHECK`, `REWRITE_DISK`, +`TOO_SMALL_BUDGET`, `FOREIGN_OWNER`. It never executes fixture paths or commands. + +The report hashes the suite and loaded cache/read-result/recipe/result classes, +records every mismatch, and fails the command if any case differs. Class hashes +identify these loaded classes under this build, not every dependency. +`json-artifact-baseline-v1.json` is the initial six-case baseline. Together with +the earlier ten evaluator and eight platform IO scenarios there are **24 fixed +scenarios**, not 24 real Agent runs. The JSON replay uses real temporary durable +files and the production recipe; it does not exercise HTTP authorization or a +browser. Those boundaries have separate service/UI regressions. Its mode is +`offline_platform_fixture_jsoncheck`; online calls are zero and Agent success +rate/online cost remain `not_measured`. diff --git a/mateclaw-server/src/test/resources/agent-evaluation/json-artifact-baseline-v1.json b/mateclaw-server/src/test/resources/agent-evaluation/json-artifact-baseline-v1.json new file mode 100644 index 00000000..a0e4af92 --- /dev/null +++ b/mateclaw-server/src/test/resources/agent-evaluation/json-artifact-baseline-v1.json @@ -0,0 +1,145 @@ +{ + "schemaVersion" : 1, + "suiteId" : "json-artifact-boundaries-v1", + "suiteSha256" : "b8772a0bcd24f16ff0b1bb2ef01e0aae7988a66cd850517152b3fcef85fee1e8", + "revisionLabel" : "ef209228f+cycle012-working", + "productionClassSha256" : { + "vip.mate.tool.document.GeneratedFileCache" : "9db54f575cdd1a2bc750b97d62adeaf93c3590d78f189c9e80bffce271cb65b1", + "vip.mate.tool.document.GeneratedFileCache$ArtifactRead" : "078a1997f925cd3b4d44f0de82934de49ff18c926111177efb567a174bc7aeb6", + "vip.mate.execution.evidence.service.JsonArtifactRecipe" : "7b56e5fb25e12d878d683432e853187c4f8e3e0d75c759be4c9762e59353a592", + "vip.mate.execution.evidence.service.JsonArtifactRecipe$Result" : "4aaa1cce1fc5a8159254c10ef80f0620feed4de3b8e919be4ecb3649143b5c92" + }, + "executionMode" : "offline_platform_fixture_jsoncheck", + "onlineModelCalls" : 0, + "agentTaskSuccessRate" : "not_measured", + "onlineCost" : "not_measured", + "matchedCases" : 6, + "mismatchedCases" : 0, + "cases" : [ { + "id" : "required-fields-present", + "task" : "required fields present", + "source" : "cycle011 JSON artifact check boundary", + "expected" : { + "status" : "MATCH", + "missingFields" : [ ], + "recipeInvoked" : true, + "acceptanceEligible" : false + }, + "actual" : { + "readStatus" : "READ", + "status" : "MATCH", + "missingFields" : [ ], + "recipeInvoked" : true, + "recipeId" : "json-required-fields", + "recipeRevision" : 1, + "acceptanceEligible" : false + }, + "matched" : true, + "error" : null + }, { + "id" : "missing-required-field", + "task" : "missing required field", + "source" : "cycle011 JSON artifact check boundary", + "expected" : { + "status" : "MISSING_FIELDS", + "missingFields" : [ "report" ], + "recipeInvoked" : true, + "acceptanceEligible" : false + }, + "actual" : { + "readStatus" : "READ", + "status" : "MISSING_FIELDS", + "missingFields" : [ "report" ], + "recipeInvoked" : true, + "recipeId" : "json-required-fields", + "recipeRevision" : 1, + "acceptanceEligible" : false + }, + "matched" : true, + "error" : null + }, { + "id" : "duplicate-json-key", + "task" : "duplicate json key", + "source" : "cycle011 JSON artifact check boundary", + "expected" : { + "status" : "INVALID_JSON", + "missingFields" : [ ], + "recipeInvoked" : true, + "acceptanceEligible" : false + }, + "actual" : { + "readStatus" : "READ", + "status" : "INVALID_JSON", + "missingFields" : [ ], + "recipeInvoked" : true, + "recipeId" : "json-required-fields", + "recipeRevision" : 1, + "acceptanceEligible" : false + }, + "matched" : true, + "error" : null + }, { + "id" : "changed-durable-file", + "task" : "changed durable file", + "source" : "cycle011 JSON artifact check boundary", + "expected" : { + "status" : "STALE", + "missingFields" : [ ], + "recipeInvoked" : false, + "acceptanceEligible" : false + }, + "actual" : { + "readStatus" : "STALE", + "status" : "STALE", + "missingFields" : [ ], + "recipeInvoked" : false, + "recipeId" : "json-required-fields", + "recipeRevision" : 1, + "acceptanceEligible" : false + }, + "matched" : true, + "error" : null + }, { + "id" : "insufficient-read-budget", + "task" : "insufficient read budget", + "source" : "cycle011 JSON artifact check boundary", + "expected" : { + "status" : "UNKNOWN", + "missingFields" : [ ], + "recipeInvoked" : false, + "acceptanceEligible" : false + }, + "actual" : { + "readStatus" : "UNKNOWN", + "status" : "UNKNOWN", + "missingFields" : [ ], + "recipeInvoked" : false, + "recipeId" : "json-required-fields", + "recipeRevision" : 1, + "acceptanceEligible" : false + }, + "matched" : true, + "error" : null + }, { + "id" : "foreign-artifact-owner", + "task" : "foreign artifact owner", + "source" : "cycle011 JSON artifact check boundary", + "expected" : { + "status" : "UNAVAILABLE", + "missingFields" : [ ], + "recipeInvoked" : false, + "acceptanceEligible" : false + }, + "actual" : { + "readStatus" : "UNAVAILABLE", + "status" : "UNAVAILABLE", + "missingFields" : [ ], + "recipeInvoked" : false, + "recipeId" : "json-required-fields", + "recipeRevision" : 1, + "acceptanceEligible" : false + }, + "matched" : true, + "error" : null + } ] +} \ No newline at end of file diff --git a/mateclaw-server/src/test/resources/agent-evaluation/json-artifact-boundaries-v1.json b/mateclaw-server/src/test/resources/agent-evaluation/json-artifact-boundaries-v1.json new file mode 100644 index 00000000..f48991c4 --- /dev/null +++ b/mateclaw-server/src/test/resources/agent-evaluation/json-artifact-boundaries-v1.json @@ -0,0 +1,104 @@ +{ + "schemaVersion": 1, + "suiteId": "json-artifact-boundaries-v1", + "tasks": [ + { + "id": "required-fields-present", + "task": "required fields present", + "source": "cycle011 JSON artifact check boundary", + "operation": "CHECK", + "content": "{\"report\":false}", + "requiredFields": [ + "report" + ], + "expected": { + "status": "MATCH", + "missingFields": [], + "recipeInvoked": true, + "acceptanceEligible": false + } + }, + { + "id": "missing-required-field", + "task": "missing required field", + "source": "cycle011 JSON artifact check boundary", + "operation": "CHECK", + "content": "{\"appendix\":1}", + "requiredFields": [ + "report" + ], + "expected": { + "status": "MISSING_FIELDS", + "missingFields": [ + "report" + ], + "recipeInvoked": true, + "acceptanceEligible": false + } + }, + { + "id": "duplicate-json-key", + "task": "duplicate json key", + "source": "cycle011 JSON artifact check boundary", + "operation": "CHECK", + "content": "{\"report\":1,\"report\":2}", + "requiredFields": [ + "report" + ], + "expected": { + "status": "INVALID_JSON", + "missingFields": [], + "recipeInvoked": true, + "acceptanceEligible": false + } + }, + { + "id": "changed-durable-file", + "task": "changed durable file", + "source": "cycle011 JSON artifact check boundary", + "operation": "REWRITE_DISK", + "content": "{\"report\":true}", + "requiredFields": [ + "report" + ], + "expected": { + "status": "STALE", + "missingFields": [], + "recipeInvoked": false, + "acceptanceEligible": false + } + }, + { + "id": "insufficient-read-budget", + "task": "insufficient read budget", + "source": "cycle011 JSON artifact check boundary", + "operation": "TOO_SMALL_BUDGET", + "content": "{\"report\":true}", + "requiredFields": [ + "report" + ], + "expected": { + "status": "UNKNOWN", + "missingFields": [], + "recipeInvoked": false, + "acceptanceEligible": false + } + }, + { + "id": "foreign-artifact-owner", + "task": "foreign artifact owner", + "source": "cycle011 JSON artifact check boundary", + "operation": "FOREIGN_OWNER", + "content": "{\"report\":true}", + "requiredFields": [ + "report" + ], + "expected": { + "status": "UNAVAILABLE", + "missingFields": [], + "recipeInvoked": false, + "acceptanceEligible": false + } + } + ] +}