mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-14 03:33:43 +08:00
test: add replayable JSON artifact check scenarios
This commit is contained in:
mateaix
parent
fb7bb04f7a
commit
5c91dcc234
@ -0,0 +1,105 @@
|
||||
package vip.mate.evaluation;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import vip.mate.execution.evidence.service.JsonArtifactRecipe;
|
||||
import vip.mate.tool.document.GeneratedFileCache;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.*;
|
||||
|
||||
/** Fixed platform IO/recipe replay. Does not call HTTP, a model, a shell or an Agent. */
|
||||
final class OfflineJsonArtifactTaskReplay {
|
||||
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
||||
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
||||
enum Operation { CHECK, REWRITE_DISK, TOO_SMALL_BUDGET, FOREIGN_OWNER }
|
||||
enum Status { MATCH, MISSING_FIELDS, INVALID_JSON, UNKNOWN, STALE, UNAVAILABLE }
|
||||
record Suite(Integer schemaVersion, String suiteId, List<Task> tasks) { }
|
||||
record Task(String id, String task, String source, Operation operation, String content,
|
||||
List<String> requiredFields, Expected expected) { }
|
||||
record Expected(Status status, List<String> missingFields, Boolean recipeInvoked, Boolean acceptanceEligible) { }
|
||||
record Actual(String readStatus, Status status, List<String> missingFields, boolean recipeInvoked,
|
||||
String recipeId, int recipeRevision, boolean acceptanceEligible) { }
|
||||
record Case(String id, String task, String source, Expected expected, Actual actual, boolean matched, String error) { }
|
||||
record Report(int schemaVersion, String suiteId, String suiteSha256, String revisionLabel,
|
||||
Map<String, String> productionClassSha256, String executionMode, int onlineModelCalls,
|
||||
String agentTaskSuccessRate, String onlineCost, int matchedCases, int mismatchedCases, List<Case> cases) { }
|
||||
|
||||
static Suite parse(byte[] bytes) throws IOException {
|
||||
Suite suite = JSON.readValue(bytes, Suite.class);
|
||||
require(suite != null && Integer.valueOf(1).equals(suite.schemaVersion()), "schemaVersion must be 1");
|
||||
require(nonblank(suite.suiteId()) && suite.tasks() != null && !suite.tasks().isEmpty()
|
||||
&& suite.tasks().size() <= 100, "suiteId and 1..100 tasks required");
|
||||
Set<String> ids = new HashSet<>();
|
||||
for (Task task : suite.tasks()) {
|
||||
require(task != null && nonblank(task.id()) && ids.add(task.id()), "unique IDs required");
|
||||
require(nonblank(task.task()) && nonblank(task.source()) && task.operation() != null
|
||||
&& task.content() != null && task.content().length() <= 16_384, "invalid task inputs");
|
||||
JsonArtifactRecipe.validate(task.requiredFields());
|
||||
Expected e = task.expected();
|
||||
require(e != null && e.status() != null && e.missingFields() != null
|
||||
&& e.recipeInvoked() != null && e.acceptanceEligible() != null, "all expectations required");
|
||||
require(e.missingFields().stream().allMatch(task.requiredFields()::contains), "unknown missing field expectation");
|
||||
}
|
||||
return suite;
|
||||
}
|
||||
|
||||
static Report run(byte[] bytes, Path root, String revisionLabel) throws IOException {
|
||||
Suite suite = parse(bytes); // Validate the entire suite before any file mutation.
|
||||
require(nonblank(revisionLabel), "revisionLabel required");
|
||||
List<Case> results = new ArrayList<>();
|
||||
for (Task task : suite.tasks()) {
|
||||
try {
|
||||
Actual actual = execute(task, Files.createTempDirectory(root, "json-artifact-case-"));
|
||||
Expected e = task.expected();
|
||||
boolean matched = e.status() == actual.status() && e.missingFields().equals(actual.missingFields())
|
||||
&& e.recipeInvoked() == actual.recipeInvoked() && e.acceptanceEligible() == actual.acceptanceEligible();
|
||||
results.add(new Case(task.id(), task.task(), task.source(), e, actual, matched, null));
|
||||
} catch (IOException | RuntimeException error) {
|
||||
results.add(new Case(task.id(), task.task(), task.source(), task.expected(), null, false,
|
||||
error.getClass().getSimpleName()));
|
||||
}
|
||||
}
|
||||
Map<String, String> classes = new LinkedHashMap<>();
|
||||
for (Class<?> type : List.of(GeneratedFileCache.class, GeneratedFileCache.ArtifactRead.class,
|
||||
JsonArtifactRecipe.class, JsonArtifactRecipe.Result.class)) {
|
||||
try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) {
|
||||
if (stream == null) throw new IOException("Missing tested class " + type.getName());
|
||||
classes.put(type.getName(), digest(stream.readAllBytes()));
|
||||
}
|
||||
}
|
||||
int matched = (int) results.stream().filter(Case::matched).count();
|
||||
return new Report(1, suite.suiteId(), digest(bytes), revisionLabel, classes,
|
||||
"offline_platform_fixture_jsoncheck", 0, "not_measured", "not_measured", matched,
|
||||
results.size() - matched, List.copyOf(results));
|
||||
}
|
||||
|
||||
private static Actual execute(Task task, Path root) throws IOException {
|
||||
var cache = new GeneratedFileCache(root);
|
||||
byte[] bytes = task.content().getBytes(StandardCharsets.UTF_8);
|
||||
String id = cache.put(bytes, "report.json", "application/json", new GeneratedFileCache.Owner(1L, 1L, "fixture"));
|
||||
if (task.operation() == Operation.REWRITE_DISK) Files.writeString(root.resolve(id), "{}");
|
||||
int budget = task.operation() == Operation.TOO_SMALL_BUDGET ? 1 : 1_048_576;
|
||||
long owner = task.operation() == Operation.FOREIGN_OWNER ? 2L : 1L;
|
||||
var read = cache.readDurableArtifactSnapshot(id, owner, "fixture", digest(bytes), budget);
|
||||
boolean invoked = "READ".equals(read.status());
|
||||
var result = invoked ? JsonArtifactRecipe.check(read.bytes(), task.requiredFields())
|
||||
: JsonArtifactRecipe.outcome(read.status(), task.requiredFields(), List.of());
|
||||
return new Actual(read.status(), Status.valueOf(result.status()), result.missingFields(), invoked,
|
||||
result.recipeId(), result.recipeRevision(), result.acceptanceEligible());
|
||||
}
|
||||
|
||||
private static String digest(byte[] bytes) {
|
||||
try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); }
|
||||
catch (NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); }
|
||||
}
|
||||
private static boolean nonblank(String value) { return value != null && !value.isBlank(); }
|
||||
private static void require(boolean condition, String message) {
|
||||
if (!condition) throw new IllegalArgumentException(message);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package vip.mate.evaluation;
|
||||
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class OfflineJsonArtifactTaskReplayTest {
|
||||
@TempDir Path root;
|
||||
private byte[] fixture() throws Exception {
|
||||
try (var stream = getClass().getResourceAsStream("/agent-evaluation/json-artifact-boundaries-v1.json")) {
|
||||
assertNotNull(stream);
|
||||
return stream.readAllBytes();
|
||||
}
|
||||
}
|
||||
@Test void executeTasksAndWriteReport() throws Exception {
|
||||
String input = System.getProperty("json.artifact.eval.suite");
|
||||
var report = OfflineJsonArtifactTaskReplay.run(input == null ? fixture() : Files.readAllBytes(Path.of(input)),
|
||||
root, System.getProperty("json.artifact.eval.revision", "unrecorded"));
|
||||
Path output = Path.of(System.getProperty("json.artifact.eval.report", "target/agent-evaluation/json-artifact-baseline.json"));
|
||||
Files.createDirectories(output.toAbsolutePath().getParent());
|
||||
OfflineJsonArtifactTaskReplay.JSON.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), report);
|
||||
assertEquals(0, report.mismatchedCases(), () -> "JSON artifact mismatches: " + output.toAbsolutePath());
|
||||
assertEquals(0, report.onlineModelCalls());
|
||||
assertEquals(4, report.productionClassSha256().size());
|
||||
}
|
||||
@Test void invalidLaterTaskFailsBeforeAnyFileIsCreated() throws Exception {
|
||||
ObjectNode suite = (ObjectNode) OfflineJsonArtifactTaskReplay.JSON.readTree(fixture());
|
||||
((ObjectNode) suite.withArray("tasks").get(1).get("expected")).remove("recipeInvoked");
|
||||
assertThrows(IllegalArgumentException.class, () -> OfflineJsonArtifactTaskReplay.run(
|
||||
suite.toString().getBytes(StandardCharsets.UTF_8), root, "test"));
|
||||
try (var files = Files.list(root)) { assertEquals(0, files.count()); }
|
||||
}
|
||||
@Test void wrongExpectationRecordsMismatchAndRunsEveryTask() throws Exception {
|
||||
ObjectNode suite = (ObjectNode) OfflineJsonArtifactTaskReplay.JSON.readTree(fixture());
|
||||
((ObjectNode) suite.withArray("tasks").get(0).get("expected")).put("acceptanceEligible", true);
|
||||
var report = OfflineJsonArtifactTaskReplay.run(suite.toString().getBytes(StandardCharsets.UTF_8), root, "test");
|
||||
assertEquals(1, report.mismatchedCases());
|
||||
assertEquals(suite.withArray("tasks").size(), report.cases().size());
|
||||
}
|
||||
}
|
||||
@ -105,3 +105,49 @@ storage, not an accepted strong-validation result. The report never emits a
|
||||
CHECK_RESULT/PASS. These eight tests plus ten evaluator replays are **18 fixed
|
||||
scenarios, not 18 Agent task executions**. Actual task-solving models, their
|
||||
latency, cost and success rate remain unmeasured.
|
||||
|
||||
## JSON artifact check pilot
|
||||
|
||||
The evidence details panel can explicitly check a registered JSON file with
|
||||
`POST /api/v1/execution-evidence/{id}/json-check` and a body such as
|
||||
`{"requiredFields":["report","appendix"]}`. The server uses the conversation and
|
||||
file permissions of the authenticated caller. Field names are exact top-level
|
||||
keys (1–16 unique keys, up to 128 characters each); values must be non-null.
|
||||
This does not validate value types, business correctness, or document quality.
|
||||
|
||||
The fixed `json-required-fields` recipe revision 1 reads at most the configured
|
||||
artifact-version budget, capped at 1 MiB. Zero disables reading. Durable bytes
|
||||
must match the registered digest before parsing. The result distinguishes
|
||||
`MATCH`, `MISSING_FIELDS`, `INVALID_JSON`, `UNKNOWN`, `STALE`, `UNAVAILABLE` and
|
||||
always has `acceptanceEligible=false`. Duplicate keys, multiple JSON documents,
|
||||
and nesting deeper than 32 are rejected. Responses contain no file values or
|
||||
parser excerpts. No Goal criterion, acceptance binding, or evidence PASS is
|
||||
written. The result describes the captured bytes at the displayed time; shared
|
||||
storage still has no managed generation/owner fence.
|
||||
|
||||
Run the separate six-case IO/recipe replay:
|
||||
|
||||
```sh
|
||||
mvn -pl mateclaw-server -am -Dtest=OfflineJsonArtifactTaskReplayTest \
|
||||
-Dsurefire.failIfNoSpecifiedTests=false \
|
||||
-Djson.artifact.eval.revision="$(git rev-parse HEAD)" test
|
||||
```
|
||||
|
||||
Default output: `mateclaw-server/target/agent-evaluation/json-artifact-baseline.json`.
|
||||
Override `json.artifact.eval.suite` and `json.artifact.eval.report` with absolute
|
||||
paths. `json-artifact-boundaries-v1.json` contains requirements and explicit
|
||||
expected status, missing fields, whether the recipe actually ran, and whether
|
||||
acceptance was granted. The harness validates all tasks before creating its own
|
||||
temporary paths. Operations are allowlisted: `CHECK`, `REWRITE_DISK`,
|
||||
`TOO_SMALL_BUDGET`, `FOREIGN_OWNER`. It never executes fixture paths or commands.
|
||||
|
||||
The report hashes the suite and loaded cache/read-result/recipe/result classes,
|
||||
records every mismatch, and fails the command if any case differs. Class hashes
|
||||
identify these loaded classes under this build, not every dependency.
|
||||
`json-artifact-baseline-v1.json` is the initial six-case baseline. Together with
|
||||
the earlier ten evaluator and eight platform IO scenarios there are **24 fixed
|
||||
scenarios**, not 24 real Agent runs. The JSON replay uses real temporary durable
|
||||
files and the production recipe; it does not exercise HTTP authorization or a
|
||||
browser. Those boundaries have separate service/UI regressions. Its mode is
|
||||
`offline_platform_fixture_jsoncheck`; online calls are zero and Agent success
|
||||
rate/online cost remain `not_measured`.
|
||||
|
||||
@ -0,0 +1,145 @@
|
||||
{
|
||||
"schemaVersion" : 1,
|
||||
"suiteId" : "json-artifact-boundaries-v1",
|
||||
"suiteSha256" : "b8772a0bcd24f16ff0b1bb2ef01e0aae7988a66cd850517152b3fcef85fee1e8",
|
||||
"revisionLabel" : "ef209228f+cycle012-working",
|
||||
"productionClassSha256" : {
|
||||
"vip.mate.tool.document.GeneratedFileCache" : "9db54f575cdd1a2bc750b97d62adeaf93c3590d78f189c9e80bffce271cb65b1",
|
||||
"vip.mate.tool.document.GeneratedFileCache$ArtifactRead" : "078a1997f925cd3b4d44f0de82934de49ff18c926111177efb567a174bc7aeb6",
|
||||
"vip.mate.execution.evidence.service.JsonArtifactRecipe" : "7b56e5fb25e12d878d683432e853187c4f8e3e0d75c759be4c9762e59353a592",
|
||||
"vip.mate.execution.evidence.service.JsonArtifactRecipe$Result" : "4aaa1cce1fc5a8159254c10ef80f0620feed4de3b8e919be4ecb3649143b5c92"
|
||||
},
|
||||
"executionMode" : "offline_platform_fixture_jsoncheck",
|
||||
"onlineModelCalls" : 0,
|
||||
"agentTaskSuccessRate" : "not_measured",
|
||||
"onlineCost" : "not_measured",
|
||||
"matchedCases" : 6,
|
||||
"mismatchedCases" : 0,
|
||||
"cases" : [ {
|
||||
"id" : "required-fields-present",
|
||||
"task" : "required fields present",
|
||||
"source" : "cycle011 JSON artifact check boundary",
|
||||
"expected" : {
|
||||
"status" : "MATCH",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : true,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"actual" : {
|
||||
"readStatus" : "READ",
|
||||
"status" : "MATCH",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : true,
|
||||
"recipeId" : "json-required-fields",
|
||||
"recipeRevision" : 1,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "missing-required-field",
|
||||
"task" : "missing required field",
|
||||
"source" : "cycle011 JSON artifact check boundary",
|
||||
"expected" : {
|
||||
"status" : "MISSING_FIELDS",
|
||||
"missingFields" : [ "report" ],
|
||||
"recipeInvoked" : true,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"actual" : {
|
||||
"readStatus" : "READ",
|
||||
"status" : "MISSING_FIELDS",
|
||||
"missingFields" : [ "report" ],
|
||||
"recipeInvoked" : true,
|
||||
"recipeId" : "json-required-fields",
|
||||
"recipeRevision" : 1,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "duplicate-json-key",
|
||||
"task" : "duplicate json key",
|
||||
"source" : "cycle011 JSON artifact check boundary",
|
||||
"expected" : {
|
||||
"status" : "INVALID_JSON",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : true,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"actual" : {
|
||||
"readStatus" : "READ",
|
||||
"status" : "INVALID_JSON",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : true,
|
||||
"recipeId" : "json-required-fields",
|
||||
"recipeRevision" : 1,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "changed-durable-file",
|
||||
"task" : "changed durable file",
|
||||
"source" : "cycle011 JSON artifact check boundary",
|
||||
"expected" : {
|
||||
"status" : "STALE",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : false,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"actual" : {
|
||||
"readStatus" : "STALE",
|
||||
"status" : "STALE",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : false,
|
||||
"recipeId" : "json-required-fields",
|
||||
"recipeRevision" : 1,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "insufficient-read-budget",
|
||||
"task" : "insufficient read budget",
|
||||
"source" : "cycle011 JSON artifact check boundary",
|
||||
"expected" : {
|
||||
"status" : "UNKNOWN",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : false,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"actual" : {
|
||||
"readStatus" : "UNKNOWN",
|
||||
"status" : "UNKNOWN",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : false,
|
||||
"recipeId" : "json-required-fields",
|
||||
"recipeRevision" : 1,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "foreign-artifact-owner",
|
||||
"task" : "foreign artifact owner",
|
||||
"source" : "cycle011 JSON artifact check boundary",
|
||||
"expected" : {
|
||||
"status" : "UNAVAILABLE",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : false,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"actual" : {
|
||||
"readStatus" : "UNAVAILABLE",
|
||||
"status" : "UNAVAILABLE",
|
||||
"missingFields" : [ ],
|
||||
"recipeInvoked" : false,
|
||||
"recipeId" : "json-required-fields",
|
||||
"recipeRevision" : 1,
|
||||
"acceptanceEligible" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
} ]
|
||||
}
|
||||
@ -0,0 +1,104 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"suiteId": "json-artifact-boundaries-v1",
|
||||
"tasks": [
|
||||
{
|
||||
"id": "required-fields-present",
|
||||
"task": "required fields present",
|
||||
"source": "cycle011 JSON artifact check boundary",
|
||||
"operation": "CHECK",
|
||||
"content": "{\"report\":false}",
|
||||
"requiredFields": [
|
||||
"report"
|
||||
],
|
||||
"expected": {
|
||||
"status": "MATCH",
|
||||
"missingFields": [],
|
||||
"recipeInvoked": true,
|
||||
"acceptanceEligible": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "missing-required-field",
|
||||
"task": "missing required field",
|
||||
"source": "cycle011 JSON artifact check boundary",
|
||||
"operation": "CHECK",
|
||||
"content": "{\"appendix\":1}",
|
||||
"requiredFields": [
|
||||
"report"
|
||||
],
|
||||
"expected": {
|
||||
"status": "MISSING_FIELDS",
|
||||
"missingFields": [
|
||||
"report"
|
||||
],
|
||||
"recipeInvoked": true,
|
||||
"acceptanceEligible": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "duplicate-json-key",
|
||||
"task": "duplicate json key",
|
||||
"source": "cycle011 JSON artifact check boundary",
|
||||
"operation": "CHECK",
|
||||
"content": "{\"report\":1,\"report\":2}",
|
||||
"requiredFields": [
|
||||
"report"
|
||||
],
|
||||
"expected": {
|
||||
"status": "INVALID_JSON",
|
||||
"missingFields": [],
|
||||
"recipeInvoked": true,
|
||||
"acceptanceEligible": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "changed-durable-file",
|
||||
"task": "changed durable file",
|
||||
"source": "cycle011 JSON artifact check boundary",
|
||||
"operation": "REWRITE_DISK",
|
||||
"content": "{\"report\":true}",
|
||||
"requiredFields": [
|
||||
"report"
|
||||
],
|
||||
"expected": {
|
||||
"status": "STALE",
|
||||
"missingFields": [],
|
||||
"recipeInvoked": false,
|
||||
"acceptanceEligible": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "insufficient-read-budget",
|
||||
"task": "insufficient read budget",
|
||||
"source": "cycle011 JSON artifact check boundary",
|
||||
"operation": "TOO_SMALL_BUDGET",
|
||||
"content": "{\"report\":true}",
|
||||
"requiredFields": [
|
||||
"report"
|
||||
],
|
||||
"expected": {
|
||||
"status": "UNKNOWN",
|
||||
"missingFields": [],
|
||||
"recipeInvoked": false,
|
||||
"acceptanceEligible": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "foreign-artifact-owner",
|
||||
"task": "foreign artifact owner",
|
||||
"source": "cycle011 JSON artifact check boundary",
|
||||
"operation": "FOREIGN_OWNER",
|
||||
"content": "{\"report\":true}",
|
||||
"requiredFields": [
|
||||
"report"
|
||||
],
|
||||
"expected": {
|
||||
"status": "UNAVAILABLE",
|
||||
"missingFields": [],
|
||||
"recipeInvoked": false,
|
||||
"acceptanceEligible": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user