diff --git a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplay.java b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplay.java index 188671aa..1d0d2342 100644 --- a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplay.java +++ b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplay.java @@ -12,6 +12,7 @@ import vip.mate.goal.config.GoalProperties; import vip.mate.goal.model.GoalCriteriaCodec; import vip.mate.goal.model.GoalCriterion; import vip.mate.goal.model.GoalEntity; +import vip.mate.goal.model.GoalEvaluationResult; import vip.mate.goal.service.GoalEvaluationService; import vip.mate.llm.chatmodel.ProviderChatModelFactory; import vip.mate.llm.model.ModelConfigEntity; @@ -24,6 +25,8 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.HexFormat; import java.util.List; +import java.util.Map; +import java.util.LinkedHashMap; import java.util.Set; import static org.mockito.ArgumentMatchers.any; @@ -47,7 +50,7 @@ final class OfflineGoalTaskReplay { record CaseResult(String id, String task, String source, String boundary, Expected expected, Actual actual, boolean matched, String error) { } record Report(int schemaVersion, String suiteId, String suiteSha256, String codeRevision, - String executionMode, int onlineModelCalls, String agentTaskSuccessRate, + String revisionSource, Map executedClassSha256, String executionMode, int onlineModelCalls, String agentTaskSuccessRate, String onlineCost, int matchedCases, int mismatchedCases, List cases) { } static Suite parse(byte[] bytes) throws IOException { @@ -103,7 +106,15 @@ final class OfflineGoalTaskReplay { } } int matched = (int) results.stream().filter(CaseResult::matched).count(); - return new Report(1, suite.suiteId(), digest(bytes), revision, MODE, 0, + Map classes = new LinkedHashMap<>(); + for (Class type : List.of(GoalEvaluationService.class, GoalCriteriaCodec.class, + GoalCriterion.class, GoalEvaluationResult.class)) { + try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) { + if (stream == null) throw new IOException("Missing tested class " + type.getName()); + classes.put(type.getName(), digest(stream.readAllBytes())); + } + } + return new Report(1, suite.suiteId(), digest(bytes), revision, "caller_supplied_label", classes, MODE, 0, "not_measured", "not_measured", matched, results.size() - matched, List.copyOf(results)); } diff --git a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplayTest.java b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplayTest.java index 1487b01f..2ad968c6 100644 --- a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplayTest.java +++ b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplayTest.java @@ -28,6 +28,20 @@ class OfflineGoalTaskReplayTest { assertEquals(0, report.mismatchedCases(), () -> "Replay mismatches: " + output.toAbsolutePath()); assertEquals(0, report.onlineModelCalls()); assertEquals("not_measured", report.agentTaskSuccessRate()); + var serialized = OfflineGoalTaskReplay.JSON.valueToTree(report); + assertEquals("caller_supplied_label", serialized.path("revisionSource").asText()); + var classes = serialized.path("executedClassSha256"); + assertEquals(4, classes.size()); + for (Class type : java.util.List.of(vip.mate.goal.service.GoalEvaluationService.class, + vip.mate.goal.model.GoalCriteriaCodec.class, vip.mate.goal.model.GoalCriterion.class, + vip.mate.goal.model.GoalEvaluationResult.class)) { + try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) { + assertNotNull(stream); + String actual = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256") + .digest(stream.readAllBytes())); + assertEquals(actual, classes.path(type.getName()).asText()); + } + } } @Test diff --git a/mateclaw-server/src/test/resources/agent-evaluation/README.md b/mateclaw-server/src/test/resources/agent-evaluation/README.md index d1ab5780..330618e9 100644 --- a/mateclaw-server/src/test/resources/agent-evaluation/README.md +++ b/mateclaw-server/src/test/resources/agent-evaluation/README.md @@ -21,8 +21,15 @@ The report is written to `mateclaw-server/target/agent-evaluation/goal-baseline. output paths when comparing versions. Without a revision argument the report says `unrecorded`; do not use that report for revision comparisons. Record a dirty working tree separately; a supplied Git revision identifies committed source, -not uncommitted modifications. The initial committed baseline's revision names -the production code from cycle-001; this new test harness was added afterward. +not uncommitted modifications. `revisionSource=caller_supplied_label` makes that +limitation explicit. Reports additionally include `executedClassSha256` for the +loaded GoalEvaluationService, GoalCriteriaCodec, GoalCriterion and +GoalEvaluationResult class bytes. Compare under the same compiler/build: these +hashes cover the named classes, not all dependencies, configuration or the OS. +The historical `baseline-v1.json` remains unchanged and predates these hashes; +its revision names production code from cycle-001. `baseline-v2.json` records a +later build of the same ten cases with class hashes. Its working-tree label is +explicit; it is not a new suite or ten additional Agent executions. Schema v1 has `schemaVersion`, `suiteId`, and 1–100 `tasks`. Each task declares: @@ -46,7 +53,8 @@ inputs, not generated by copying the current runtime result. For a deliberate behavior change, review and explain each changed expectation. Reports bind the exact suite bytes with SHA-256 and record the supplied source -revision, execution mode, every expected/actual result, and mismatches. A mismatch +revision label and its source, executed class hashes, execution mode, every +expected/actual result, and mismatches. A mismatch writes the report then fails the test/Maven command; all cases are still run. Invalid suites fail before a new report is written, so an existing report may be stale: always check the command exit status. `matchedCases` counts policy replay diff --git a/mateclaw-server/src/test/resources/agent-evaluation/baseline-v2.json b/mateclaw-server/src/test/resources/agent-evaluation/baseline-v2.json new file mode 100644 index 00000000..6621fcca --- /dev/null +++ b/mateclaw-server/src/test/resources/agent-evaluation/baseline-v2.json @@ -0,0 +1,230 @@ +{ + "schemaVersion" : 1, + "suiteId" : "goal-boundaries-v1", + "suiteSha256" : "3399b3a40ea1f8ad2ac4307b85f4e045c3dba0b87d7474c9367486b3d6b6f78d", + "codeRevision" : "612231ce2+cycle023-working", + "revisionSource" : "caller_supplied_label", + "executedClassSha256" : { + "vip.mate.goal.service.GoalEvaluationService" : "ac21f8db1f509b55da93cbb17ef61d3c11a0da82f2f9cc127bdee02fef5682a8", + "vip.mate.goal.model.GoalCriteriaCodec" : "6963dbb3a89736cc0e75ebd40d68d3df7b6537d2e35a946e39f1dad7a9345d7b", + "vip.mate.goal.model.GoalCriterion" : "51e90826a90dd5d7cc7f1e5aefa7a7eecbdfea82f173b5c0a0a583134fbf96b7", + "vip.mate.goal.model.GoalEvaluationResult" : "bc150a255e8503a8760b5ff323562da4c2f93ba11c9ec17df65334a18f725d87" + }, + "executionMode" : "offline_synthetic_evaluator_replay", + "onlineModelCalls" : 0, + "agentTaskSuccessRate" : "not_measured", + "onlineCost" : "not_measured", + "matchedCases" : 10, + "mismatchedCases" : 0, + "cases" : [ { + "id" : "empty-evidence", + "task" : "Deliver a report and verify its contents", + "source" : "cycle-001 reproduced failure; GoalCriteriaCodecTest.blankEvidenceCannotPassNewOrPersistedCriteria", + "boundary" : "Claimed pass without evidence is rejected", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1", "C2" ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1", "C2" ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "inherited-empty-pass", + "task" : "Finish a report after resuming a persisted checklist", + "source" : "cycle-001 persisted blank-evidence boundary", + "boundary" : "An omitted historical pass with blank evidence cannot complete", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "cumulative-progress", + "task" : "Verify a report created in a previous turn", + "source" : "GoalEvaluationServiceTest", + "boundary" : "Previously evidenced criterion survives omitted delta", + "expected" : { + "completed" : true, + "score" : 1.0, + "decision" : "completed", + "remainingIds" : [ ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : true, + "score" : 1.0, + "decision" : "completed", + "remainingIds" : [ ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "unknown-criterion", + "task" : "Create report.md with a fixed checklist", + "source" : "GoalCriteriaCodecTest.merge_unknownVerdictId_isIgnored", + "boundary" : "A model inventing C99 cannot pass C1", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "contradicted-prior", + "task" : "Recheck a previously created report", + "source" : "GoalEvaluationServiceTest", + "boundary" : "Explicit contradiction revokes the prior pass", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "bootstrap-only", + "task" : "Create a report with no checklist yet", + "source" : "GoalEvaluationServiceTest", + "boundary" : "Bootstrap defines criteria and cannot complete", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "continue", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "malformed-response", + "task" : "Verify the report after evaluator output corruption", + "source" : "GoalEvaluationServiceTest", + "boundary" : "Malformed model output degrades to fallback", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "fallback", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "fallback", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "valid-semantic-pass", + "task" : "Create report and verify a supplied excerpt", + "source" : "GoalEvaluationServiceTest", + "boundary" : "Nonblank semantic evidence retains legacy completion", + "expected" : { + "completed" : true, + "score" : 1.0, + "decision" : "completed", + "remainingIds" : [ ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : true, + "score" : 1.0, + "decision" : "completed", + "remainingIds" : [ ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + }, { + "id" : "empty-terminal-answer", + "task" : "Continue report task without a final answer", + "source" : "GoalEvaluationServiceTest", + "boundary" : "No answer means no evaluator fixture call", + "expected" : { + "completed" : false, + "score" : 0.0, + "decision" : "fallback", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 0 + }, + "actual" : { + "completed" : false, + "score" : 0.0, + "decision" : "fallback", + "remainingIds" : [ "C1" ], + "fixtureCalls" : 0 + }, + "matched" : true, + "error" : null + }, { + "id" : "forged-text-is-not-strong-acceptance", + "task" : "Deliver a file that exists in the workspace", + "source" : "RFC-096 text/Observe trust boundary; synthetic limitation probe, not observed model behavior", + "boundary" : "Known limitation: fabricated nonblank evidence still passes semantic policy; no file execution is performed", + "expected" : { + "completed" : true, + "score" : 1.0, + "decision" : "completed", + "remainingIds" : [ ], + "fixtureCalls" : 1 + }, + "actual" : { + "completed" : true, + "score" : 1.0, + "decision" : "completed", + "remainingIds" : [ ], + "fixtureCalls" : 1 + }, + "matched" : true, + "error" : null + } ] +} \ No newline at end of file