opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-14 03:33:43 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

test(evaluation): record compiled provenance in goal replay reports

Browse Source
This commit is contained in:
mateaix 2026-09-14 00:30:49 +08:00
parent 02f31fdaf6
commit ebe4981476
4 changed files with 268 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplay.java
View File

@ -12,6 +12,7 @@ import vip.mate.goal.config.GoalProperties;
import vip.mate.goal.model.GoalCriteriaCodec;
import vip.mate.goal.model.GoalCriterion;
import vip.mate.goal.model.GoalEntity;
import vip.mate.goal.model.GoalEvaluationResult;
import vip.mate.goal.service.GoalEvaluationService;
import vip.mate.llm.chatmodel.ProviderChatModelFactory;
import vip.mate.llm.model.ModelConfigEntity;
@ -24,6 +25,8 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.LinkedHashMap;
import java.util.Set;
import static org.mockito.ArgumentMatchers.any;
@ -47,7 +50,7 @@ final class OfflineGoalTaskReplay {
record CaseResult(String id, String task, String source, String boundary, Expected expected,
Actual actual, boolean matched, String error) { }
record Report(int schemaVersion, String suiteId, String suiteSha256, String codeRevision,
String executionMode, int onlineModelCalls, String agentTaskSuccessRate,
String revisionSource, Map<String, String> executedClassSha256, String executionMode, int onlineModelCalls, String agentTaskSuccessRate,
String onlineCost, int matchedCases, int mismatchedCases, List<CaseResult> cases) { }
static Suite parse(byte[] bytes) throws IOException {
@ -103,7 +106,15 @@ final class OfflineGoalTaskReplay {
}
}
int matched = (int) results.stream().filter(CaseResult::matched).count();
return new Report(1, suite.suiteId(), digest(bytes), revision, MODE, 0,
Map<String, String> classes = new LinkedHashMap<>();
for (Class<?> type : List.of(GoalEvaluationService.class, GoalCriteriaCodec.class,
GoalCriterion.class, GoalEvaluationResult.class)) {
try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) {
if (stream == null) throw new IOException("Missing tested class " + type.getName());
classes.put(type.getName(), digest(stream.readAllBytes()));
}
}
return new Report(1, suite.suiteId(), digest(bytes), revision, "caller_supplied_label", classes, MODE, 0,
"not_measured", "not_measured", matched, results.size() - matched, List.copyOf(results));
}

14
mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalTaskReplayTest.java
View File

@ -28,6 +28,20 @@ class OfflineGoalTaskReplayTest {
assertEquals(0, report.mismatchedCases(), () -> "Replay mismatches: " + output.toAbsolutePath());
assertEquals(0, report.onlineModelCalls());
assertEquals("not_measured", report.agentTaskSuccessRate());
var serialized = OfflineGoalTaskReplay.JSON.valueToTree(report);
assertEquals("caller_supplied_label", serialized.path("revisionSource").asText());
var classes = serialized.path("executedClassSha256");
assertEquals(4, classes.size());
for (Class<?> type : java.util.List.of(vip.mate.goal.service.GoalEvaluationService.class,
vip.mate.goal.model.GoalCriteriaCodec.class, vip.mate.goal.model.GoalCriterion.class,
vip.mate.goal.model.GoalEvaluationResult.class)) {
try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) {
assertNotNull(stream);
String actual = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256")
.digest(stream.readAllBytes()));
assertEquals(actual, classes.path(type.getName()).asText());
}
}
}
@Test

14
mateclaw-server/src/test/resources/agent-evaluation/README.md
View File

@ -21,8 +21,15 @@ The report is written to `mateclaw-server/target/agent-evaluation/goal-baseline.
output paths when comparing versions. Without a revision argument the report
says `unrecorded`; do not use that report for revision comparisons. Record a dirty
working tree separately; a supplied Git revision identifies committed source,
not uncommitted modifications. The initial committed baseline's revision names
the production code from cycle-001; this new test harness was added afterward.
not uncommitted modifications. `revisionSource=caller_supplied_label` makes that
limitation explicit. Reports additionally include `executedClassSha256` for the
loaded GoalEvaluationService, GoalCriteriaCodec, GoalCriterion and
GoalEvaluationResult class bytes. Compare under the same compiler/build: these
hashes cover the named classes, not all dependencies, configuration or the OS.
The historical `baseline-v1.json` remains unchanged and predates these hashes;
its revision names production code from cycle-001. `baseline-v2.json` records a
later build of the same ten cases with class hashes. Its working-tree label is
explicit; it is not a new suite or ten additional Agent executions.
Schema v1 has `schemaVersion`, `suiteId`, and 1–100 `tasks`. Each task declares:
@ -46,7 +53,8 @@ inputs, not generated by copying the current runtime result. For a deliberate
behavior change, review and explain each changed expectation.
Reports bind the exact suite bytes with SHA-256 and record the supplied source
revision, execution mode, every expected/actual result, and mismatches. A mismatch
revision label and its source, executed class hashes, execution mode, every
expected/actual result, and mismatches. A mismatch
writes the report then fails the test/Maven command; all cases are still run.
Invalid suites fail before a new report is written, so an existing report may be
stale: always check the command exit status. `matchedCases` counts policy replay

230
mateclaw-server/src/test/resources/agent-evaluation/baseline-v2.json Normal file
View File

@ -0,0 +1,230 @@
{
"schemaVersion" : 1,
"suiteId" : "goal-boundaries-v1",
"suiteSha256" : "3399b3a40ea1f8ad2ac4307b85f4e045c3dba0b87d7474c9367486b3d6b6f78d",
"codeRevision" : "612231ce2+cycle023-working",
"revisionSource" : "caller_supplied_label",
"executedClassSha256" : {
"vip.mate.goal.service.GoalEvaluationService" : "ac21f8db1f509b55da93cbb17ef61d3c11a0da82f2f9cc127bdee02fef5682a8",
"vip.mate.goal.model.GoalCriteriaCodec" : "6963dbb3a89736cc0e75ebd40d68d3df7b6537d2e35a946e39f1dad7a9345d7b",
"vip.mate.goal.model.GoalCriterion" : "51e90826a90dd5d7cc7f1e5aefa7a7eecbdfea82f173b5c0a0a583134fbf96b7",
"vip.mate.goal.model.GoalEvaluationResult" : "bc150a255e8503a8760b5ff323562da4c2f93ba11c9ec17df65334a18f725d87"
},
"executionMode" : "offline_synthetic_evaluator_replay",
"onlineModelCalls" : 0,
"agentTaskSuccessRate" : "not_measured",
"onlineCost" : "not_measured",
"matchedCases" : 10,
"mismatchedCases" : 0,
"cases" : [ {
"id" : "empty-evidence",
"task" : "Deliver a report and verify its contents",
"source" : "cycle-001 reproduced failure; GoalCriteriaCodecTest.blankEvidenceCannotPassNewOrPersistedCriteria",
"boundary" : "Claimed pass without evidence is rejected",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1", "C2" ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1", "C2" ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "inherited-empty-pass",
"task" : "Finish a report after resuming a persisted checklist",
"source" : "cycle-001 persisted blank-evidence boundary",
"boundary" : "An omitted historical pass with blank evidence cannot complete",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "cumulative-progress",
"task" : "Verify a report created in a previous turn",
"source" : "GoalEvaluationServiceTest",
"boundary" : "Previously evidenced criterion survives omitted delta",
"expected" : {
"completed" : true,
"score" : 1.0,
"decision" : "completed",
"remainingIds" : [ ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : true,
"score" : 1.0,
"decision" : "completed",
"remainingIds" : [ ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "unknown-criterion",
"task" : "Create report.md with a fixed checklist",
"source" : "GoalCriteriaCodecTest.merge_unknownVerdictId_isIgnored",
"boundary" : "A model inventing C99 cannot pass C1",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "contradicted-prior",
"task" : "Recheck a previously created report",
"source" : "GoalEvaluationServiceTest",
"boundary" : "Explicit contradiction revokes the prior pass",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "bootstrap-only",
"task" : "Create a report with no checklist yet",
"source" : "GoalEvaluationServiceTest",
"boundary" : "Bootstrap defines criteria and cannot complete",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "continue",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "malformed-response",
"task" : "Verify the report after evaluator output corruption",
"source" : "GoalEvaluationServiceTest",
"boundary" : "Malformed model output degrades to fallback",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "fallback",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "fallback",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "valid-semantic-pass",
"task" : "Create report and verify a supplied excerpt",
"source" : "GoalEvaluationServiceTest",
"boundary" : "Nonblank semantic evidence retains legacy completion",
"expected" : {
"completed" : true,
"score" : 1.0,
"decision" : "completed",
"remainingIds" : [ ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : true,
"score" : 1.0,
"decision" : "completed",
"remainingIds" : [ ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
}, {
"id" : "empty-terminal-answer",
"task" : "Continue report task without a final answer",
"source" : "GoalEvaluationServiceTest",
"boundary" : "No answer means no evaluator fixture call",
"expected" : {
"completed" : false,
"score" : 0.0,
"decision" : "fallback",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 0
},
"actual" : {
"completed" : false,
"score" : 0.0,
"decision" : "fallback",
"remainingIds" : [ "C1" ],
"fixtureCalls" : 0
},
"matched" : true,
"error" : null
}, {
"id" : "forged-text-is-not-strong-acceptance",
"task" : "Deliver a file that exists in the workspace",
"source" : "RFC-096 text/Observe trust boundary; synthetic limitation probe, not observed model behavior",
"boundary" : "Known limitation: fabricated nonblank evidence still passes semantic policy; no file execution is performed",
"expected" : {
"completed" : true,
"score" : 1.0,
"decision" : "completed",
"remainingIds" : [ ],
"fixtureCalls" : 1
},
"actual" : {
"completed" : true,
"score" : 1.0,
"decision" : "completed",
"remainingIds" : [ ],
"fixtureCalls" : 1
},
"matched" : true,
"error" : null
} ]
}