diff --git a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplay.java b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplay.java new file mode 100644 index 00000000..6a0f5a74 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplay.java @@ -0,0 +1,171 @@ +package vip.mate.evaluation; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.jdbc.core.JdbcTemplate; +import vip.mate.exception.MateClawException; +import vip.mate.goal.model.*; +import vip.mate.goal.service.GoalService; +import vip.mate.goal.service.GoalServiceImpl; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.*; + +/** Fixed service transitions on a real test database; evaluation results are fixtures, not model calls. */ +final class OfflineGoalServiceTaskReplay { + static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS) + .enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS); + enum Operation { CURRENT_REVISION_COMPLETION, APPEND_BEFORE_VERDICT, PAUSE_BEFORE_VERDICT, + APPEND_BEFORE_BOOTSTRAP, REPLACE_DEFINITION, ABA_DEFINITION } + record Suite(Integer schemaVersion, String suiteId, List tasks) { } + record Task(String id, String task, String source, Operation operation, Expected expected) { } + record Expected(GoalStatus status, Double score, Integer criteriaCount, Integer passedCount, + String firstCriterionText, Long evaluationRevision, Integer recordedEvalCallsUsed, + Integer completionCode) { } + record Actual(GoalStatus status, Double score, int criteriaCount, int passedCount, + String firstCriterionText, long evaluationRevision, int recordedEvalCallsUsed, int completionCode) { } + record Case(String id, String task, String source, Expected expected, Actual actual, boolean matched, String error) { } + record Report(int schemaVersion, String suiteId, String suiteSha256, String revisionLabel, + Map productionClassSha256, String databaseProduct, String latestMigration, + List mockedBoundaries, String executionMode, int onlineModelCalls, String agentTaskSuccessRate, String onlineCost, + int matchedCases, int mismatchedCases, List cases) { } + + static Suite parse(byte[] bytes) throws IOException { + Suite suite = JSON.readValue(bytes, Suite.class); + require(suite != null && Integer.valueOf(1).equals(suite.schemaVersion()), "schemaVersion must be 1"); + require(nonblank(suite.suiteId()) && suite.tasks() != null && !suite.tasks().isEmpty() + && suite.tasks().size() <= 100, "suiteId and 1..100 tasks required"); + Set ids = new HashSet<>(); + for (Task task : suite.tasks()) { + require(task != null && nonblank(task.id()) && ids.add(task.id()) && nonblank(task.task()) + && nonblank(task.source()) && task.operation() != null, "valid unique tasks required"); + Expected e = task.expected(); + require(e != null && e.status() != null && e.score() != null && Double.isFinite(e.score()) + && e.score() >= 0 && e.score() <= 1 && e.criteriaCount() != null && e.criteriaCount() >= 0 + && e.passedCount() != null && e.passedCount() >= 0 && e.passedCount() <= e.criteriaCount() + && e.evaluationRevision() != null && e.evaluationRevision() >= 0 + && e.recordedEvalCallsUsed() != null && e.recordedEvalCallsUsed() >= 0 + && e.completionCode() != null && Set.of(0, 200, 409).contains(e.completionCode()) + && (e.criteriaCount() > 0 ? nonblank(e.firstCriterionText()) : e.firstCriterionText() == null), + "all state expectations must be explicit and valid"); + } + return suite; + } + + static Report run(byte[] bytes, GoalService goals, JdbcTemplate jdbc, String revisionLabel) throws Exception { + Suite suite = parse(bytes); // Entire input checked before any service writes. + require(nonblank(revisionLabel), "revisionLabel required"); + List results = new ArrayList<>(); + for (Task task : suite.tasks()) { + try { + Actual a = execute(task.operation(), goals); + Expected e = task.expected(); + boolean matched = e.status() == a.status() && Objects.equals(e.score(), a.score()) + && e.criteriaCount() == a.criteriaCount() && e.passedCount() == a.passedCount() + && Objects.equals(e.firstCriterionText(), a.firstCriterionText()) + && e.evaluationRevision() == a.evaluationRevision() + && e.recordedEvalCallsUsed() == a.recordedEvalCallsUsed() && e.completionCode() == a.completionCode(); + results.add(new Case(task.id(), task.task(), task.source(), e, a, matched, null)); + } catch (RuntimeException error) { + results.add(new Case(task.id(), task.task(), task.source(), task.expected(), null, false, + error.getClass().getSimpleName())); + } + } + Map classes = new LinkedHashMap<>(); + for (Class type : List.of(GoalServiceImpl.class, GoalCriteriaCodec.class, GoalEntity.class, GoalEvaluationResult.class)) { + try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) { + if (stream == null) throw new IOException("Missing tested class " + type.getName()); + classes.put(type.getName(), digest(stream.readAllBytes())); + } + } + String database; + try (var connection = Objects.requireNonNull(jdbc.getDataSource()).getConnection()) { + database = connection.getMetaData().getDatabaseProductName(); + } + String migration = jdbc.queryForObject( + "SELECT version FROM flyway_schema_history WHERE success=TRUE ORDER BY installed_rank DESC LIMIT 1", String.class); + int matched = (int) results.stream().filter(Case::matched).count(); + return new Report(1, suite.suiteId(), digest(bytes), revisionLabel, classes, database, migration, List.of("MemoryManager"), + "offline_h2_goal_service_scenarios", 0, "not_measured", "not_measured", matched, + results.size() - matched, List.copyOf(results)); + } + + private static Actual execute(Operation operation, GoalService goals) { + var request = new GoalCreateRequest(); + request.setConversationId("service-fixture-" + UUID.randomUUID()); + request.setAgentId(1L); request.setWorkspaceId(1L); request.setTitle("Service fixture"); + request.setAutoFollowupEnabled(false); + request.setPersistentExecution(operation == Operation.PAUSE_BEFORE_VERDICT); + if (operation != Operation.APPEND_BEFORE_BOOTSTRAP && operation != Operation.ABA_DEFINITION) { + request.setCriteria(List.of(new GoalCriterion("C1", "report", false, ""))); + } + if (operation == Operation.ABA_DEFINITION) request.setExitCriteria("A"); + Long id = goals.create(request, "fixture-owner").getId(); + var passed = verdict(true, 0); + boolean attemptCompletion = true; + switch (operation) { + case CURRENT_REVISION_COMPLETION -> { + var edit = new GoalUpdateRequest(); edit.setDescription("revised context"); + goals.update(id, edit, "fixture-owner"); + passed = verdict(true, 1); + goals.recordEvaluation(id, passed, 0, 1); + } + case APPEND_BEFORE_VERDICT -> { + goals.appendCriterion(id, "appendix", "fixture-owner"); + goals.recordEvaluation(id, passed, 0, 1); + } + case PAUSE_BEFORE_VERDICT -> { + goals.recordEvaluation(id, verdict(false, 0), 0, 1); + goals.pause(id, "fixture-owner"); + goals.recordEvaluation(id, passed, 0, 1); + } + case APPEND_BEFORE_BOOTSTRAP -> { + goals.appendCriterion(id, "user appendix", "fixture-owner"); + goals.recordEvaluation(id, draft(), 0, 1); + attemptCompletion = false; + } + case REPLACE_DEFINITION -> { + goals.recordEvaluation(id, passed, 0, 1); + var edit = new GoalUpdateRequest(); edit.setExitCriteria("new requirement"); + goals.update(id, edit, "fixture-owner"); + goals.recordEvaluation(id, passed, 0, 1); + } + case ABA_DEFINITION -> { + var edit = new GoalUpdateRequest(); edit.setExitCriteria("B"); goals.update(id, edit, "fixture-owner"); + edit.setExitCriteria("A"); goals.update(id, edit, "fixture-owner"); + goals.recordEvaluation(id, draft(), 0, 1); + attemptCompletion = false; + } + } + int completionCode = 0; + if (attemptCompletion) { + try { goals.markEvaluatedCompleted(id, passed); completionCode = 200; } + catch (MateClawException denied) { completionCode = denied.getCode(); } + } + GoalEntity saved = goals.getById(id); + List criteria = GoalCriteriaCodec.parse(saved.getCriteria(), JSON); + return new Actual(saved.getStatus(), saved.getCompletionScore(), criteria.size(), + (int) criteria.stream().filter(GoalCriterion::passed).count(), + criteria.isEmpty() ? null : criteria.getFirst().text(), saved.getEvaluationRevision(), + saved.getEvalLlmCallsUsed(), completionCode); + } + + private static GoalEvaluationResult verdict(boolean passed, long revision) { + return new GoalEvaluationResult(passed ? 1.0 : 0.0, passed ? "" : "missing report", + passed ? "completed" : "continue", passed, "service-fixture", 1, 0, + List.of(new GoalChecklistVerdict.CriterionVerdict("C1", passed, passed ? "fixture report evidence" : "")), + null).withEvaluationRevision(revision); + } + private static GoalEvaluationResult draft() { + return new GoalEvaluationResult(0, "draft", "continue", false, "service-fixture", 1, 0, + List.of(), List.of(new GoalCriterion("C1", "model draft", false, ""))); + } + private static String digest(byte[] bytes) { + try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); } + catch (NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); } + } + private static boolean nonblank(String value) { return value != null && !value.isBlank(); } + private static void require(boolean condition, String message) { if (!condition) throw new IllegalArgumentException(message); } +} diff --git a/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplayTest.java b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplayTest.java new file mode 100644 index 00000000..d342284e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/evaluation/OfflineGoalServiceTaskReplayTest.java @@ -0,0 +1,62 @@ +package vip.mate.evaluation; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import vip.mate.memory.spi.MemoryManager; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.goal.service.GoalService; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest(classes = MateClawApplication.class, webEnvironment = SpringBootTest.WebEnvironment.NONE) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:goal_replay_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", "spring.main.web-application-type=none", "mateclaw.goal.enabled=false", + "mateclaw.plugin.enabled=false", "mateclaw.skill.workspace.auto-init=false", + "mateclaw.skill.workspace.root=${java.io.tmpdir}/mateclaw-goal-replay-skills-${random.uuid}" +}) +class OfflineGoalServiceTaskReplayTest { + @MockBean MemoryManager memory; + @Autowired GoalService goals; + @Autowired JdbcTemplate jdbc; + private byte[] fixture() throws Exception { + try (var stream = getClass().getResourceAsStream("/agent-evaluation/goal-service-boundaries-v1.json")) { + assertNotNull(stream); return stream.readAllBytes(); + } + } + @Test void executeServiceTasksAndWriteReport() throws Exception { + String input = System.getProperty("goal.service.eval.suite"); + var report = OfflineGoalServiceTaskReplay.run(input == null ? fixture() : Files.readAllBytes(Path.of(input)), + goals, jdbc, System.getProperty("goal.service.eval.revision", "unrecorded")); + Path output = Path.of(System.getProperty("goal.service.eval.report", "target/agent-evaluation/goal-service-baseline.json")); + Files.createDirectories(output.toAbsolutePath().getParent()); + OfflineGoalServiceTaskReplay.JSON.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), report); + assertEquals(0, report.mismatchedCases(), () -> "Goal service mismatches: " + output.toAbsolutePath()); + assertEquals("H2", report.databaseProduct()); + assertTrue(org.mockito.Mockito.mockingDetails(memory).isMock()); + assertNotNull(report.latestMigration()); + assertEquals(0, report.onlineModelCalls()); + } + @Test void invalidLaterTaskIsRejectedBeforeDatabaseWrites() throws Exception { + ObjectNode suite = (ObjectNode) OfflineGoalServiceTaskReplay.JSON.readTree(fixture()); + ((ObjectNode) suite.withArray("tasks").get(1).get("expected")).remove("completionCode"); + Long before = jdbc.queryForObject("SELECT COUNT(*) FROM mate_agent_goal", Long.class); + assertThrows(IllegalArgumentException.class, () -> OfflineGoalServiceTaskReplay.run( + suite.toString().getBytes(StandardCharsets.UTF_8), goals, jdbc, "test")); + assertEquals(before, jdbc.queryForObject("SELECT COUNT(*) FROM mate_agent_goal", Long.class)); + } + @Test void wrongExpectationReportsMismatchAndRunsAllCases() throws Exception { + ObjectNode suite = (ObjectNode) OfflineGoalServiceTaskReplay.JSON.readTree(fixture()); + ((ObjectNode) suite.withArray("tasks").get(1).get("expected")).put("completionCode", 200); + var report = OfflineGoalServiceTaskReplay.run(suite.toString().getBytes(StandardCharsets.UTF_8), goals, jdbc, "test"); + assertEquals(1, report.mismatchedCases()); + assertEquals(suite.withArray("tasks").size(), report.cases().size()); + } +} diff --git a/mateclaw-server/src/test/resources/agent-evaluation/README.md b/mateclaw-server/src/test/resources/agent-evaluation/README.md index 7eb609ae..d1ab5780 100644 --- a/mateclaw-server/src/test/resources/agent-evaluation/README.md +++ b/mateclaw-server/src/test/resources/agent-evaluation/README.md @@ -151,3 +151,47 @@ files and the production recipe; it does not exercise HTTP authorization or a browser. Those boundaries have separate service/UI regressions. Its mode is `offline_platform_fixture_jsoncheck`; online calls are zero and Agent success rate/online cost remain `not_measured`. + +## H2 Goal service scenarios + +`goal-service-boundaries-v1.json` adds six service workflows from the real +completion/append/pause/bootstrap/definition-edit regressions. Run: + +```sh +mvn -pl mateclaw-server -am -Dtest=OfflineGoalServiceTaskReplayTest \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dgoal.service.eval.revision="$(git rev-parse HEAD)" test +``` + +Output: `mateclaw-server/target/agent-evaluation/goal-service-baseline.json`. +`goal.service.eval.suite` and `goal.service.eval.report` accept absolute paths. +The runner uses the actual Spring transactional Goal service and a fresh H2 +MySQL-compatibility database with Flyway migrations. This is **not** a real +MySQL/Kingbase run. It disables the autonomous Goal scheduler and supplies +fixed evaluator results; no task-solving Agent or model is invoked. The external +`MemoryManager` boundary is explicitly mocked so completion cannot sync to +configured memory providers. Plugin loading is disabled and skill workspace +reads use a temporary root; the Goal service, repositories and H2 remain real. +The report lists this mocked boundary, rather than calling the entire app real. + +Expected fields describe persisted status, score, criterion counts and first +text, evaluation-definition revision, recorded evaluator usage and completion +outcome. Completion code `0` means not attempted, `200` means the existing +semantic completion service accepted the fixture, and `409` means it rejected +the transition. A semantic completion is not execution-required acceptance. +`recordedEvalCallsUsed` exercises bookkeeping with fixture deltas; it is not a +count of online calls. The entire suite is validated before any service write; +conversations use generated IDs and fixture input cannot supply SQL or paths. + +Reports include suite and named production class hashes, the actual database +product and latest applied Flyway version. These identify selected build/schema +facts, not the entire runtime. Mismatches preserve all case results and fail the +CLI. `goal-service-baseline-v1.json` is the initial six-case baseline. + +The four suites now contain **30 fixed scenarios**: ten synthetic evaluator +responses, eight platform artifact IO scenarios, six JSON artifact checks and +six H2 Goal service workflows. They are different execution modes and include +related regression boundaries; do not treat them as 30 independent online +Agent task attempts. Online calls remain zero, and online cost/Agent success +rate remain `not_measured`. HTTP authorization, browser flows and distributed +owner/scope fences are not exercised by the H2 replay. diff --git a/mateclaw-server/src/test/resources/agent-evaluation/goal-service-baseline-v1.json b/mateclaw-server/src/test/resources/agent-evaluation/goal-service-baseline-v1.json new file mode 100644 index 00000000..726bbc11 --- /dev/null +++ b/mateclaw-server/src/test/resources/agent-evaluation/goal-service-baseline-v1.json @@ -0,0 +1,178 @@ +{ + "schemaVersion" : 1, + "suiteId" : "goal-service-boundaries-v1", + "suiteSha256" : "ca38905298a2b8cf1379b14f04b4a4e52b759504d8aa2314c52e6a37eb3cdaa6", + "revisionLabel" : "b166793d1+cycle015-working", + "productionClassSha256" : { + "vip.mate.goal.service.GoalServiceImpl" : "7151a5e4a8918f16f8e17d9c9fa478086d15b24a1d705cb87811505be40c5387", + "vip.mate.goal.model.GoalCriteriaCodec" : "6963dbb3a89736cc0e75ebd40d68d3df7b6537d2e35a946e39f1dad7a9345d7b", + "vip.mate.goal.model.GoalEntity" : "0ba945500694f12bce2ac302927e8d468b303a36f96c78be2a07143539a07163", + "vip.mate.goal.model.GoalEvaluationResult" : "bc150a255e8503a8760b5ff323562da4c2f93ba11c9ec17df65334a18f725d87" + }, + "databaseProduct" : "H2", + "latestMigration" : "193", + "mockedBoundaries" : [ "MemoryManager" ], + "executionMode" : "offline_h2_goal_service_scenarios", + "onlineModelCalls" : 0, + "agentTaskSuccessRate" : "not_measured", + "onlineCost" : "not_measured", + "matchedCases" : 6, + "mismatchedCases" : 0, + "cases" : [ { + "id" : "current-revision-completion", + "task" : "current revision completion", + "source" : "cycles 004/006/010/013 real service boundaries", + "expected" : { + "status" : "completed", + "score" : 1.0, + "criteriaCount" : 1, + "passedCount" : 1, + "firstCriterionText" : "report", + "evaluationRevision" : 1, + "recordedEvalCallsUsed" : 1, + "completionCode" : 200 + }, + "actual" : { + "status" : "completed", + "score" : 1.0, + "criteriaCount" : 1, + "passedCount" : 1, + "firstCriterionText" : "report", + "evaluationRevision" : 1, + "recordedEvalCallsUsed" : 1, + "completionCode" : 200 + }, + "matched" : true, + "error" : null + }, { + "id" : "append-before-verdict", + "task" : "append before verdict", + "source" : "cycles 004/006/010/013 real service boundaries", + "expected" : { + "status" : "active", + "score" : 0.5, + "criteriaCount" : 2, + "passedCount" : 1, + "firstCriterionText" : "report", + "evaluationRevision" : 0, + "recordedEvalCallsUsed" : 1, + "completionCode" : 409 + }, + "actual" : { + "status" : "active", + "score" : 0.5, + "criteriaCount" : 2, + "passedCount" : 1, + "firstCriterionText" : "report", + "evaluationRevision" : 0, + "recordedEvalCallsUsed" : 1, + "completionCode" : 409 + }, + "matched" : true, + "error" : null + }, { + "id" : "pause-before-verdict", + "task" : "pause before verdict", + "source" : "cycles 004/006/010/013 real service boundaries", + "expected" : { + "status" : "paused", + "score" : 0.0, + "criteriaCount" : 1, + "passedCount" : 0, + "firstCriterionText" : "report", + "evaluationRevision" : 0, + "recordedEvalCallsUsed" : 2, + "completionCode" : 409 + }, + "actual" : { + "status" : "paused", + "score" : 0.0, + "criteriaCount" : 1, + "passedCount" : 0, + "firstCriterionText" : "report", + "evaluationRevision" : 0, + "recordedEvalCallsUsed" : 2, + "completionCode" : 409 + }, + "matched" : true, + "error" : null + }, { + "id" : "append-before-bootstrap", + "task" : "append before bootstrap", + "source" : "cycles 004/006/010/013 real service boundaries", + "expected" : { + "status" : "active", + "score" : 0.0, + "criteriaCount" : 1, + "passedCount" : 0, + "firstCriterionText" : "user appendix", + "evaluationRevision" : 0, + "recordedEvalCallsUsed" : 1, + "completionCode" : 0 + }, + "actual" : { + "status" : "active", + "score" : 0.0, + "criteriaCount" : 1, + "passedCount" : 0, + "firstCriterionText" : "user appendix", + "evaluationRevision" : 0, + "recordedEvalCallsUsed" : 1, + "completionCode" : 0 + }, + "matched" : true, + "error" : null + }, { + "id" : "replace-definition", + "task" : "replace definition", + "source" : "cycles 004/006/010/013 real service boundaries", + "expected" : { + "status" : "active", + "score" : 0.0, + "criteriaCount" : 0, + "passedCount" : 0, + "firstCriterionText" : null, + "evaluationRevision" : 1, + "recordedEvalCallsUsed" : 2, + "completionCode" : 409 + }, + "actual" : { + "status" : "active", + "score" : 0.0, + "criteriaCount" : 0, + "passedCount" : 0, + "firstCriterionText" : null, + "evaluationRevision" : 1, + "recordedEvalCallsUsed" : 2, + "completionCode" : 409 + }, + "matched" : true, + "error" : null + }, { + "id" : "aba-definition", + "task" : "aba definition", + "source" : "cycles 004/006/010/013 real service boundaries", + "expected" : { + "status" : "active", + "score" : 0.0, + "criteriaCount" : 0, + "passedCount" : 0, + "firstCriterionText" : null, + "evaluationRevision" : 2, + "recordedEvalCallsUsed" : 1, + "completionCode" : 0 + }, + "actual" : { + "status" : "active", + "score" : 0.0, + "criteriaCount" : 0, + "passedCount" : 0, + "firstCriterionText" : null, + "evaluationRevision" : 2, + "recordedEvalCallsUsed" : 1, + "completionCode" : 0 + }, + "matched" : true, + "error" : null + } ] +} \ No newline at end of file diff --git a/mateclaw-server/src/test/resources/agent-evaluation/goal-service-boundaries-v1.json b/mateclaw-server/src/test/resources/agent-evaluation/goal-service-boundaries-v1.json new file mode 100644 index 00000000..247922e1 --- /dev/null +++ b/mateclaw-server/src/test/resources/agent-evaluation/goal-service-boundaries-v1.json @@ -0,0 +1,102 @@ +{ + "schemaVersion": 1, + "suiteId": "goal-service-boundaries-v1", + "tasks": [ + { + "id": "current-revision-completion", + "task": "current revision completion", + "source": "cycles 004/006/010/013 real service boundaries", + "operation": "CURRENT_REVISION_COMPLETION", + "expected": { + "status": "COMPLETED", + "score": 1.0, + "criteriaCount": 1, + "passedCount": 1, + "firstCriterionText": "report", + "evaluationRevision": 1, + "recordedEvalCallsUsed": 1, + "completionCode": 200 + } + }, + { + "id": "append-before-verdict", + "task": "append before verdict", + "source": "cycles 004/006/010/013 real service boundaries", + "operation": "APPEND_BEFORE_VERDICT", + "expected": { + "status": "ACTIVE", + "score": 0.5, + "criteriaCount": 2, + "passedCount": 1, + "firstCriterionText": "report", + "evaluationRevision": 0, + "recordedEvalCallsUsed": 1, + "completionCode": 409 + } + }, + { + "id": "pause-before-verdict", + "task": "pause before verdict", + "source": "cycles 004/006/010/013 real service boundaries", + "operation": "PAUSE_BEFORE_VERDICT", + "expected": { + "status": "PAUSED", + "score": 0.0, + "criteriaCount": 1, + "passedCount": 0, + "firstCriterionText": "report", + "evaluationRevision": 0, + "recordedEvalCallsUsed": 2, + "completionCode": 409 + } + }, + { + "id": "append-before-bootstrap", + "task": "append before bootstrap", + "source": "cycles 004/006/010/013 real service boundaries", + "operation": "APPEND_BEFORE_BOOTSTRAP", + "expected": { + "status": "ACTIVE", + "score": 0.0, + "criteriaCount": 1, + "passedCount": 0, + "firstCriterionText": "user appendix", + "evaluationRevision": 0, + "recordedEvalCallsUsed": 1, + "completionCode": 0 + } + }, + { + "id": "replace-definition", + "task": "replace definition", + "source": "cycles 004/006/010/013 real service boundaries", + "operation": "REPLACE_DEFINITION", + "expected": { + "status": "ACTIVE", + "score": 0.0, + "criteriaCount": 0, + "passedCount": 0, + "firstCriterionText": null, + "evaluationRevision": 1, + "recordedEvalCallsUsed": 2, + "completionCode": 409 + } + }, + { + "id": "aba-definition", + "task": "aba definition", + "source": "cycles 004/006/010/013 real service boundaries", + "operation": "ABA_DEFINITION", + "expected": { + "status": "ACTIVE", + "score": 0.0, + "criteriaCount": 0, + "passedCount": 0, + "firstCriterionText": null, + "evaluationRevision": 2, + "recordedEvalCallsUsed": 1, + "completionCode": 0 + } + } + ] +}