mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-14 03:33:43 +08:00
test(evaluation): replay artifact version boundaries with real file IO
This commit is contained in:
mateaix
parent
501a1f8bdb
commit
bb3d477aa2
@ -0,0 +1,140 @@
|
||||
package vip.mate.evaluation;
|
||||
|
||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.springframework.ai.chat.model.ToolContext;
|
||||
import vip.mate.agent.context.ChatOrigin;
|
||||
import vip.mate.execution.evidence.service.ExecutionObservationSink;
|
||||
import vip.mate.tool.document.GeneratedFileCache;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
||||
/** Runs allowlisted platform actions on temporary files. No model, shell command or user path is executed. */
|
||||
final class OfflineArtifactTaskReplay {
|
||||
static final ObjectMapper JSON = new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS)
|
||||
.enable(DeserializationFeature.FAIL_ON_NUMBERS_FOR_ENUMS);
|
||||
enum Operation { REGISTER, MUTATE_INPUT, MUTATE_DOWNLOAD, RESTART_MUTATE_DOWNLOAD,
|
||||
STORAGE_UNAVAILABLE, DIRECT_RETURN, MISSING_OWNER, REWRITE_DISK }
|
||||
record Suite(Integer schemaVersion, String suiteId, List<Task> tasks) { }
|
||||
record Task(String id, String task, String source, Operation operation, String content,
|
||||
Expected expected) { }
|
||||
record Expected(String hotContent, String coldContent, List<String> observations,
|
||||
Boolean hotMatchesSnapshot, Boolean coldMatchesSnapshot) { }
|
||||
record Actual(String hotContent, String coldContent, List<String> observations,
|
||||
String snapshotDigest, Boolean hotMatchesSnapshot, Boolean coldMatchesSnapshot) { }
|
||||
record Result(String id, String task, String source, Operation operation, Expected expected,
|
||||
Actual actual, boolean matched, String error) { }
|
||||
record Report(int schemaVersion, String suiteId, String suiteSha256, String revisionLabel,
|
||||
Map<String, String> productionClassSha256, String executionMode, int onlineModelCalls,
|
||||
String agentTaskSuccessRate, String onlineCost, int matchedCases, int mismatchedCases,
|
||||
List<Result> cases) { }
|
||||
|
||||
static Suite parse(byte[] bytes) throws IOException {
|
||||
Suite suite = JSON.readValue(bytes, Suite.class);
|
||||
require(suite != null && Integer.valueOf(1).equals(suite.schemaVersion()), "schemaVersion must be 1");
|
||||
require(nonblank(suite.suiteId()) && suite.tasks() != null && !suite.tasks().isEmpty()
|
||||
&& suite.tasks().size() <= 100, "suiteId and 1..100 tasks required");
|
||||
var ids = new HashSet<String>();
|
||||
for (Task task : suite.tasks()) {
|
||||
require(task != null && nonblank(task.id()) && ids.add(task.id()), "unique task IDs required");
|
||||
require(nonblank(task.task()) && nonblank(task.source()) && task.operation() != null
|
||||
&& nonblank(task.content()) && task.content().length() <= 16_384,
|
||||
task.id() + ": invalid task inputs");
|
||||
Expected expected = task.expected();
|
||||
require(expected != null && expected.hotContent() != null && expected.observations() != null,
|
||||
task.id() + ": hotContent and observations expectations required");
|
||||
require(expected.observations().isEmpty()
|
||||
|| expected.observations().equals(List.of("ARTIFACT_SNAPSHOT:OBSERVED")),
|
||||
task.id() + ": unsupported observations expectation");
|
||||
boolean snapshotExpected = !expected.observations().isEmpty();
|
||||
require(snapshotExpected == (expected.hotMatchesSnapshot() != null)
|
||||
&& (snapshotExpected && expected.coldContent() != null) == (expected.coldMatchesSnapshot() != null),
|
||||
task.id() + ": snapshot match expectations must be explicit when comparable");
|
||||
}
|
||||
return suite;
|
||||
}
|
||||
|
||||
static Report run(byte[] bytes, Path root, String revisionLabel) throws IOException {
|
||||
Suite suite = parse(bytes);
|
||||
require(nonblank(revisionLabel), "revisionLabel required");
|
||||
List<Result> results = new ArrayList<>();
|
||||
for (Task task : suite.tasks()) {
|
||||
try {
|
||||
// Paths are generated by the harness, never taken from task JSON.
|
||||
Actual actual = execute(task, Files.createTempDirectory(root, "artifact-case-"));
|
||||
Expected e = task.expected();
|
||||
boolean matched = Objects.equals(e.hotContent(), actual.hotContent())
|
||||
&& Objects.equals(e.coldContent(), actual.coldContent())
|
||||
&& e.observations().equals(actual.observations())
|
||||
&& Objects.equals(e.hotMatchesSnapshot(), actual.hotMatchesSnapshot())
|
||||
&& Objects.equals(e.coldMatchesSnapshot(), actual.coldMatchesSnapshot());
|
||||
results.add(new Result(task.id(), task.task(), task.source(), task.operation(), e, actual, matched, null));
|
||||
} catch (IOException | RuntimeException error) {
|
||||
results.add(new Result(task.id(), task.task(), task.source(), task.operation(), task.expected(),
|
||||
null, false, error.getClass().getSimpleName()));
|
||||
}
|
||||
}
|
||||
int matched = (int) results.stream().filter(Result::matched).count();
|
||||
Map<String, String> classes = new LinkedHashMap<>();
|
||||
for (Class<?> type : List.of(GeneratedFileCache.class, GeneratedFileCache.Entry.class, ExecutionObservationSink.class)) {
|
||||
try (var stream = type.getResourceAsStream("/" + type.getName().replace('.', '/') + ".class")) {
|
||||
if (stream == null) throw new IOException("Missing tested class " + type.getName());
|
||||
classes.put(type.getName(), digest(stream.readAllBytes()));
|
||||
}
|
||||
}
|
||||
return new Report(1, suite.suiteId(), digest(bytes), revisionLabel, classes,
|
||||
"offline_platform_fixture_io", 0, "not_measured", "not_measured", matched,
|
||||
results.size() - matched, List.copyOf(results));
|
||||
}
|
||||
|
||||
private static Actual execute(Task task, Path root) throws IOException {
|
||||
Path storage = root.resolve("cache");
|
||||
if (task.operation() == Operation.STORAGE_UNAVAILABLE) Files.writeString(storage, "not a directory");
|
||||
var cache = new GeneratedFileCache(storage);
|
||||
var sink = new ExecutionObservationSink(task.operation() == Operation.DIRECT_RETURN);
|
||||
ToolContext origin = task.operation() == Operation.MISSING_OWNER ? new ToolContext(Map.of())
|
||||
: ChatOrigin.web("offline-artifact", "fixture-owner", 1L, root.toString()).toToolContext();
|
||||
byte[] input = task.content().getBytes(StandardCharsets.UTF_8);
|
||||
String id = cache.put(input, "report.txt", "text/plain", sink.attach(origin));
|
||||
switch (task.operation()) {
|
||||
case MUTATE_INPUT -> input[0] ^= 0x01;
|
||||
case MUTATE_DOWNLOAD -> cache.get(id).orElseThrow().bytes()[0] ^= 0x01;
|
||||
case RESTART_MUTATE_DOWNLOAD -> {
|
||||
cache = new GeneratedFileCache(storage);
|
||||
cache.get(id).orElseThrow().bytes()[0] ^= 0x01;
|
||||
}
|
||||
case REWRITE_DISK -> Files.writeString(storage.resolve(id), "externally replaced");
|
||||
default -> { }
|
||||
}
|
||||
byte[] hot = cache.get(id).orElseThrow().bytes();
|
||||
byte[] cold = new GeneratedFileCache(storage).get(id).map(GeneratedFileCache.Entry::bytes).orElse(null);
|
||||
var observations = sink.observations();
|
||||
String snapshot = observations.isEmpty() ? null : observations.getFirst().artifactDigest();
|
||||
return new Actual(new String(hot, StandardCharsets.UTF_8),
|
||||
cold == null ? null : new String(cold, StandardCharsets.UTF_8),
|
||||
observations.stream().map(o -> o.kind() + ":" + o.result()).toList(), snapshot,
|
||||
snapshot == null ? null : snapshot.equals(digest(hot)),
|
||||
snapshot == null || cold == null ? null : snapshot.equals(digest(cold)));
|
||||
}
|
||||
|
||||
private static String digest(byte[] bytes) {
|
||||
try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes)); }
|
||||
catch (NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); }
|
||||
}
|
||||
private static boolean nonblank(String s) { return s != null && !s.isBlank(); }
|
||||
private static void require(boolean condition, String message) {
|
||||
if (!condition) throw new IllegalArgumentException(message);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,53 @@
|
||||
package vip.mate.evaluation;
|
||||
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class OfflineArtifactTaskReplayTest {
|
||||
@TempDir Path root;
|
||||
|
||||
private byte[] fixture() throws Exception {
|
||||
try (var stream = getClass().getResourceAsStream("/agent-evaluation/artifact-boundaries-v1.json")) {
|
||||
assertNotNull(stream);
|
||||
return stream.readAllBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void executePlatformTasksAndWriteReport() throws Exception {
|
||||
String input = System.getProperty("artifact.eval.suite");
|
||||
var report = OfflineArtifactTaskReplay.run(input == null ? fixture() : Files.readAllBytes(Path.of(input)),
|
||||
root, System.getProperty("artifact.eval.revision", "unrecorded"));
|
||||
Path output = Path.of(System.getProperty("artifact.eval.report", "target/agent-evaluation/artifact-baseline.json"));
|
||||
Files.createDirectories(output.toAbsolutePath().getParent());
|
||||
OfflineArtifactTaskReplay.JSON.writerWithDefaultPrettyPrinter().writeValue(output.toFile(), report);
|
||||
assertEquals(0, report.mismatchedCases(), () -> "Artifact task mismatches: " + output.toAbsolutePath());
|
||||
assertEquals(0, report.onlineModelCalls());
|
||||
assertEquals(3, report.productionClassSha256().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void invalidInputIsRejectedBeforeCreatingFiles() throws Exception {
|
||||
ObjectNode suite = (ObjectNode) OfflineArtifactTaskReplay.JSON.readTree(fixture());
|
||||
((ObjectNode) suite.withArray("tasks").get(0).get("expected")).remove("hotMatchesSnapshot");
|
||||
assertThrows(IllegalArgumentException.class, () -> OfflineArtifactTaskReplay.run(
|
||||
suite.toString().getBytes(StandardCharsets.UTF_8), root, "test"));
|
||||
try (var files = Files.list(root)) { assertEquals(0, files.count()); }
|
||||
}
|
||||
|
||||
@Test
|
||||
void wrongExpectationReportsMismatchAndContinues() throws Exception {
|
||||
ObjectNode suite = (ObjectNode) OfflineArtifactTaskReplay.JSON.readTree(fixture());
|
||||
((ObjectNode) suite.withArray("tasks").get(0).get("expected")).put("hotContent", "wrong");
|
||||
var report = OfflineArtifactTaskReplay.run(suite.toString().getBytes(StandardCharsets.UTF_8), root, "test");
|
||||
assertEquals(1, report.mismatchedCases());
|
||||
assertEquals(suite.withArray("tasks").size(), report.cases().size());
|
||||
}
|
||||
}
|
||||
@ -60,3 +60,48 @@ strong acceptance. A future trusted file recipe must use a separate execution
|
||||
mode and record its actual filesystem/version checks. A real model comparison
|
||||
must additionally pin runtime/skill/model versions and measure real calls; it
|
||||
cannot reuse these matched counts as its success rate.
|
||||
|
||||
## Artifact tasks with actual temporary-file I/O
|
||||
|
||||
`artifact-boundaries-v1.json` is a separate schema/execution mode. Run it from the
|
||||
repository root:
|
||||
|
||||
```sh
|
||||
mvn -pl mateclaw-server -am -Dtest=OfflineArtifactTaskReplayTest \
|
||||
-Dsurefire.failIfNoSpecifiedTests=false \
|
||||
-Dartifact.eval.revision="$(git rev-parse HEAD)" test
|
||||
```
|
||||
|
||||
Output: `mateclaw-server/target/agent-evaluation/artifact-baseline.json`.
|
||||
Override `artifact.eval.suite` and `artifact.eval.report` with absolute paths for
|
||||
candidate runs. The initial committed artifact baseline's revision label refers
|
||||
to the production source before this harness was added. Reports additionally
|
||||
hash the actual loaded `GeneratedFileCache`, `Entry`, and
|
||||
`ExecutionObservationSink` class bytes, so a revision label alone is not treated
|
||||
as proof of which implementations were exercised. Class hashes depend on the
|
||||
compiler/build; compare revisions under the same build environment. They cover
|
||||
these named classes, not the entire transitive runtime or a signed attestation.
|
||||
|
||||
Each artifact task has `id`, `task`, `source`, `operation`, `content`, and
|
||||
`expected`. Operations are a closed set: `REGISTER`, `MUTATE_INPUT`,
|
||||
`MUTATE_DOWNLOAD`, `RESTART_MUTATE_DOWNLOAD`, `STORAGE_UNAVAILABLE`,
|
||||
`DIRECT_RETURN`, `MISSING_OWNER`, `REWRITE_DISK`. Input strings are bounded to
|
||||
16,384 characters. The harness generates its own temporary paths and never
|
||||
executes fixture-provided paths, scripts or shell commands.
|
||||
|
||||
Expected results include `hotContent`, nullable `coldContent`, `observations`
|
||||
(`[]` or `["ARTIFACT_SNAPSHOT:OBSERVED"]`), and nullable
|
||||
`hotMatchesSnapshot`/`coldMatchesSnapshot`. A match is null when no snapshot or
|
||||
readable version exists; it never silently becomes a pass. Every task uses the
|
||||
production cache and sink, writes/reads real temporary files, and reopens the
|
||||
cache to test restart behavior. The directory is removed by JUnit after the run.
|
||||
A mismatched expectation preserves all per-case results and fails the command.
|
||||
The checked-in `artifact-baseline-v1.json` documents these eight scenarios.
|
||||
|
||||
`REWRITE_DISK` intentionally observes that the hot cache keeps the original
|
||||
bytes while a new cache reads the externally replaced version, which no longer
|
||||
matches the historic snapshot digest. This is a documented limit of unmanaged
|
||||
storage, not an accepted strong-validation result. The report never emits a
|
||||
CHECK_RESULT/PASS. These eight tests plus ten evaluator replays are **18 fixed
|
||||
scenarios, not 18 Agent task executions**. Actual task-solving models, their
|
||||
latency, cost and success rate remain unmeasured.
|
||||
|
||||
@ -0,0 +1,194 @@
|
||||
{
|
||||
"schemaVersion" : 1,
|
||||
"suiteId" : "artifact-boundaries-v1",
|
||||
"suiteSha256" : "d3c50469bc767b532790745437635d9e92234bc04d0732d7a6d26e8dcc6bedcf",
|
||||
"revisionLabel" : "0513e2cde98b153ffa18f946e9edee55dd24b7f8",
|
||||
"productionClassSha256" : {
|
||||
"vip.mate.tool.document.GeneratedFileCache" : "7a795aadcad53c8f4f5baf1e5f79cd303dbc0098b0bd5459eaa3a9eb951b2d8f",
|
||||
"vip.mate.tool.document.GeneratedFileCache$Entry" : "440483cf1084202180a60b7959996e216362af12f4034cd2a7d9f92dd59254f0",
|
||||
"vip.mate.execution.evidence.service.ExecutionObservationSink" : "23975edeb8970c70d5ef756e4e4b68ff1b98818d64e63dba9edc55b0b134a2ae"
|
||||
},
|
||||
"executionMode" : "offline_platform_fixture_io",
|
||||
"onlineModelCalls" : 0,
|
||||
"agentTaskSuccessRate" : "not_measured",
|
||||
"onlineCost" : "not_measured",
|
||||
"matchedCases" : 8,
|
||||
"mismatchedCases" : 0,
|
||||
"cases" : [ {
|
||||
"id" : "register",
|
||||
"task" : "Register a durable report and reopen it after restart",
|
||||
"source" : "TrustedExecutionObservationTest.onlyDurablyReadableArtifactsProduceSnapshots",
|
||||
"operation" : "REGISTER",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"snapshotDigest" : "231d40769a0f1d875c0fee33d8dc02eaab83925e80c9e95a2ed00eae1a9916d1",
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "mutate-input",
|
||||
"task" : "Producer reuses its buffer after registering a report",
|
||||
"source" : "cycle003 reproduced input byte alias failure",
|
||||
"operation" : "MUTATE_INPUT",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"snapshotDigest" : "231d40769a0f1d875c0fee33d8dc02eaab83925e80c9e95a2ed00eae1a9916d1",
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "mutate-download",
|
||||
"task" : "Consumer modifies a returned download buffer",
|
||||
"source" : "cycle003 reproduced returned byte alias failure",
|
||||
"operation" : "MUTATE_DOWNLOAD",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"snapshotDigest" : "231d40769a0f1d875c0fee33d8dc02eaab83925e80c9e95a2ed00eae1a9916d1",
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "restart-mutate-download",
|
||||
"task" : "Consumer modifies bytes reloaded after restart",
|
||||
"source" : "GeneratedFileCachePersistenceTest persistence contract",
|
||||
"operation" : "RESTART_MUTATE_DOWNLOAD",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"snapshotDigest" : "231d40769a0f1d875c0fee33d8dc02eaab83925e80c9e95a2ed00eae1a9916d1",
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : true
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "storage-unavailable",
|
||||
"task" : "Register a report when the durable storage path is not a directory",
|
||||
"source" : "TrustedExecutionObservationTest durable storage failure",
|
||||
"operation" : "STORAGE_UNAVAILABLE",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : null,
|
||||
"observations" : [ ],
|
||||
"hotMatchesSnapshot" : null,
|
||||
"coldMatchesSnapshot" : null
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : null,
|
||||
"observations" : [ ],
|
||||
"snapshotDigest" : null,
|
||||
"hotMatchesSnapshot" : null,
|
||||
"coldMatchesSnapshot" : null
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "direct-return",
|
||||
"task" : "Deliver a direct-return file without recording content evidence",
|
||||
"source" : "TrustedExecutionObservationTest direct-return privacy boundary",
|
||||
"operation" : "DIRECT_RETURN",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ ],
|
||||
"hotMatchesSnapshot" : null,
|
||||
"coldMatchesSnapshot" : null
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ ],
|
||||
"snapshotDigest" : null,
|
||||
"hotMatchesSnapshot" : null,
|
||||
"coldMatchesSnapshot" : null
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "missing-owner",
|
||||
"task" : "Register a file without canonical workspace/conversation ownership",
|
||||
"source" : "GeneratedFileCache.put typed observation owner checks",
|
||||
"operation" : "MISSING_OWNER",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ ],
|
||||
"hotMatchesSnapshot" : null,
|
||||
"coldMatchesSnapshot" : null
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "registered report",
|
||||
"observations" : [ ],
|
||||
"snapshotDigest" : null,
|
||||
"hotMatchesSnapshot" : null,
|
||||
"coldMatchesSnapshot" : null
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
}, {
|
||||
"id" : "rewrite-disk",
|
||||
"task" : "Reopen a report after an external writer replaces its persisted bytes",
|
||||
"source" : "RFC096 unmanaged shared-directory boundary; known Observe limitation",
|
||||
"operation" : "REWRITE_DISK",
|
||||
"expected" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "externally replaced",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : false
|
||||
},
|
||||
"actual" : {
|
||||
"hotContent" : "registered report",
|
||||
"coldContent" : "externally replaced",
|
||||
"observations" : [ "ARTIFACT_SNAPSHOT:OBSERVED" ],
|
||||
"snapshotDigest" : "231d40769a0f1d875c0fee33d8dc02eaab83925e80c9e95a2ed00eae1a9916d1",
|
||||
"hotMatchesSnapshot" : true,
|
||||
"coldMatchesSnapshot" : false
|
||||
},
|
||||
"matched" : true,
|
||||
"error" : null
|
||||
} ]
|
||||
}
|
||||
@ -0,0 +1,128 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"suiteId": "artifact-boundaries-v1",
|
||||
"tasks": [
|
||||
{
|
||||
"id": "register",
|
||||
"task": "Register a durable report and reopen it after restart",
|
||||
"source": "TrustedExecutionObservationTest.onlyDurablyReadableArtifactsProduceSnapshots",
|
||||
"operation": "REGISTER",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "registered report",
|
||||
"observations": [
|
||||
"ARTIFACT_SNAPSHOT:OBSERVED"
|
||||
],
|
||||
"hotMatchesSnapshot": true,
|
||||
"coldMatchesSnapshot": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "mutate-input",
|
||||
"task": "Producer reuses its buffer after registering a report",
|
||||
"source": "cycle003 reproduced input byte alias failure",
|
||||
"operation": "MUTATE_INPUT",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "registered report",
|
||||
"observations": [
|
||||
"ARTIFACT_SNAPSHOT:OBSERVED"
|
||||
],
|
||||
"hotMatchesSnapshot": true,
|
||||
"coldMatchesSnapshot": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "mutate-download",
|
||||
"task": "Consumer modifies a returned download buffer",
|
||||
"source": "cycle003 reproduced returned byte alias failure",
|
||||
"operation": "MUTATE_DOWNLOAD",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "registered report",
|
||||
"observations": [
|
||||
"ARTIFACT_SNAPSHOT:OBSERVED"
|
||||
],
|
||||
"hotMatchesSnapshot": true,
|
||||
"coldMatchesSnapshot": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "restart-mutate-download",
|
||||
"task": "Consumer modifies bytes reloaded after restart",
|
||||
"source": "GeneratedFileCachePersistenceTest persistence contract",
|
||||
"operation": "RESTART_MUTATE_DOWNLOAD",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "registered report",
|
||||
"observations": [
|
||||
"ARTIFACT_SNAPSHOT:OBSERVED"
|
||||
],
|
||||
"hotMatchesSnapshot": true,
|
||||
"coldMatchesSnapshot": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "storage-unavailable",
|
||||
"task": "Register a report when the durable storage path is not a directory",
|
||||
"source": "TrustedExecutionObservationTest durable storage failure",
|
||||
"operation": "STORAGE_UNAVAILABLE",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": null,
|
||||
"observations": [],
|
||||
"hotMatchesSnapshot": null,
|
||||
"coldMatchesSnapshot": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "direct-return",
|
||||
"task": "Deliver a direct-return file without recording content evidence",
|
||||
"source": "TrustedExecutionObservationTest direct-return privacy boundary",
|
||||
"operation": "DIRECT_RETURN",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "registered report",
|
||||
"observations": [],
|
||||
"hotMatchesSnapshot": null,
|
||||
"coldMatchesSnapshot": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "missing-owner",
|
||||
"task": "Register a file without canonical workspace/conversation ownership",
|
||||
"source": "GeneratedFileCache.put typed observation owner checks",
|
||||
"operation": "MISSING_OWNER",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "registered report",
|
||||
"observations": [],
|
||||
"hotMatchesSnapshot": null,
|
||||
"coldMatchesSnapshot": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rewrite-disk",
|
||||
"task": "Reopen a report after an external writer replaces its persisted bytes",
|
||||
"source": "RFC096 unmanaged shared-directory boundary; known Observe limitation",
|
||||
"operation": "REWRITE_DISK",
|
||||
"content": "registered report",
|
||||
"expected": {
|
||||
"hotContent": "registered report",
|
||||
"coldContent": "externally replaced",
|
||||
"observations": [
|
||||
"ARTIFACT_SNAPSHOT:OBSERVED"
|
||||
],
|
||||
"hotMatchesSnapshot": true,
|
||||
"coldMatchesSnapshot": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user