fix(goal): skip live continuation leases during recovery scans

This commit is contained in:
mateaix 2026-09-15 01:00:56 +08:00
parent 537f43cb6e
commit bd85d88355
6 changed files with 72 additions and 2 deletions

View File

@ -135,6 +135,15 @@ public class GoalContinuationStore {
""",state,state,nextRunAt,failures,bounded(reason),now,goalId,token,attemptId,revision)==1;
}
/** Current read under the caller's goal lock, before recovery mutates its attempt. */
boolean hasExpiredFence(Long goalId, String token, String attemptId, long cutoffEpoch) {
return jdbc.queryForList("""
SELECT goal_id FROM mate_goal_continuation
WHERE goal_id=? AND lease_owner=? AND current_attempt_id=? AND state='running'
AND lease_until_epoch_second<=? FOR UPDATE
""", Long.class, goalId, token, attemptId, cutoffEpoch).size() == 1;
}
public boolean recoverExpired(Long goalId,String token,String attemptId,long expiredEpoch,
String state,LocalDateTime nextRunAt,int failures,String reason,LocalDateTime now) {
return jdbc.update("""

View File

@ -68,6 +68,9 @@ public class GoalRecoveryService {
var continuation=continuations.get(attempt.goalId());
if(continuation==null || !attempt.id().equals(continuation.currentAttemptId())
|| !attempt.leaseToken().equals(continuation.leaseOwner())) return false;
// The scan only proves the attempt expired. A live or changed projection
// is not recoverable yet and must not abort recovery of later goals.
if(!continuations.hasExpiredFence(attempt.goalId(),attempt.leaseToken(),attempt.id(),nowEpoch)) return false;
RecoveryDecision decision=classify(attempt);
String attemptState=decision==RecoveryDecision.BLOCK_UNCERTAIN_SIDE_EFFECT ? "blocked" : "retryable";
String projectionState=decision==RecoveryDecision.BLOCK_UNCERTAIN_SIDE_EFFECT ? "blocked" : "retry";

View File

@ -51,7 +51,7 @@ V197 makes expiry authoritative in epoch seconds, independent of the JVM/JDBC ti
The host clock, database credentials and service host remain trusted. Run arbitrary external code without service/database credentials and outside the service's storage permissions if it is not trusted; setting a working directory or scanning paths does not provide OS isolation. Use coordinated backups of the database for requirements, bodies, pointers and bindings; workspace-file backups alone cannot restore managed acceptance.
V198 also stores absolute scheduler lease deadlines. Existing leases expire during upgrade and are recovered from their persisted checkpoints: safe work may receive a new attempt; uncertain side effects remain blocked for review. Expired owners cannot renew, checkpoint, settle or use managed JSON tools. Renewal checks both current lease records after acquiring the goal lock, so a delayed scheduler tick cannot reuse an old timestamp to revive its owner. New valid owners can continue under the existing requirements.
V198 also stores absolute scheduler lease deadlines. Existing leases expire during upgrade and are recovered from their persisted checkpoints: safe work may receive a new attempt; uncertain side effects remain blocked for review. Expired owners cannot renew, checkpoint, settle or use managed JSON tools. Renewal checks both current lease records after acquiring the goal lock, so a delayed scheduler tick cannot reuse an old timestamp to revive its owner. New valid owners can continue under the existing requirements. Recovery skips a scanned attempt while its continuation lease is still live or its owner has changed, so other eligible recoveries can proceed. A passing JSON binding does not override a pause caused by an uncertain tool outcome.
Validation snapshot (2026-09-14): the full default backend test run passed 5,249 executed tests with 33 conditional skips; the frontend passed 386 tests. The managed contract also has real compiled ReAct/Plan graph tests for account and scheduled-owner execution. Their model choices and semantic verdicts are controlled fixtures, not online-model benchmarks. Specialized integration profiles and the proprietary Kingbase engine are outside that full-default-suite claim.

View File

@ -51,7 +51,7 @@ V197 使用 epoch 秒作为有效期依据,不受 JVM/JDBC 时区变化影响
宿主时钟、数据库凭据和服务宿主仍属于可信基础。若外部任意代码不可信,应在没有服务/数据库凭据、没有服务存储权限的隔离环境执行;仅设置工作目录或扫描路径不是操作系统隔离。备份应协调保存数据库中的要求、正文、指针和绑定,仅备份工作区文件无法恢复受管验收。
V198 同样以绝对时间保存调度租约截止。升级时旧租约失效,按已有检查点恢复:安全工作可建立新 attempt不确定副作用仍阻断并要求核实。过期 owner 不能续租、提交检查点、结算或使用受管 JSON 工具。续租在获得 Goal 锁后检查两侧当前租约,迟到调度 tick 不能利用旧时间戳复活 owner新的有效 owner 可按现有要求继续。
V198 同样以绝对时间保存调度租约截止。升级时旧租约失效,按已有检查点恢复:安全工作可建立新 attempt不确定副作用仍阻断并要求核实。过期 owner 不能续租、提交检查点、结算或使用受管 JSON 工具。续租在获得 Goal 锁后检查两侧当前租约,迟到调度 tick 不能利用旧时间戳复活 owner新的有效 owner 可按现有要求继续。恢复扫描遇到 continuation 租约仍有效或 owner 已变化的记录时会跳过继续处理其他可恢复目标。JSON 绑定通过也不能越过不确定工具结果导致的暂停。
验证快照2026-09-14默认后端完整测试集实际通过 5,249 项、条件跳过 33 项,前端通过 386 项。受管契约还覆盖真实编译的 ReAct/Plan 图及账户/调度 owner 四种组合;模型选择和语义评价是受控夹具,不是在线模型基准。专门集成 profile 与 Kingbase 专有引擎不包含在“默认完整测试集”结论中。

View File

@ -42,6 +42,8 @@ class GoalJsonAcceptanceIntegrationTest {
@Autowired private vip.mate.goal.service.GoalJsonBindingService bindings;
@Autowired private vip.mate.goal.service.GoalContinuationStore continuations;
@Autowired private vip.mate.goal.service.GoalRunCoordinator coordinator;
@Autowired private vip.mate.goal.service.GoalRecoveryService recovery;
@Autowired private vip.mate.goal.service.GoalAttemptStore attempts;
private String alice;
private String bob;
@ -608,6 +610,36 @@ class GoalJsonAcceptanceIntegrationTest {
assertEquals(GoalStatus.COMPLETED, goals.getById(goal.getId()).getStatus());
}
@Test void uncertainRecoveryPreservesSelectedJsonAndBlocksEvenAPreviouslyPassingBinding() {
GoalEntity goal = goal(true);
goals.appendCriterion(goal.getId(), "report", alice);
var evaluation = new GoalEvaluationResult(1, "offline fixture", "completed", true, "fixture", 1, 0,
List.of(new GoalChecklistVerdict.CriterionVerdict("C1", true, "fixture only")), null);
goals.recordEvaluation(goal.getId(), evaluation, 1, 1);
acceptance.configure(goal.getId(), "r", request(0, "summary"), alice);
var run = claimed(goal);
var origin = attemptOrigin(goal, run);
var version = artifacts.publishForRuntime(origin, "report", publication(0, "{\"summary\":true}"));
bindings.checkForRuntime(origin, "r", checkRequest(1, version));
assertTrue(bindings.state(goal.getId(), alice).getFirst().acceptanceEligible());
assertTrue(coordinator.checkpoint(run, "uncertain", "tool_started", null, java.time.LocalDateTime.now()));
long expired = java.time.Instant.now().minusSeconds(1).getEpochSecond();
jdbc.update("UPDATE mate_goal_attempt SET lease_until_epoch_second=? WHERE attempt_id=?", expired, run.attempt().id());
jdbc.update("UPDATE mate_goal_continuation SET lease_until_epoch_second=? WHERE goal_id=?", expired, goal.getId());
assertTrue(recovery.recoverExpired(java.time.Instant.now()) >= 1);
assertEquals(GoalStatus.PAUSED, goals.getById(goal.getId()).getStatus());
assertTrue(goals.getById(goal.getId()).isJsonAcceptanceRequired());
assertEquals("blocked", attempts.get(run.attempt().id()).state());
assertEquals("blocked", continuations.get(goal.getId()).state());
assertNull(coordinator.claim(continuations.get(goal.getId()), goals.getById(goal.getId()), java.time.LocalDateTime.now()));
assertFalse(coordinator.renew(run, java.time.LocalDateTime.now()));
assertThrows(MateClawException.class, () -> goals.markRuntimeCompleted(goal.getId(), null, origin));
assertThrows(MateClawException.class, () -> goals.markCompleted(goal.getId(), null));
assertEquals("{\"summary\":true}", artifacts.read(goal.getId(), version.artifactId(), alice).jsonContent());
assertTrue(bindings.state(goal.getId(), alice).getFirst().acceptanceEligible(),
"Valid JSON is preserved, but cannot override an uncertain execution's paused state");
}
@Test void expiredRuntimeCannotCompleteEvenWithCurrentPassingBindings() {
GoalEntity goal = goal(true);
goals.appendCriterion(goal.getId(), "report", alice);

View File

@ -78,6 +78,32 @@ class GoalRecoveryServiceTest {
assertEquals(queued.id(),inputs.listQueued("conv").getFirst().id());
}
@Test void liveProjectionDoesNotAbortRecoveryOfOtherExpiredAttempts() {
var live = coordinator.claim(continuations.get(1L), goal, now);
assertTrue(coordinator.markRunning(live, now));
jdbc.update("""
INSERT INTO mate_agent_goal(id,conversation_id,agent_id,workspace_id,created_by,title,
description,status,persistent_execution,auto_followup_enabled,create_time,update_time)
VALUES(2,'conv2',2,3,'alice','second','objective','active',TRUE,TRUE,?,?)
""", now, now);
var second = new GoalEntity();
org.springframework.beans.BeanUtils.copyProperties(goal, second);
second.setId(2L); second.setConversationId("conv2");
continuations.discover(now);
var expired = coordinator.claim(continuations.get(2L), second, now);
assertTrue(coordinator.markRunning(expired, now));
long moment = now.atZone(java.time.ZoneId.systemDefault()).toEpochSecond();
// The first scan candidate has a still-live projection; a later one is eligible.
jdbc.update("UPDATE mate_goal_attempt SET lease_until_epoch_second=? WHERE attempt_id=?", moment - 2, live.attempt().id());
jdbc.update("UPDATE mate_goal_attempt SET lease_until_epoch_second=? WHERE attempt_id=?", moment - 1, expired.attempt().id());
jdbc.update("UPDATE mate_goal_continuation SET lease_until_epoch_second=? WHERE goal_id=2", moment - 1);
assertEquals(1, recovery.recoverExpired(java.time.Instant.ofEpochSecond(moment)));
assertEquals("running", attempts.get(live.attempt().id()).state());
assertEquals(live.attempt().id(), continuations.get(1L).currentAttemptId());
assertEquals("retryable", attempts.get(expired.attempt().id()).state());
assertEquals("retry", continuations.get(2L).state());
}
@Test void uncertainToolAttemptBlocksInsteadOfReplaying() {
var old=coordinator.claim(continuations.get(1L),goal,now);
assertTrue(coordinator.markRunning(old,now));