fix(goal): reject checklist passes without evidence

This commit is contained in:
mateaix 2026-09-13 20:38:19 +08:00
parent 00d774fb85
commit 88cd78af5e
3 changed files with 50 additions and 8 deletions

View File

@ -58,8 +58,8 @@ public final class GoalCriteriaCodec {
/** /**
* Merge a per-round verdict delta into the full checklist by id. Criteria * Merge a per-round verdict delta into the full checklist by id. Criteria
* absent from the delta are preserved unchanged; the criterion text is * absent from the delta retain their state unless their pass lacks evidence.
* always kept from the existing item (the verdict never carries text). * The criterion text is always kept from the existing item (the verdict never carries text).
*/ */
public static List<GoalCriterion> merge(List<GoalCriterion> existing, public static List<GoalCriterion> merge(List<GoalCriterion> existing,
List<GoalChecklistVerdict.CriterionVerdict> verdicts) { List<GoalChecklistVerdict.CriterionVerdict> verdicts) {
@ -77,18 +77,22 @@ public final class GoalCriteriaCodec {
List<GoalCriterion> merged = new ArrayList<>(existing.size()); List<GoalCriterion> merged = new ArrayList<>(existing.size());
for (GoalCriterion c : existing) { for (GoalCriterion c : existing) {
GoalChecklistVerdict.CriterionVerdict v = byId.get(c.id()); GoalChecklistVerdict.CriterionVerdict v = byId.get(c.id());
merged.add(v == null GoalCriterion candidate = v == null ? c
? c
: new GoalCriterion(c.id(), c.text(), v.passed(), : new GoalCriterion(c.id(), c.text(), v.passed(),
v.evidence() != null ? v.evidence() : "")); v.evidence() != null ? v.evidence() : "");
// A model boolean alone cannot satisfy even the legacy semantic
// checklist. This checks presence, not truth or execution provenance.
merged.add(candidate.passed() && !hasEvidence(candidate)
? new GoalCriterion(candidate.id(), candidate.text(), false, candidate.evidence())
: candidate);
} }
return merged; return merged;
} }
/** True only when the list is non-empty and every criterion is passed. */ /** True only when the list is non-empty and every criterion is passed with nonblank evidence. */
public static boolean allPassed(List<GoalCriterion> criteria) { public static boolean allPassed(List<GoalCriterion> criteria) {
return criteria != null && !criteria.isEmpty() return criteria != null && !criteria.isEmpty()
&& criteria.stream().allMatch(GoalCriterion::passed); && criteria.stream().allMatch(c -> c != null && c.passed() && hasEvidence(c));
} }
/** Criteria not yet passed (used for the continuation prompt + gap text). */ /** Criteria not yet passed (used for the continuation prompt + gap text). */
@ -96,7 +100,11 @@ public final class GoalCriteriaCodec {
if (criteria == null) { if (criteria == null) {
return List.of(); return List.of();
} }
return criteria.stream().filter(c -> !c.passed()).toList(); return criteria.stream().filter(c -> !c.passed() || !hasEvidence(c)).toList();
}
private static boolean hasEvidence(GoalCriterion criterion) {
return criterion.evidence() != null && !criterion.evidence().isBlank();
} }
/** Reassign stable ids {@code C1..Cn} in list order. */ /** Reassign stable ids {@code C1..Cn} in list order. */

View File

@ -2,6 +2,9 @@ package vip.mate.goal.model;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;
import java.util.List; import java.util.List;
@ -77,6 +80,23 @@ class GoalCriteriaCodecTest {
assertFalse(merged.get(0).passed()); assertFalse(merged.get(0).passed());
} }
@ParameterizedTest
@NullAndEmptySource
@ValueSource(strings = {" ", "\t\n"})
void blankEvidenceCannotPassNewOrPersistedCriteria(String evidence) {
var existing = List.of(new GoalCriterion("C1", "deliver report", false, ""));
var merged = GoalCriteriaCodec.merge(existing, List.of(
new GoalChecklistVerdict.CriterionVerdict("C1", true, evidence)));
assertFalse(merged.getFirst().passed());
assertFalse(GoalCriteriaCodec.allPassed(merged));
assertEquals(1, GoalCriteriaCodec.remaining(merged).size());
var persisted = List.of(new GoalCriterion("C1", "deliver report", true, evidence));
assertFalse(GoalCriteriaCodec.allPassed(persisted));
assertEquals(1, GoalCriteriaCodec.remaining(persisted).size());
assertFalse(GoalCriteriaCodec.merge(persisted, List.of()).getFirst().passed());
}
// ---------- allPassed / remaining ---------- // ---------- allPassed / remaining ----------
@Test @Test

View File

@ -94,6 +94,20 @@ class GoalEvaluationServiceTest {
when(chatModel.call(any(Prompt.class))).thenReturn(response); when(chatModel.call(any(Prompt.class))).thenReturn(response);
} }
@Test
void blankEvidenceVerdictCannotCompleteOrInflateProgress() {
stubChatResponse("""
{"criterionVerdicts":[
{"id":"C1","passed":true,"evidence":""},
{"id":"C2","passed":true,"evidence":null}],"summary":"done"}
""");
var result = svc.evaluate(goalWithCriteria(), List.of(), "All done");
assertFalse(result.completed());
assertEquals(0.0, result.score());
assertTrue(result.gap().contains("DNS configured"));
assertTrue(result.gap().contains("TLS enabled"));
}
// ==================== Pre-flight guards ==================== // ==================== Pre-flight guards ====================
@Test @Test