fix(goal): reject checklist passes without evidence

This commit is contained in:
mateaix 2026-09-13 20:38:19 +08:00
parent 00d774fb85
commit 88cd78af5e
3 changed files with 50 additions and 8 deletions

View File

@ -58,8 +58,8 @@ public final class GoalCriteriaCodec {
/**
* Merge a per-round verdict delta into the full checklist by id. Criteria
* absent from the delta are preserved unchanged; the criterion text is
* always kept from the existing item (the verdict never carries text).
* absent from the delta retain their state unless their pass lacks evidence.
* The criterion text is always kept from the existing item (the verdict never carries text).
*/
public static List<GoalCriterion> merge(List<GoalCriterion> existing,
List<GoalChecklistVerdict.CriterionVerdict> verdicts) {
@ -77,18 +77,22 @@ public final class GoalCriteriaCodec {
List<GoalCriterion> merged = new ArrayList<>(existing.size());
for (GoalCriterion c : existing) {
GoalChecklistVerdict.CriterionVerdict v = byId.get(c.id());
merged.add(v == null
? c
GoalCriterion candidate = v == null ? c
: new GoalCriterion(c.id(), c.text(), v.passed(),
v.evidence() != null ? v.evidence() : ""));
v.evidence() != null ? v.evidence() : "");
// A model boolean alone cannot satisfy even the legacy semantic
// checklist. This checks presence, not truth or execution provenance.
merged.add(candidate.passed() && !hasEvidence(candidate)
? new GoalCriterion(candidate.id(), candidate.text(), false, candidate.evidence())
: candidate);
}
return merged;
}
/** True only when the list is non-empty and every criterion is passed. */
/** True only when the list is non-empty and every criterion is passed with nonblank evidence. */
public static boolean allPassed(List<GoalCriterion> criteria) {
return criteria != null && !criteria.isEmpty()
&& criteria.stream().allMatch(GoalCriterion::passed);
&& criteria.stream().allMatch(c -> c != null && c.passed() && hasEvidence(c));
}
/** Criteria not yet passed (used for the continuation prompt + gap text). */
@ -96,7 +100,11 @@ public final class GoalCriteriaCodec {
if (criteria == null) {
return List.of();
}
return criteria.stream().filter(c -> !c.passed()).toList();
return criteria.stream().filter(c -> !c.passed() || !hasEvidence(c)).toList();
}
private static boolean hasEvidence(GoalCriterion criterion) {
return criterion.evidence() != null && !criterion.evidence().isBlank();
}
/** Reassign stable ids {@code C1..Cn} in list order. */

View File

@ -2,6 +2,9 @@ package vip.mate.goal.model;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;
import java.util.List;
@ -77,6 +80,23 @@ class GoalCriteriaCodecTest {
assertFalse(merged.get(0).passed());
}
@ParameterizedTest
@NullAndEmptySource
@ValueSource(strings = {" ", "\t\n"})
void blankEvidenceCannotPassNewOrPersistedCriteria(String evidence) {
var existing = List.of(new GoalCriterion("C1", "deliver report", false, ""));
var merged = GoalCriteriaCodec.merge(existing, List.of(
new GoalChecklistVerdict.CriterionVerdict("C1", true, evidence)));
assertFalse(merged.getFirst().passed());
assertFalse(GoalCriteriaCodec.allPassed(merged));
assertEquals(1, GoalCriteriaCodec.remaining(merged).size());
var persisted = List.of(new GoalCriterion("C1", "deliver report", true, evidence));
assertFalse(GoalCriteriaCodec.allPassed(persisted));
assertEquals(1, GoalCriteriaCodec.remaining(persisted).size());
assertFalse(GoalCriteriaCodec.merge(persisted, List.of()).getFirst().passed());
}
// ---------- allPassed / remaining ----------
@Test

View File

@ -94,6 +94,20 @@ class GoalEvaluationServiceTest {
when(chatModel.call(any(Prompt.class))).thenReturn(response);
}
@Test
void blankEvidenceVerdictCannotCompleteOrInflateProgress() {
stubChatResponse("""
{"criterionVerdicts":[
{"id":"C1","passed":true,"evidence":""},
{"id":"C2","passed":true,"evidence":null}],"summary":"done"}
""");
var result = svc.evaluate(goalWithCriteria(), List.of(), "All done");
assertFalse(result.completed());
assertEquals(0.0, result.score());
assertTrue(result.gap().contains("DNS configured"));
assertTrue(result.gap().contains("TLS enabled"));
}
// ==================== Pre-flight guards ====================
@Test