diff --git a/mateclaw-server/src/main/java/vip/mate/goal/model/GoalEvaluationResult.java b/mateclaw-server/src/main/java/vip/mate/goal/model/GoalEvaluationResult.java index d1fa4cdb..415e10b7 100644 --- a/mateclaw-server/src/main/java/vip/mate/goal/model/GoalEvaluationResult.java +++ b/mateclaw-server/src/main/java/vip/mate/goal/model/GoalEvaluationResult.java @@ -43,8 +43,8 @@ public record GoalEvaluationResult( public static final String DECISION_CONTINUE = "continue"; public static final String DECISION_FALLBACK = "fallback"; - /** Failure fallback used when the evaluator LLM call errors out. - * Does NOT charge eval_llm_calls_used. */ + /** Failure fallback for the "no call was made" cases (no goal, empty + * answer, no model). Does NOT charge eval_llm_calls_used. */ public static GoalEvaluationResult fallback(String reason) { return new GoalEvaluationResult( 0.0, "evaluator unavailable: " + reason, @@ -53,6 +53,18 @@ public record GoalEvaluationResult( List.of(), null); } + /** Failure fallback for cases where the evaluator LLM call already + * succeeded but its output was unusable (empty / unparseable). The call + * was really spent, so it charges {@code llmCallsConsumed = 1} and + * records the model + latency for accurate budget accounting. */ + public static GoalEvaluationResult fallbackAfterCall(String reason, String model, long latencyMs) { + return new GoalEvaluationResult( + 0.0, "evaluator unavailable: " + reason, + DECISION_FALLBACK, false, + model == null ? "" : model, 1, latencyMs, + List.of(), null); + } + public Map toMap() { Map m = new LinkedHashMap<>(); m.put("completionScore", score); diff --git a/mateclaw-server/src/main/java/vip/mate/goal/service/GoalEvaluationService.java b/mateclaw-server/src/main/java/vip/mate/goal/service/GoalEvaluationService.java index cd8a5e02..5aca7f1d 100644 --- a/mateclaw-server/src/main/java/vip/mate/goal/service/GoalEvaluationService.java +++ b/mateclaw-server/src/main/java/vip/mate/goal/service/GoalEvaluationService.java @@ -145,7 +145,8 @@ public class GoalEvaluationService implements Evaluator { String body = extractText(response); if (body == null || body.isBlank()) { log.warn("[GoalEvaluation] empty response from evaluator model={}", model.getModelName()); - return GoalEvaluationResult.fallback("empty_response"); + // The call was really spent — bill it. + return GoalEvaluationResult.fallbackAfterCall("empty_response", model.getModelName(), elapsed); } return bootstrap @@ -162,24 +163,31 @@ public class GoalEvaluationService implements Evaluator { /** * Generic SPI surface: judge whether {@code request.getResponseContent()} - * satisfies the objective in {@code request.getUserText()}. Used for - * standardization/testing; goal-aware callers use the - * {@link #evaluate(GoalEntity, List, String)} overload which carries the - * checklist context the request cannot. Detail rides in metadata. + * satisfies the objective stated in {@code request.getUserText()}. + * + *

The objective is wrapped as a single checklist criterion so the call + * runs in verdict mode (a real pass/fail judgement of the response), + * not bootstrap mode. {@code isPass()} is true only when that criterion is + * satisfied; detail rides in {@code metadata.criterionVerdicts}. + * + *

Goal-aware callers use the {@link #evaluate(GoalEntity, List, String)} + * overload, which carries the full multi-criterion checklist context the + * generic request cannot. */ @Override public EvaluationResponse evaluate(EvaluationRequest request) { + String objective = request.getUserText() != null ? request.getUserText() : ""; GoalEntity probe = new GoalEntity(); - probe.setTitle(request.getUserText()); - probe.setDescription(""); + probe.setTitle("Does the response satisfy the objective?"); + probe.setDescription(objective); + // One criterion = the objective -> non-empty criteria -> verdict mode. + probe.setCriteria(GoalCriteriaCodec.serialize( + List.of(new GoalCriterion("C1", objective, false, "")), objectMapper)); + GoalEvaluationResult r = evaluate(probe, List.of(), request.getResponseContent()); Map metadata = new LinkedHashMap<>(); metadata.put("decision", r.decision()); - if (r.bootstrapCriteria() != null) { - metadata.put("bootstrapCriteria", r.bootstrapCriteria()); - } else { - metadata.put("criterionVerdicts", r.criterionVerdicts()); - } + metadata.put("criterionVerdicts", r.criterionVerdicts()); return new EvaluationResponse(r.completed(), (float) r.score(), r.gap() == null ? "" : r.gap(), metadata); } @@ -267,16 +275,21 @@ public class GoalEvaluationService implements Evaluator { try { GoalCriteriaDraft dto = draftConverter.convert(stripFences(body)); if (dto == null || dto.criteria() == null || dto.criteria().isEmpty()) { - return GoalEvaluationResult.fallback("bootstrap_empty"); + return GoalEvaluationResult.fallbackAfterCall("bootstrap_empty", modelName, latencyMs); } List normalized = new ArrayList<>(); for (GoalCriterion c : dto.criteria()) { if (c != null && c.text() != null && !c.text().isBlank()) { normalized.add(new GoalCriterion("", c.text().trim(), false, "")); } + // Hard cap regardless of what the model returned — the prompt + // asks for <= MAX but a verbose model could exceed it. + if (normalized.size() >= MAX_BOOTSTRAP_CRITERIA) { + break; + } } if (normalized.isEmpty()) { - return GoalEvaluationResult.fallback("bootstrap_empty"); + return GoalEvaluationResult.fallbackAfterCall("bootstrap_empty", modelName, latencyMs); } normalized = GoalCriteriaCodec.reindex(normalized); // Bootstrap never judges completion: the checklist is freshly created. @@ -286,7 +299,7 @@ public class GoalEvaluationService implements Evaluator { List.of(), normalized); } catch (Exception e) { log.warn("[GoalEvaluation] bootstrap parse failed: {}", e.getMessage()); - return GoalEvaluationResult.fallback("parse_failed"); + return GoalEvaluationResult.fallbackAfterCall("parse_failed", modelName, latencyMs); } } @@ -314,7 +327,7 @@ public class GoalEvaluationService implements Evaluator { deltas, null); } catch (Exception e) { log.warn("[GoalEvaluation] verdict parse failed: {}", e.getMessage()); - return GoalEvaluationResult.fallback("parse_failed"); + return GoalEvaluationResult.fallbackAfterCall("parse_failed", modelName, latencyMs); } } diff --git a/mateclaw-server/src/main/java/vip/mate/goal/service/GoalFollowupService.java b/mateclaw-server/src/main/java/vip/mate/goal/service/GoalFollowupService.java index 9410403c..2bf5066f 100644 --- a/mateclaw-server/src/main/java/vip/mate/goal/service/GoalFollowupService.java +++ b/mateclaw-server/src/main/java/vip/mate/goal/service/GoalFollowupService.java @@ -37,7 +37,7 @@ public class GoalFollowupService { *

  • {@code allow-auto-followup} runtime hard gate (operator kill * switch; overrides per-goal flag).
    • *
  • Per-goal {@code autoFollowupEnabled}.
    • - *
  • Evaluator decision is "continue" with score < 0.95.
    • + *
  • Evaluator decision is "continue" (not all criteria passed).
    • *
  • Cooldown since the last follow-up has elapsed.
    • *
  • turn_budget has at least one slot left after this turn.
    • *
  • (agent + eval) LLM calls below 90% of llm_call_budget.
    • @@ -49,10 +49,13 @@ public class GoalFollowupService { // Runtime hard gate first — overrides any per-goal flag. if (!properties.isAllowAutoFollowup()) return Optional.empty(); if (!Boolean.TRUE.equals(goal.getAutoFollowupEnabled())) return Optional.empty(); + // Completion is deterministic now: the evaluator sets decision=completed + // only when every checklist criterion passed. Anything still "continue" + // has remaining work regardless of the numeric score, so there is no + // score threshold here — a 20/21 goal (score 0.95) must still follow up. if (!GoalEvaluationResult.DECISION_CONTINUE.equals(result.decision())) { return Optional.empty(); } - if (result.score() >= 0.95) return Optional.empty(); // Cooldown — last_followup_at recorded by recordFollowupInjected(). Integer cooldownSec = goal.getFollowupCooldownSeconds(); diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/GoalManagementTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/GoalManagementTool.java index 06a796e1..94195a1c 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/GoalManagementTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/GoalManagementTool.java @@ -146,10 +146,14 @@ public class GoalManagementTool { } @Tool(description = """ - Explicitly mark the active goal as completed. Use ONLY when all \ - exit criteria are satisfied (e.g. tests passed, feature deployed, \ - user confirmed). The runtime evaluator will also mark goals \ - completed automatically when score >= 0.95 — prefer that path.""") + Explicitly mark the active goal as completed. Use ONLY when EVERY \ + checklist criterion is genuinely satisfied with concrete evidence \ + in the conversation (e.g. tests actually passed, feature actually \ + deployed, user confirmed). Do NOT call this to close out work that \ + is unfinished, blocked, or impossible. In normal operation you do \ + not need this tool at all: the runtime evaluator marks the goal \ + completed automatically once all checklist criteria pass — prefer \ + that path and just keep working.""") public String completeGoal(@Nullable ToolContext ctx) { if (!properties.isEnabled()) return errorJson("Goal subsystem is disabled"); GoalEntity goal = resolveActive(ctx); diff --git a/mateclaw-server/src/test/java/vip/mate/goal/service/GoalEvaluationServiceTest.java b/mateclaw-server/src/test/java/vip/mate/goal/service/GoalEvaluationServiceTest.java index bf306266..0a4cd348 100644 --- a/mateclaw-server/src/test/java/vip/mate/goal/service/GoalEvaluationServiceTest.java +++ b/mateclaw-server/src/test/java/vip/mate/goal/service/GoalEvaluationServiceTest.java @@ -12,6 +12,8 @@ import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.Generation; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.evaluation.EvaluationRequest; +import org.springframework.ai.evaluation.EvaluationResponse; import org.springframework.retry.support.RetryTemplate; import vip.mate.goal.config.GoalProperties; import vip.mate.goal.model.GoalEntity; @@ -188,7 +190,8 @@ class GoalEvaluationServiceTest { stubChatResponse("I think it's about 60% done."); GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x"); assertEquals(GoalEvaluationResult.DECISION_FALLBACK, r.decision()); - assertEquals(0, r.llmCallsConsumed()); + // The call was made and returned garbage — it still spends one call. + assertEquals(1, r.llmCallsConsumed()); } @Test @@ -249,4 +252,54 @@ class GoalEvaluationServiceTest { assertFalse(r.completed()); assertTrue(r.gap().contains("evaluator unavailable")); } + + // ==================== Post-call billing (failed output still spends a call) ==================== + + @Test + void emptyResponseAfterCall_billsOneLlmCall() { + stubChatResponse(" "); + GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x"); + assertEquals(GoalEvaluationResult.DECISION_FALLBACK, r.decision()); + assertEquals(1, r.llmCallsConsumed(), "a spent-but-empty evaluator call must charge 1"); + assertEquals("qwen-turbo", r.evaluatorModel()); + } + + @Test + void parseFailureAfterCall_billsOneLlmCall() { + stubChatResponse("{\"criteria\": }"); // malformed -> parse fail (call already spent) + GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x"); + assertEquals(GoalEvaluationResult.DECISION_FALLBACK, r.decision()); + assertEquals(1, r.llmCallsConsumed()); + } + + // ==================== Bootstrap criteria cap ==================== + + @Test + void bootstrap_capsCriteriaAtMax() { + StringBuilder sb = new StringBuilder("{\"criteria\":["); + for (int i = 0; i < 12; i++) { + if (i > 0) sb.append(','); + sb.append("{\"text\":\"criterion ").append(i).append("\"}"); + } + sb.append("]}"); + stubChatResponse(sb.toString()); + GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x"); + assertNotNull(r.bootstrapCriteria()); + assertTrue(r.bootstrapCriteria().size() <= 8, + "bootstrap must cap criteria at MAX_BOOTSTRAP_CRITERIA; got " + r.bootstrapCriteria().size()); + } + + // ==================== Evaluator SPI ==================== + + @Test + void evaluatorSpi_judgesResponseInVerdictMode() { + // The objective is wrapped as criterion C1; a verdict JSON marking C1 + // passed must surface as isPass()=true with score 1.0 — NOT a bootstrap. + stubChatResponse("{\"criterionVerdicts\":[{\"id\":\"C1\",\"passed\":true,\"evidence\":\"matches\"}],\"summary\":\"ok\"}"); + EvaluationResponse resp = svc.evaluate( + new EvaluationRequest("Return a greeting", "Hello, world!")); + assertTrue(resp.isPass()); + assertEquals(1.0f, resp.getScore(), 1e-6); + assertTrue(resp.getMetadata().containsKey("criterionVerdicts")); + } } diff --git a/mateclaw-server/src/test/java/vip/mate/goal/service/GoalFollowupServiceTest.java b/mateclaw-server/src/test/java/vip/mate/goal/service/GoalFollowupServiceTest.java index e9c9c507..a9fc0f91 100644 --- a/mateclaw-server/src/test/java/vip/mate/goal/service/GoalFollowupServiceTest.java +++ b/mateclaw-server/src/test/java/vip/mate/goal/service/GoalFollowupServiceTest.java @@ -75,11 +75,14 @@ class GoalFollowupServiceTest { } @Test - void highScore_returnsEmpty() { + void highScoreButStillContinue_followsUp() { + // No score gate: completion is decided by decision==completed, not a + // numeric threshold. A 20/21 goal (score ~0.95) that is still "continue" + // has remaining criteria and MUST follow up. Optional out = svc.maybeBuildFollowup( goal(true), res(0.96, GoalEvaluationResult.DECISION_CONTINUE)); - assertTrue(out.isEmpty()); + assertTrue(out.isPresent()); } @Test diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 34628a1f..12239f0c 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -1316,8 +1316,8 @@ export interface Goal { lastFollowupAt?: string | null createTime: string updateTime: string - /** Parsed checklist; always an array on the wire (empty when none). */ - criteria?: GoalCriterion[] + /** Parsed checklist; the backend always sends an array (empty when none). */ + criteria: GoalCriterion[] } export interface GoalEvent { diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index f44e34d7..ffaf38d9 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -3842,6 +3842,7 @@ export default { inlinePromptAccept: 'Yes', inlinePromptDecline: 'No thanks', autoFollowup: 'Auto continuation', + sidebarActive: 'This conversation has an active goal', completedTitle: 'Goal completed', completedDetail: 'Stored in long-term memory; askable later', exhaustedTitle: 'Budget exhausted', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 58d7c2b5..8844f702 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -3934,6 +3934,7 @@ export default { inlinePromptAccept: '好', inlinePromptDecline: '不用', autoFollowup: '自动延续', + sidebarActive: '此对话有正在进行的目标', completedTitle: '目标达成', completedDetail: '已存入长期记忆，下次问起能找回来', exhaustedTitle: '这次的预算用完了',