opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(goal): drop score gate, bill failed evaluator calls, fix Evaluator SPI + tool prompt + bootstrap cap

Browse Source
This commit is contained in:
matevip 2026-06-03 21:19:27 +08:00
parent a756da7817
commit 3883b68cca
9 changed files with 119 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mateclaw-server/src/main/java/vip/mate/goal/model/GoalEvaluationResult.java
View File

@ -43,8 +43,8 @@ public record GoalEvaluationResult(
public static final String DECISION_CONTINUE = "continue";
public static final String DECISION_FALLBACK = "fallback";
/** Failure fallback used when the evaluator LLM call errors out.
* Does NOT charge eval_llm_calls_used. */
/** Failure fallback for the "no call was made" cases (no goal, empty
* answer, no model). Does NOT charge eval_llm_calls_used. */
public static GoalEvaluationResult fallback(String reason) {
return new GoalEvaluationResult(
0.0, "evaluator unavailable: " + reason,
@ -53,6 +53,18 @@ public record GoalEvaluationResult(
List.of(), null);
}
/** Failure fallback for cases where the evaluator LLM call already
* succeeded but its output was unusable (empty / unparseable). The call
* was really spent, so it charges {@code llmCallsConsumed = 1} and
* records the model + latency for accurate budget accounting. */
public static GoalEvaluationResult fallbackAfterCall(String reason, String model, long latencyMs) {
return new GoalEvaluationResult(
0.0, "evaluator unavailable: " + reason,
DECISION_FALLBACK, false,
model == null ? "" : model, 1, latencyMs,
List.of(), null);
}
public Map<String, Object> toMap() {
Map<String, Object> m = new LinkedHashMap<>();
m.put("completionScore", score);

45
mateclaw-server/src/main/java/vip/mate/goal/service/GoalEvaluationService.java
View File

@ -145,7 +145,8 @@ public class GoalEvaluationService implements Evaluator {
String body = extractText(response);
if (body == null || body.isBlank()) {
log.warn("[GoalEvaluation] empty response from evaluator model={}", model.getModelName());
return GoalEvaluationResult.fallback("empty_response");
// The call was really spent — bill it.
return GoalEvaluationResult.fallbackAfterCall("empty_response", model.getModelName(), elapsed);
}
return bootstrap
@ -162,24 +163,31 @@ public class GoalEvaluationService implements Evaluator {
/**
* Generic SPI surface: judge whether {@code request.getResponseContent()}
* satisfies the objective in {@code request.getUserText()}. Used for
* standardization/testing; goal-aware callers use the
* {@link #evaluate(GoalEntity, List, String)} overload which carries the
* checklist context the request cannot. Detail rides in metadata.
* satisfies the objective stated in {@code request.getUserText()}.
*
* <p>The objective is wrapped as a single checklist criterion so the call
* runs in <b>verdict</b> mode (a real pass/fail judgement of the response),
* not bootstrap mode. {@code isPass()} is true only when that criterion is
* satisfied; detail rides in {@code metadata.criterionVerdicts}.
*
* <p>Goal-aware callers use the {@link #evaluate(GoalEntity, List, String)}
* overload, which carries the full multi-criterion checklist context the
* generic request cannot.
*/
@Override
public EvaluationResponse evaluate(EvaluationRequest request) {
String objective = request.getUserText() != null ? request.getUserText() : "";
GoalEntity probe = new GoalEntity();
probe.setTitle(request.getUserText());
probe.setDescription("");
probe.setTitle("Does the response satisfy the objective?");
probe.setDescription(objective);
// One criterion = the objective -> non-empty criteria -> verdict mode.
probe.setCriteria(GoalCriteriaCodec.serialize(
List.of(new GoalCriterion("C1", objective, false, "")), objectMapper));
GoalEvaluationResult r = evaluate(probe, List.of(), request.getResponseContent());
Map<String, Object> metadata = new LinkedHashMap<>();
metadata.put("decision", r.decision());
if (r.bootstrapCriteria() != null) {
metadata.put("bootstrapCriteria", r.bootstrapCriteria());
} else {
metadata.put("criterionVerdicts", r.criterionVerdicts());
}
metadata.put("criterionVerdicts", r.criterionVerdicts());
return new EvaluationResponse(r.completed(), (float) r.score(),
r.gap() == null ? "" : r.gap(), metadata);
}
@ -267,16 +275,21 @@ public class GoalEvaluationService implements Evaluator {
try {
GoalCriteriaDraft dto = draftConverter.convert(stripFences(body));
if (dto == null || dto.criteria() == null || dto.criteria().isEmpty()) {
return GoalEvaluationResult.fallback("bootstrap_empty");
return GoalEvaluationResult.fallbackAfterCall("bootstrap_empty", modelName, latencyMs);
}
List<GoalCriterion> normalized = new ArrayList<>();
for (GoalCriterion c : dto.criteria()) {
if (c != null && c.text() != null && !c.text().isBlank()) {
normalized.add(new GoalCriterion("", c.text().trim(), false, ""));
}
// Hard cap regardless of what the model returned — the prompt
// asks for <= MAX but a verbose model could exceed it.
if (normalized.size() >= MAX_BOOTSTRAP_CRITERIA) {
break;
}
}
if (normalized.isEmpty()) {
return GoalEvaluationResult.fallback("bootstrap_empty");
return GoalEvaluationResult.fallbackAfterCall("bootstrap_empty", modelName, latencyMs);
}
normalized = GoalCriteriaCodec.reindex(normalized);
// Bootstrap never judges completion: the checklist is freshly created.
@ -286,7 +299,7 @@ public class GoalEvaluationService implements Evaluator {
List.of(), normalized);
} catch (Exception e) {
log.warn("[GoalEvaluation] bootstrap parse failed: {}", e.getMessage());
return GoalEvaluationResult.fallback("parse_failed");
return GoalEvaluationResult.fallbackAfterCall("parse_failed", modelName, latencyMs);
}
}
@ -314,7 +327,7 @@ public class GoalEvaluationService implements Evaluator {
deltas, null);
} catch (Exception e) {
log.warn("[GoalEvaluation] verdict parse failed: {}", e.getMessage());
return GoalEvaluationResult.fallback("parse_failed");
return GoalEvaluationResult.fallbackAfterCall("parse_failed", modelName, latencyMs);
}
}

7
mateclaw-server/src/main/java/vip/mate/goal/service/GoalFollowupService.java
View File

@ -37,7 +37,7 @@ public class GoalFollowupService {
* <li>{@code allow-auto-followup} runtime hard gate (operator kill
* switch; overrides per-goal flag).</li>
* <li>Per-goal {@code autoFollowupEnabled}.</li>
* <li>Evaluator decision is "continue" with score &lt; 0.95.</li>
* <li>Evaluator decision is "continue" (not all criteria passed).</li>
* <li>Cooldown since the last follow-up has elapsed.</li>
* <li>turn_budget has at least one slot left after this turn.</li>
* <li>(agent + eval) LLM calls below 90% of llm_call_budget.</li>
@ -49,10 +49,13 @@ public class GoalFollowupService {
// Runtime hard gate first — overrides any per-goal flag.
if (!properties.isAllowAutoFollowup()) return Optional.empty();
if (!Boolean.TRUE.equals(goal.getAutoFollowupEnabled())) return Optional.empty();
// Completion is deterministic now: the evaluator sets decision=completed
// only when every checklist criterion passed. Anything still "continue"
// has remaining work regardless of the numeric score, so there is no
// score threshold here — a 20/21 goal (score 0.95) must still follow up.
if (!GoalEvaluationResult.DECISION_CONTINUE.equals(result.decision())) {
return Optional.empty();
}
if (result.score() >= 0.95) return Optional.empty();
// Cooldown — last_followup_at recorded by recordFollowupInjected().
Integer cooldownSec = goal.getFollowupCooldownSeconds();

12
mateclaw-server/src/main/java/vip/mate/tool/builtin/GoalManagementTool.java
View File

@ -146,10 +146,14 @@ public class GoalManagementTool {
}
@Tool(description = """
Explicitly mark the active goal as completed. Use ONLY when all \
exit criteria are satisfied (e.g. tests passed, feature deployed, \
user confirmed). The runtime evaluator will also mark goals \
completed automatically when score >= 0.95 — prefer that path.""")
Explicitly mark the active goal as completed. Use ONLY when EVERY \
checklist criterion is genuinely satisfied with concrete evidence \
in the conversation (e.g. tests actually passed, feature actually \
deployed, user confirmed). Do NOT call this to close out work that \
is unfinished, blocked, or impossible. In normal operation you do \
not need this tool at all: the runtime evaluator marks the goal \
completed automatically once all checklist criteria pass — prefer \
that path and just keep working.""")
public String completeGoal(@Nullable ToolContext ctx) {
if (!properties.isEnabled()) return errorJson("Goal subsystem is disabled");
GoalEntity goal = resolveActive(ctx);

55
mateclaw-server/src/test/java/vip/mate/goal/service/GoalEvaluationServiceTest.java
View File

@ -12,6 +12,8 @@ import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.evaluation.EvaluationRequest;
import org.springframework.ai.evaluation.EvaluationResponse;
import org.springframework.retry.support.RetryTemplate;
import vip.mate.goal.config.GoalProperties;
import vip.mate.goal.model.GoalEntity;
@ -188,7 +190,8 @@ class GoalEvaluationServiceTest {
stubChatResponse("I think it's about 60% done.");
GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x");
assertEquals(GoalEvaluationResult.DECISION_FALLBACK, r.decision());
assertEquals(0, r.llmCallsConsumed());
// The call was made and returned garbage — it still spends one call.
assertEquals(1, r.llmCallsConsumed());
}
@Test
@ -249,4 +252,54 @@ class GoalEvaluationServiceTest {
assertFalse(r.completed());
assertTrue(r.gap().contains("evaluator unavailable"));
}
// ==================== Post-call billing (failed output still spends a call) ====================
@Test
void emptyResponseAfterCall_billsOneLlmCall() {
stubChatResponse(" ");
GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x");
assertEquals(GoalEvaluationResult.DECISION_FALLBACK, r.decision());
assertEquals(1, r.llmCallsConsumed(), "a spent-but-empty evaluator call must charge 1");
assertEquals("qwen-turbo", r.evaluatorModel());
}
@Test
void parseFailureAfterCall_billsOneLlmCall() {
stubChatResponse("{\"criteria\": }"); // malformed -> parse fail (call already spent)
GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x");
assertEquals(GoalEvaluationResult.DECISION_FALLBACK, r.decision());
assertEquals(1, r.llmCallsConsumed());
}
// ==================== Bootstrap criteria cap ====================
@Test
void bootstrap_capsCriteriaAtMax() {
StringBuilder sb = new StringBuilder("{\"criteria\":[");
for (int i = 0; i < 12; i++) {
if (i > 0) sb.append(',');
sb.append("{\"text\":\"criterion ").append(i).append("\"}");
}
sb.append("]}");
stubChatResponse(sb.toString());
GoalEvaluationResult r = svc.evaluate(goal(), List.of(), "x");
assertNotNull(r.bootstrapCriteria());
assertTrue(r.bootstrapCriteria().size() <= 8,
"bootstrap must cap criteria at MAX_BOOTSTRAP_CRITERIA; got " + r.bootstrapCriteria().size());
}
// ==================== Evaluator SPI ====================
@Test
void evaluatorSpi_judgesResponseInVerdictMode() {
// The objective is wrapped as criterion C1; a verdict JSON marking C1
// passed must surface as isPass()=true with score 1.0 — NOT a bootstrap.
stubChatResponse("{\"criterionVerdicts\":[{\"id\":\"C1\",\"passed\":true,\"evidence\":\"matches\"}],\"summary\":\"ok\"}");
EvaluationResponse resp = svc.evaluate(
new EvaluationRequest("Return a greeting", "Hello, world!"));
assertTrue(resp.isPass());
assertEquals(1.0f, resp.getScore(), 1e-6);
assertTrue(resp.getMetadata().containsKey("criterionVerdicts"));
}
}

7
mateclaw-server/src/test/java/vip/mate/goal/service/GoalFollowupServiceTest.java
View File

@ -75,11 +75,14 @@ class GoalFollowupServiceTest {
}
@Test
void highScore_returnsEmpty() {
void highScoreButStillContinue_followsUp() {
// No score gate: completion is decided by decision==completed, not a
// numeric threshold. A 20/21 goal (score ~0.95) that is still "continue"
// has remaining criteria and MUST follow up.
Optional<String> out = svc.maybeBuildFollowup(
goal(true),
res(0.96, GoalEvaluationResult.DECISION_CONTINUE));
assertTrue(out.isEmpty());
assertTrue(out.isPresent());
}
@Test

4
mateclaw-ui/src/api/index.ts
View File

@ -1316,8 +1316,8 @@ export interface Goal {
lastFollowupAt?: string | null
createTime: string
updateTime: string
/** Parsed checklist; always an array on the wire (empty when none). */
criteria?: GoalCriterion[]
/** Parsed checklist; the backend always sends an array (empty when none). */
criteria: GoalCriterion[]
}
export interface GoalEvent {

1
mateclaw-ui/src/i18n/locales/en-US.ts
View File

@ -3842,6 +3842,7 @@ export default {
inlinePromptAccept: 'Yes',
inlinePromptDecline: 'No thanks',
autoFollowup: 'Auto continuation',
sidebarActive: 'This conversation has an active goal',
completedTitle: 'Goal completed',
completedDetail: 'Stored in long-term memory; askable later',
exhaustedTitle: 'Budget exhausted',

1
mateclaw-ui/src/i18n/locales/zh-CN.ts
View File

@ -3934,6 +3934,7 @@ export default {
inlinePromptAccept: '好',
inlinePromptDecline: '不用',
autoFollowup: '自动延续',
sidebarActive: '此对话有正在进行的目标',
completedTitle: '目标达成',
completedDetail: '已存入长期记忆，下次问起能找回来',
exhaustedTitle: '这次的预算用完了',