mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 19:23:42 +08:00
fix(cron): harden long-task reliability
This commit is contained in:
parent
b706f9a610
commit
bd41a71826
@ -1410,6 +1410,17 @@ public class ConversationWindowManager {
|
|||||||
ChatResponse response = chatModel.call(new Prompt(promptMessages, options));
|
ChatResponse response = chatModel.call(new Prompt(promptMessages, options));
|
||||||
if (response != null && response.getResult() != null
|
if (response != null && response.getResult() != null
|
||||||
&& response.getResult().getOutput() != null) {
|
&& response.getResult().getOutput() != null) {
|
||||||
|
String finishReason = response.getResult().getMetadata() != null
|
||||||
|
? response.getResult().getMetadata().getFinishReason() : null;
|
||||||
|
if ("length".equalsIgnoreCase(finishReason)) {
|
||||||
|
// A non-empty response can still be structurally incomplete
|
||||||
|
// when the provider exhausts max tokens. Persisting it would
|
||||||
|
// poison every later iterative summary.
|
||||||
|
log.warn("[ConversationWindow] LLM 摘要因 token 上限被截断,丢弃结果, conv={}",
|
||||||
|
conversationId);
|
||||||
|
setSummaryCooldown(conversationId);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
String summary = response.getResult().getOutput().getText();
|
String summary = response.getResult().getOutput().getText();
|
||||||
if (summary != null && !summary.isBlank()) {
|
if (summary != null && !summary.isBlank()) {
|
||||||
// 成功:保存摘要供下次迭代更新,清除冷却
|
// 成功:保存摘要供下次迭代更新,清除冷却
|
||||||
|
|||||||
@ -96,36 +96,44 @@ public abstract class AbstractCronResultDelivery implements CronResultDelivery {
|
|||||||
// ---------- SQL state-machine helpers ----------
|
// ---------- SQL state-machine helpers ----------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Atomic SQL CAS: transition delivery_status from {@code NONE} or
|
* Atomic SQL CAS: transition delivery_status from {@code NONE} (or legacy
|
||||||
* {@code PENDING} → {@code PENDING}. Returns true iff this instance won
|
* {@code NULL}) to {@code PENDING}. An already-pending row is owned by the
|
||||||
* the race. NONE-eligibility lets fresh runs claim without a separate
|
* worker that claimed it and must never be claimable again.
|
||||||
* "first-time" branch; PENDING-eligibility covers the rare same-instance
|
|
||||||
* retry inside the listener.
|
|
||||||
*
|
*
|
||||||
* <p>SQL semantics gotcha: {@code IN (...)} never matches NULL. Legacy
|
* <p>SQL semantics gotcha: {@code IN (...)} never matches NULL. Legacy
|
||||||
* rows from before V57 (pre-RFC) may have null delivery_status, so the
|
* rows from before V57 (pre-RFC) may have null delivery_status, so the
|
||||||
* predicate explicitly tests {@code IS NULL OR IN (NONE, PENDING)} via
|
* predicate explicitly tests {@code IS NULL OR = NONE} via
|
||||||
* a nested OR group rather than putting null inside the IN list.
|
* a nested OR group rather than putting null inside the IN list.
|
||||||
*/
|
*/
|
||||||
private boolean claimRun(CronJobRunEntity run) {
|
private boolean claimRun(CronJobRunEntity run) {
|
||||||
return runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
return runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getId, run.getId())
|
.eq(CronJobRunEntity::getId, run.getId())
|
||||||
.and(w -> w.isNull(CronJobRunEntity::getDeliveryStatus)
|
.and(w -> w.isNull(CronJobRunEntity::getDeliveryStatus)
|
||||||
.or().in(CronJobRunEntity::getDeliveryStatus, "NONE", "PENDING"))
|
.or().eq(CronJobRunEntity::getDeliveryStatus, "NONE"))
|
||||||
.set(CronJobRunEntity::getDeliveryStatus, "PENDING")) == 1;
|
.set(CronJobRunEntity::getDeliveryStatus, "PENDING")) == 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markDelivered(CronJobRunEntity run, DeliveryOutcome o) {
|
private void markDelivered(CronJobRunEntity run, DeliveryOutcome o) {
|
||||||
runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
int updated = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getId, run.getId())
|
.eq(CronJobRunEntity::getId, run.getId())
|
||||||
|
.eq(CronJobRunEntity::getDeliveryStatus, "PENDING")
|
||||||
.set(CronJobRunEntity::getDeliveryStatus, "DELIVERED")
|
.set(CronJobRunEntity::getDeliveryStatus, "DELIVERED")
|
||||||
.set(CronJobRunEntity::getDeliveryTarget, o.target()));
|
.set(CronJobRunEntity::getDeliveryTarget, o.target()));
|
||||||
|
if (updated == 0) {
|
||||||
|
log.warn("[CronDelivery] Run {} lost its PENDING fence before success was persisted",
|
||||||
|
run.getId());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void markNotDelivered(CronJobRunEntity run, Exception e) {
|
private void markNotDelivered(CronJobRunEntity run, Exception e) {
|
||||||
runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
int updated = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getId, run.getId())
|
.eq(CronJobRunEntity::getId, run.getId())
|
||||||
|
.eq(CronJobRunEntity::getDeliveryStatus, "PENDING")
|
||||||
.set(CronJobRunEntity::getDeliveryStatus, "NOT_DELIVERED")
|
.set(CronJobRunEntity::getDeliveryStatus, "NOT_DELIVERED")
|
||||||
.set(CronJobRunEntity::getDeliveryError, StrUtil.maxLength(e.getMessage(), 500)));
|
.set(CronJobRunEntity::getDeliveryError, StrUtil.maxLength(e.getMessage(), 500)));
|
||||||
|
if (updated == 0) {
|
||||||
|
log.warn("[CronDelivery] Run {} lost its PENDING fence before failure was persisted",
|
||||||
|
run.getId());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -21,7 +21,7 @@ import java.time.LocalDateTime;
|
|||||||
* {@code NOT_DELIVERED} with {@code stale-pending-timeout} reason.
|
* {@code NOT_DELIVERED} with {@code stale-pending-timeout} reason.
|
||||||
* Covers listener crashes / OOMs / forced kills after a successful
|
* Covers listener crashes / OOMs / forced kills after a successful
|
||||||
* {@code claimRun()} but before {@code markDelivered}.</li>
|
* {@code claimRun()} but before {@code markDelivered}.</li>
|
||||||
* <li>{@code status='running'} older than 30 min → mark {@code failed}
|
* <li>{@code status='running'} without a heartbeat for 2 min → mark {@code failed}
|
||||||
* with {@code stale-running-timeout}. Covers
|
* with {@code stale-running-timeout}. Covers
|
||||||
* {@code CronJobLifecycleService.markRunFailed()} itself failing under
|
* {@code CronJobLifecycleService.markRunFailed()} itself failing under
|
||||||
* DB jitter (the LLM call already terminated by then).</li>
|
* DB jitter (the LLM call already terminated by then).</li>
|
||||||
@ -38,7 +38,7 @@ public class CronRunStaleCleanup {
|
|||||||
private final CronJobRunMapper runMapper;
|
private final CronJobRunMapper runMapper;
|
||||||
|
|
||||||
private static final Duration DELIVERY_STALE = Duration.ofMinutes(15);
|
private static final Duration DELIVERY_STALE = Duration.ofMinutes(15);
|
||||||
private static final Duration RUN_STALE = Duration.ofMinutes(30);
|
private static final Duration RUN_STALE = Duration.ofMinutes(2);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RFC-03 Lane G2: in a multi-instance deployment, the sweep is purely
|
* RFC-03 Lane G2: in a multi-instance deployment, the sweep is purely
|
||||||
@ -62,7 +62,7 @@ public class CronRunStaleCleanup {
|
|||||||
|
|
||||||
int staleRunning = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
int staleRunning = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getStatus, "running")
|
.eq(CronJobRunEntity::getStatus, "running")
|
||||||
.lt(CronJobRunEntity::getStartedAt, now.minus(RUN_STALE))
|
.apply("COALESCE(heartbeat_at, started_at) < {0}", now.minus(RUN_STALE))
|
||||||
.set(CronJobRunEntity::getStatus, "failed")
|
.set(CronJobRunEntity::getStatus, "failed")
|
||||||
.set(CronJobRunEntity::getFinishedAt, now)
|
.set(CronJobRunEntity::getFinishedAt, now)
|
||||||
.set(CronJobRunEntity::getErrorMessage, "stale-running-timeout"));
|
.set(CronJobRunEntity::getErrorMessage, "stale-running-timeout"));
|
||||||
|
|||||||
@ -71,7 +71,9 @@ public class CronJobLifecycleService {
|
|||||||
run.setConversationId(conversationId);
|
run.setConversationId(conversationId);
|
||||||
run.setStatus("running");
|
run.setStatus("running");
|
||||||
run.setTriggerType(triggerType != null ? triggerType : "scheduled");
|
run.setTriggerType(triggerType != null ? triggerType : "scheduled");
|
||||||
run.setStartedAt(LocalDateTime.now());
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
run.setStartedAt(now);
|
||||||
|
run.setHeartbeatAt(now);
|
||||||
run.setDeliveryStatus("NONE");
|
run.setDeliveryStatus("NONE");
|
||||||
runMapper.insert(run);
|
runMapper.insert(run);
|
||||||
|
|
||||||
@ -130,6 +132,7 @@ public class CronJobLifecycleService {
|
|||||||
String message = error != null && error.getMessage() != null ? error.getMessage() : "unknown error";
|
String message = error != null && error.getMessage() != null ? error.getMessage() : "unknown error";
|
||||||
runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getId, run.getId())
|
.eq(CronJobRunEntity::getId, run.getId())
|
||||||
|
.eq(CronJobRunEntity::getStatus, "running")
|
||||||
.set(CronJobRunEntity::getStatus, "failed")
|
.set(CronJobRunEntity::getStatus, "failed")
|
||||||
.set(CronJobRunEntity::getFinishedAt, LocalDateTime.now())
|
.set(CronJobRunEntity::getFinishedAt, LocalDateTime.now())
|
||||||
.set(CronJobRunEntity::getErrorMessage, StrUtil.maxLength(message, 1000)));
|
.set(CronJobRunEntity::getErrorMessage, StrUtil.maxLength(message, 1000)));
|
||||||
@ -148,7 +151,9 @@ public class CronJobLifecycleService {
|
|||||||
run.setConversationId(null);
|
run.setConversationId(null);
|
||||||
run.setStatus("running");
|
run.setStatus("running");
|
||||||
run.setTriggerType(triggerType != null ? triggerType : "scheduled");
|
run.setTriggerType(triggerType != null ? triggerType : "scheduled");
|
||||||
run.setStartedAt(LocalDateTime.now());
|
LocalDateTime now = LocalDateTime.now();
|
||||||
|
run.setStartedAt(now);
|
||||||
|
run.setHeartbeatAt(now);
|
||||||
run.setDeliveryStatus("NONE");
|
run.setDeliveryStatus("NONE");
|
||||||
runMapper.insert(run);
|
runMapper.insert(run);
|
||||||
return run;
|
return run;
|
||||||
@ -162,12 +167,16 @@ public class CronJobLifecycleService {
|
|||||||
*/
|
*/
|
||||||
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
@Transactional(propagation = Propagation.REQUIRES_NEW)
|
||||||
public void markRunSucceeded(CronJobRunEntity run, String description) {
|
public void markRunSucceeded(CronJobRunEntity run, String description) {
|
||||||
runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
int updated = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getId, run.getId())
|
.eq(CronJobRunEntity::getId, run.getId())
|
||||||
|
.eq(CronJobRunEntity::getStatus, "running")
|
||||||
.set(CronJobRunEntity::getStatus, "succeeded")
|
.set(CronJobRunEntity::getStatus, "succeeded")
|
||||||
.set(CronJobRunEntity::getFinishedAt, LocalDateTime.now())
|
.set(CronJobRunEntity::getFinishedAt, LocalDateTime.now())
|
||||||
.set(CronJobRunEntity::getErrorMessage,
|
.set(CronJobRunEntity::getErrorMessage,
|
||||||
description != null ? StrUtil.maxLength(description, 1000) : null));
|
description != null ? StrUtil.maxLength(description, 1000) : null));
|
||||||
|
if (updated == 0) {
|
||||||
|
log.warn("[CronLifecycle] Run {} lost its running fence before system completion", run.getId());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -205,11 +214,17 @@ public class CronJobLifecycleService {
|
|||||||
|
|
||||||
int totalTokens = chatResult != null
|
int totalTokens = chatResult != null
|
||||||
? chatResult.promptTokens() + chatResult.completionTokens() : 0;
|
? chatResult.promptTokens() + chatResult.completionTokens() : 0;
|
||||||
runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
int updated = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
.eq(CronJobRunEntity::getId, run.getId())
|
.eq(CronJobRunEntity::getId, run.getId())
|
||||||
|
.eq(CronJobRunEntity::getStatus, "running")
|
||||||
.set(CronJobRunEntity::getStatus, "succeeded")
|
.set(CronJobRunEntity::getStatus, "succeeded")
|
||||||
.set(CronJobRunEntity::getFinishedAt, LocalDateTime.now())
|
.set(CronJobRunEntity::getFinishedAt, LocalDateTime.now())
|
||||||
.set(totalTokens > 0, CronJobRunEntity::getTokenUsage, totalTokens));
|
.set(totalTokens > 0, CronJobRunEntity::getTokenUsage, totalTokens));
|
||||||
|
if (updated == 0) {
|
||||||
|
log.warn("[CronLifecycle] Run {} lost its running fence before completion; dropping late result",
|
||||||
|
run.getId());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (silent) {
|
if (silent) {
|
||||||
// No-op run: persist a short marker so the tasks_<wsId>
|
// No-op run: persist a short marker so the tasks_<wsId>
|
||||||
|
|||||||
@ -42,6 +42,7 @@ import vip.mate.wiki.service.WikiProcessingService;
|
|||||||
public class CronJobRunner {
|
public class CronJobRunner {
|
||||||
|
|
||||||
private final CronJobLifecycleService lifecycle;
|
private final CronJobLifecycleService lifecycle;
|
||||||
|
private final CronRunHeartbeatService heartbeat;
|
||||||
private final AgentService agentService;
|
private final AgentService agentService;
|
||||||
private final CronChatOriginFactory originFactory;
|
private final CronChatOriginFactory originFactory;
|
||||||
private final vip.mate.cron.CronConversationResolver conversationResolver;
|
private final vip.mate.cron.CronConversationResolver conversationResolver;
|
||||||
@ -144,14 +145,16 @@ public class CronJobRunner {
|
|||||||
try {
|
try {
|
||||||
ChatOrigin origin = originFactory.from(
|
ChatOrigin origin = originFactory.from(
|
||||||
job, conversationId, started.originMessageId());
|
job, conversationId, started.originMessageId());
|
||||||
chatResult = runAgent(job, userMessage, origin, conversationId);
|
try (CronRunHeartbeatService.Lease ignored = heartbeat.begin(run.getId())) {
|
||||||
|
chatResult = runAgent(job, userMessage, origin, conversationId);
|
||||||
|
}
|
||||||
result = new AssistantMessage(chatResult.content());
|
result = new AssistantMessage(chatResult.content());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("[CronRunner] runAgent failed for job {}: {}", job.getId(), e.getMessage(), e);
|
log.error("[CronRunner] runAgent failed for job {}: {}", job.getId(), e.getMessage(), e);
|
||||||
try {
|
try {
|
||||||
lifecycle.markRunFailed(run, e);
|
lifecycle.markRunFailed(run, e);
|
||||||
} catch (Exception markErr) {
|
} catch (Exception markErr) {
|
||||||
// CronRunStaleCleanup will sweep status='running' rows older than 30 min.
|
// CronRunStaleCleanup will recover a run after its heartbeat expires.
|
||||||
log.warn("[CronRunner] markRunFailed itself failed for run {}: {} (stale-cleanup will recover)",
|
log.warn("[CronRunner] markRunFailed itself failed for run {}: {} (stale-cleanup will recover)",
|
||||||
run.getId(), markErr.getMessage());
|
run.getId(), markErr.getMessage());
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,107 @@
|
|||||||
|
package vip.mate.cron.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
|
||||||
|
import jakarta.annotation.PreDestroy;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import vip.mate.dashboard.model.CronJobRunEntity;
|
||||||
|
import vip.mate.dashboard.repository.CronJobRunMapper;
|
||||||
|
|
||||||
|
import java.time.Clock;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
|
import java.util.concurrent.ScheduledFuture;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
|
||||||
|
/** Maintains a durable liveness signal while a cron run is inside a long agent call. */
|
||||||
|
@Slf4j
|
||||||
|
@Service
|
||||||
|
public class CronRunHeartbeatService {
|
||||||
|
|
||||||
|
static final Duration DEFAULT_INTERVAL = Duration.ofSeconds(30);
|
||||||
|
|
||||||
|
private final CronJobRunMapper runMapper;
|
||||||
|
private final ScheduledExecutorService scheduler;
|
||||||
|
private final Duration interval;
|
||||||
|
private final Clock clock;
|
||||||
|
private final boolean ownsScheduler;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
public CronRunHeartbeatService(CronJobRunMapper runMapper) {
|
||||||
|
this(runMapper, newScheduler(), DEFAULT_INTERVAL, Clock.systemDefaultZone(), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
CronRunHeartbeatService(CronJobRunMapper runMapper,
|
||||||
|
ScheduledExecutorService scheduler,
|
||||||
|
Duration interval,
|
||||||
|
Clock clock,
|
||||||
|
boolean ownsScheduler) {
|
||||||
|
this.runMapper = Objects.requireNonNull(runMapper, "runMapper");
|
||||||
|
this.scheduler = Objects.requireNonNull(scheduler, "scheduler");
|
||||||
|
this.interval = Objects.requireNonNull(interval, "interval");
|
||||||
|
this.clock = Objects.requireNonNull(clock, "clock");
|
||||||
|
this.ownsScheduler = ownsScheduler;
|
||||||
|
if (interval.isZero() || interval.isNegative()) {
|
||||||
|
throw new IllegalArgumentException("heartbeat interval must be positive");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start refreshing one run. The returned lease is idempotent and must be
|
||||||
|
* closed when the long call exits, including exceptional exits.
|
||||||
|
*/
|
||||||
|
public Lease begin(Long runId) {
|
||||||
|
Objects.requireNonNull(runId, "runId");
|
||||||
|
long periodMillis = interval.toMillis();
|
||||||
|
ScheduledFuture<?> future = scheduler.scheduleAtFixedRate(
|
||||||
|
() -> safeTouch(runId), periodMillis, periodMillis, TimeUnit.MILLISECONDS);
|
||||||
|
AtomicBoolean closed = new AtomicBoolean();
|
||||||
|
return () -> {
|
||||||
|
if (closed.compareAndSet(false, true)) {
|
||||||
|
future.cancel(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private void safeTouch(Long runId) {
|
||||||
|
try {
|
||||||
|
int updated = runMapper.update(null, new LambdaUpdateWrapper<CronJobRunEntity>()
|
||||||
|
.eq(CronJobRunEntity::getId, runId)
|
||||||
|
.eq(CronJobRunEntity::getStatus, "running")
|
||||||
|
.set(CronJobRunEntity::getHeartbeatAt, LocalDateTime.now(clock)));
|
||||||
|
if (updated == 0) {
|
||||||
|
log.debug("[CronHeartbeat] Run {} is no longer running; heartbeat ignored", runId);
|
||||||
|
}
|
||||||
|
} catch (RuntimeException e) {
|
||||||
|
// ScheduledExecutorService suppresses all later ticks if a task
|
||||||
|
// escapes with an exception. Keep the liveness loop recoverable.
|
||||||
|
log.warn("[CronHeartbeat] Failed to refresh run {}: {}", runId, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@PreDestroy
|
||||||
|
void shutdown() {
|
||||||
|
if (ownsScheduler) {
|
||||||
|
scheduler.shutdownNow();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ScheduledExecutorService newScheduler() {
|
||||||
|
return Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||||
|
Thread thread = new Thread(runnable, "cron-run-heartbeat");
|
||||||
|
thread.setDaemon(true);
|
||||||
|
return thread;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
@FunctionalInterface
|
||||||
|
public interface Lease extends AutoCloseable {
|
||||||
|
@Override
|
||||||
|
void close();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -16,6 +16,8 @@ public class CronJobRunEntity {
|
|||||||
/** scheduled / manual */
|
/** scheduled / manual */
|
||||||
private String triggerType;
|
private String triggerType;
|
||||||
private LocalDateTime startedAt;
|
private LocalDateTime startedAt;
|
||||||
|
/** Last durable liveness signal while the run is executing. */
|
||||||
|
private LocalDateTime heartbeatAt;
|
||||||
private LocalDateTime finishedAt;
|
private LocalDateTime finishedAt;
|
||||||
private String errorMessage;
|
private String errorMessage;
|
||||||
private Integer tokenUsage;
|
private Integer tokenUsage;
|
||||||
|
|||||||
@ -0,0 +1,6 @@
|
|||||||
|
-- Durable liveness for long cron runs. Stale cleanup falls back to started_at
|
||||||
|
-- for pre-migration rows whose heartbeat_at is null.
|
||||||
|
ALTER TABLE mate_cron_job_run ADD COLUMN IF NOT EXISTS heartbeat_at TIMESTAMP;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_cron_run_status_heartbeat
|
||||||
|
ON mate_cron_job_run(status, heartbeat_at);
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
-- Durable liveness for long cron runs (Kingbase/PostgreSQL dialect).
|
||||||
|
ALTER TABLE mate_cron_job_run ADD COLUMN IF NOT EXISTS heartbeat_at TIMESTAMP;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_cron_run_status_heartbeat
|
||||||
|
ON mate_cron_job_run(status, heartbeat_at);
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
-- Durable liveness for long cron runs (MySQL dialect).
|
||||||
|
SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_cron_job_run'
|
||||||
|
AND COLUMN_NAME = 'heartbeat_at');
|
||||||
|
SET @s := IF(@c = 0,
|
||||||
|
'ALTER TABLE mate_cron_job_run ADD COLUMN heartbeat_at TIMESTAMP NULL',
|
||||||
|
'SELECT 1');
|
||||||
|
PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||||
|
|
||||||
|
SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_cron_job_run'
|
||||||
|
AND INDEX_NAME = 'idx_cron_run_status_heartbeat');
|
||||||
|
SET @s := IF(@c = 0,
|
||||||
|
'CREATE INDEX idx_cron_run_status_heartbeat ON mate_cron_job_run(status, heartbeat_at)',
|
||||||
|
'SELECT 1');
|
||||||
|
PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||||
@ -87,6 +87,28 @@ class ConversationWindowManagerSummaryBudgetTest {
|
|||||||
"iterative-update: literal placeholder must not leak into the SystemMessage (the bug regression guard)");
|
"iterative-update: literal placeholder must not leak into the SystemMessage (the bug regression guard)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("Token-limit summaries are rejected and never become iterative state")
|
||||||
|
void lengthFinishReasonRejectsTruncatedSummary() throws Exception {
|
||||||
|
stubResponse("TRUNCATED BUT NONEMPTY", "length");
|
||||||
|
|
||||||
|
String result = invokeGenerateSummary("conv-truncated", null);
|
||||||
|
|
||||||
|
assertNull(result);
|
||||||
|
assertFalse(previousSummaries().containsKey("conv-truncated"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("Normally stopped summaries remain accepted")
|
||||||
|
void stopFinishReasonAcceptsCompleteSummary() throws Exception {
|
||||||
|
stubResponse("COMPLETE SUMMARY", "stop");
|
||||||
|
|
||||||
|
String result = invokeGenerateSummary("conv-complete", null);
|
||||||
|
|
||||||
|
assertEquals("COMPLETE SUMMARY", result);
|
||||||
|
assertEquals("COMPLETE SUMMARY", previousSummaries().get("conv-complete"));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reflectively invoke the private {@code generateSummary} method and capture
|
* Reflectively invoke the private {@code generateSummary} method and capture
|
||||||
* the {@link Prompt} sent to the mocked {@link ChatModel}.
|
* the {@link Prompt} sent to the mocked {@link ChatModel}.
|
||||||
@ -106,4 +128,26 @@ class ConversationWindowManagerSummaryBudgetTest {
|
|||||||
org.mockito.Mockito.verify(chatModel).call(captor.capture());
|
org.mockito.Mockito.verify(chatModel).call(captor.capture());
|
||||||
return captor.getValue();
|
return captor.getValue();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String invokeGenerateSummary(String conversationId, String memoryExtra) throws Exception {
|
||||||
|
List<Message> oldMessages = List.of(new UserMessage("hello"), new UserMessage("world"));
|
||||||
|
Method method = ConversationWindowManager.class.getDeclaredMethod(
|
||||||
|
"generateSummary", List.class, ChatModel.class, String.class, int.class, String.class);
|
||||||
|
method.setAccessible(true);
|
||||||
|
return (String) method.invoke(manager, oldMessages, chatModel, conversationId, 1500, memoryExtra);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void stubResponse(String text, String finishReason) {
|
||||||
|
Generation generation = new Generation(
|
||||||
|
new org.springframework.ai.chat.messages.AssistantMessage(text),
|
||||||
|
ChatGenerationMetadata.builder().finishReason(finishReason).build());
|
||||||
|
when(chatModel.call(any(Prompt.class))).thenReturn(new ChatResponse(List.of(generation)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
private ConcurrentHashMap<String, String> previousSummaries() throws Exception {
|
||||||
|
Field field = ConversationWindowManager.class.getDeclaredField("previousSummaries");
|
||||||
|
field.setAccessible(true);
|
||||||
|
return (ConcurrentHashMap<String, String>) field.get(manager);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,18 +1,21 @@
|
|||||||
package vip.mate.cron.delivery;
|
package vip.mate.cron.delivery;
|
||||||
|
|
||||||
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.AbstractWrapper;
|
||||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||||
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
import org.junit.jupiter.api.BeforeAll;
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||||
import vip.mate.cron.model.CronJobEntity;
|
import vip.mate.cron.model.CronJobEntity;
|
||||||
import vip.mate.dashboard.model.CronJobRunEntity;
|
import vip.mate.dashboard.model.CronJobRunEntity;
|
||||||
import vip.mate.dashboard.repository.CronJobRunMapper;
|
import vip.mate.dashboard.repository.CronJobRunMapper;
|
||||||
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
import java.util.concurrent.ExecutionException;
|
import java.util.concurrent.ExecutionException;
|
||||||
@ -101,6 +104,75 @@ class AbstractCronResultDeliveryTest {
|
|||||||
verify(runMapper, times(1)).update(any(), any(Wrapper.class)); // only the failed claim
|
verify(runMapper, times(1)).update(any(), any(Wrapper.class)); // only the failed claim
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void claimRun_acceptsOnlyFreshOrLegacyDeliveryState() {
|
||||||
|
when(runMapper.update(any(), any(Wrapper.class))).thenReturn(0);
|
||||||
|
|
||||||
|
AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) {
|
||||||
|
@Override public boolean supports(CronJobEntity j) { return true; }
|
||||||
|
@Override protected DeliveryOutcome doDeliver(
|
||||||
|
CronJobEntity j, AssistantMessage r, CronJobRunEntity ignored) {
|
||||||
|
return DeliveryOutcome.delivered("never");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
strategy.deliver(job, new AssistantMessage("hi"), run);
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
ArgumentCaptor<Wrapper> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(runMapper).update(isNull(), captor.capture());
|
||||||
|
Wrapper<?> claim = captor.getValue();
|
||||||
|
assertTrue(claim.getSqlSegment().contains("delivery_status IS NULL"));
|
||||||
|
assertTrue(whereValues(claim).contains("NONE"));
|
||||||
|
assertFalse(whereValues(claim).contains("PENDING"),
|
||||||
|
"an already-PENDING delivery must not be claimable again");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void deliveredTerminalWrite_requiresPendingFence() {
|
||||||
|
when(runMapper.update(any(), any(Wrapper.class))).thenReturn(1, 1);
|
||||||
|
|
||||||
|
AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) {
|
||||||
|
@Override public boolean supports(CronJobEntity j) { return true; }
|
||||||
|
@Override protected DeliveryOutcome doDeliver(
|
||||||
|
CronJobEntity j, AssistantMessage r, CronJobRunEntity ignored) {
|
||||||
|
return DeliveryOutcome.delivered("user-x");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
strategy.deliver(job, new AssistantMessage("hi"), run);
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
ArgumentCaptor<Wrapper> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(runMapper, times(2)).update(isNull(), captor.capture());
|
||||||
|
List<Wrapper> writes = captor.getAllValues();
|
||||||
|
assertTrue(whereValues(writes.get(1)).contains("PENDING"),
|
||||||
|
"late success must not overwrite a stale-cleanup terminal state");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void failedTerminalWrite_requiresPendingFence() {
|
||||||
|
when(runMapper.update(any(), any(Wrapper.class))).thenReturn(1, 1);
|
||||||
|
|
||||||
|
AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) {
|
||||||
|
@Override public boolean supports(CronJobEntity j) { return true; }
|
||||||
|
@Override protected DeliveryOutcome doDeliver(
|
||||||
|
CronJobEntity j, AssistantMessage r, CronJobRunEntity ignored) {
|
||||||
|
throw new IllegalStateException("delivery failed");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
assertThrows(IllegalStateException.class,
|
||||||
|
() -> strategy.deliver(job, new AssistantMessage("hi"), run));
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
ArgumentCaptor<Wrapper> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(runMapper, times(2)).update(isNull(), captor.capture());
|
||||||
|
List<Wrapper> writes = captor.getAllValues();
|
||||||
|
assertTrue(whereValues(writes.get(1)).contains("PENDING"),
|
||||||
|
"late failure must not overwrite a stale-cleanup terminal state");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void deliver_doDeliverThrows_marksNotDeliveredAndRethrows() {
|
void deliver_doDeliverThrows_marksNotDeliveredAndRethrows() {
|
||||||
// Claim returns 1, then markNotDelivered returns 1
|
// Claim returns 1, then markNotDelivered returns 1
|
||||||
@ -173,4 +245,16 @@ class AbstractCronResultDeliveryTest {
|
|||||||
pool.shutdownNow();
|
pool.shutdownNow();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static Set<Object> whereValues(Wrapper<?> rawWrapper) {
|
||||||
|
AbstractWrapper<?, ?, ?> wrapper = (AbstractWrapper<?, ?, ?>) rawWrapper;
|
||||||
|
String where = wrapper.getSqlSegment();
|
||||||
|
Set<Object> values = new HashSet<>();
|
||||||
|
wrapper.getParamNameValuePairs().forEach((key, value) -> {
|
||||||
|
if (where.contains(key)) {
|
||||||
|
values.add(value);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return values;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,44 @@
|
|||||||
|
package vip.mate.cron.delivery;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import vip.mate.dashboard.model.CronJobRunEntity;
|
||||||
|
import vip.mate.dashboard.repository.CronJobRunMapper;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.isNull;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
class CronRunStaleCleanupTest {
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMpLambdaCache() {
|
||||||
|
MybatisConfiguration cfg = new MybatisConfiguration();
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(cfg, ""), CronJobRunEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void runningSweep_usesHeartbeatWithLegacyStartedAtFallback() {
|
||||||
|
CronJobRunMapper mapper = mock(CronJobRunMapper.class);
|
||||||
|
when(mapper.update(isNull(), any(Wrapper.class))).thenReturn(0);
|
||||||
|
CronRunStaleCleanup cleanup = new CronRunStaleCleanup(mapper);
|
||||||
|
|
||||||
|
cleanup.sweep();
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
ArgumentCaptor<Wrapper> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(mapper, times(2)).update(isNull(), captor.capture());
|
||||||
|
List<Wrapper> writes = captor.getAllValues();
|
||||||
|
String runningWhere = writes.get(1).getSqlSegment();
|
||||||
|
assertTrue(runningWhere.contains("status"), runningWhere);
|
||||||
|
assertTrue(runningWhere.contains("COALESCE(heartbeat_at, started_at)"), runningWhere);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
package vip.mate.cron.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||||
|
import org.springframework.context.ApplicationEventPublisher;
|
||||||
|
import vip.mate.cron.model.CronJobEntity;
|
||||||
|
import vip.mate.dashboard.model.CronJobRunEntity;
|
||||||
|
import vip.mate.dashboard.repository.CronJobRunMapper;
|
||||||
|
import vip.mate.i18n.I18nService;
|
||||||
|
import vip.mate.memory.event.ConversationCompletionPublisher;
|
||||||
|
import vip.mate.workspace.conversation.ConversationService;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
|
import static org.mockito.ArgumentMatchers.isNull;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
class CronJobLifecycleFenceTest {
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMpLambdaCache() {
|
||||||
|
MybatisConfiguration cfg = new MybatisConfiguration();
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(cfg, ""), CronJobRunEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void failedTerminalWrite_isFencedByRunningStatus() {
|
||||||
|
Fixture fixture = new Fixture();
|
||||||
|
fixture.service.markRunFailed(run(), new IllegalStateException("failed"));
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
ArgumentCaptor<Wrapper> captor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(fixture.mapper).update(isNull(), captor.capture());
|
||||||
|
assertTrue(captor.getValue().getSqlSegment().contains("status"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void lateCompletion_dropsMessagesAndEventsAfterFenceIsLost() {
|
||||||
|
Fixture fixture = new Fixture();
|
||||||
|
when(fixture.mapper.update(isNull(), any(Wrapper.class))).thenReturn(0);
|
||||||
|
CronJobEntity job = new CronJobEntity();
|
||||||
|
job.setId(1L);
|
||||||
|
|
||||||
|
fixture.service.finishRunAndPublish(job, run(), "request",
|
||||||
|
new AssistantMessage("late result"), "cron-1", false);
|
||||||
|
|
||||||
|
verifyNoInteractions(fixture.conversations, fixture.completionPublisher, fixture.events);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CronJobRunEntity run() {
|
||||||
|
CronJobRunEntity run = new CronJobRunEntity();
|
||||||
|
run.setId(42L);
|
||||||
|
run.setConversationId("cron-1");
|
||||||
|
return run;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class Fixture {
|
||||||
|
private final CronJobRunMapper mapper = mock(CronJobRunMapper.class);
|
||||||
|
private final ConversationService conversations = mock(ConversationService.class);
|
||||||
|
private final ConversationCompletionPublisher completionPublisher =
|
||||||
|
mock(ConversationCompletionPublisher.class);
|
||||||
|
private final ApplicationEventPublisher events = mock(ApplicationEventPublisher.class);
|
||||||
|
private final CronJobLifecycleService service = new CronJobLifecycleService(
|
||||||
|
mapper, conversations, completionPublisher, events, mock(I18nService.class));
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -17,6 +17,7 @@ import vip.mate.workspace.conversation.ConversationService;
|
|||||||
import vip.mate.workspace.conversation.model.MessageEntity;
|
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.ArgumentMatchers.anyString;
|
import static org.mockito.ArgumentMatchers.anyString;
|
||||||
import static org.mockito.ArgumentMatchers.eq;
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
@ -36,8 +37,9 @@ class CronJobOriginPropagationTest {
|
|||||||
@Test
|
@Test
|
||||||
void lifecycleReturnsThePersistedUserMessageIdWithoutSavingTwice() {
|
void lifecycleReturnsThePersistedUserMessageIdWithoutSavingTwice() {
|
||||||
ConversationService conversations = mock(ConversationService.class);
|
ConversationService conversations = mock(ConversationService.class);
|
||||||
|
CronJobRunMapper runMapper = mock(CronJobRunMapper.class);
|
||||||
CronJobLifecycleService lifecycle = new CronJobLifecycleService(
|
CronJobLifecycleService lifecycle = new CronJobLifecycleService(
|
||||||
mock(CronJobRunMapper.class), conversations,
|
runMapper, conversations,
|
||||||
mock(ConversationCompletionPublisher.class),
|
mock(ConversationCompletionPublisher.class),
|
||||||
mock(ApplicationEventPublisher.class), mock(I18nService.class));
|
mock(ApplicationEventPublisher.class), mock(I18nService.class));
|
||||||
CronJobEntity job = job();
|
CronJobEntity job = job();
|
||||||
@ -50,6 +52,8 @@ class CronJobOriginPropagationTest {
|
|||||||
job, "do work", "scheduled", CONVERSATION_ID);
|
job, "do work", "scheduled", CONVERSATION_ID);
|
||||||
|
|
||||||
assertEquals(MESSAGE_ID, result.originMessageId());
|
assertEquals(MESSAGE_ID, result.originMessageId());
|
||||||
|
assertEquals(result.run().getStartedAt(), result.run().getHeartbeatAt(),
|
||||||
|
"a new run must be live before the first scheduled heartbeat");
|
||||||
verify(conversations, times(1)).saveMessage(CONVERSATION_ID, "user", "do work");
|
verify(conversations, times(1)).saveMessage(CONVERSATION_ID, "user", "do work");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -59,6 +63,8 @@ class CronJobOriginPropagationTest {
|
|||||||
AgentService agentService = mock(AgentService.class);
|
AgentService agentService = mock(AgentService.class);
|
||||||
CronChatOriginFactory originFactory = mock(CronChatOriginFactory.class);
|
CronChatOriginFactory originFactory = mock(CronChatOriginFactory.class);
|
||||||
CronConversationResolver resolver = mock(CronConversationResolver.class);
|
CronConversationResolver resolver = mock(CronConversationResolver.class);
|
||||||
|
CronRunHeartbeatService heartbeat = mock(CronRunHeartbeatService.class);
|
||||||
|
CronRunHeartbeatService.Lease lease = mock(CronRunHeartbeatService.Lease.class);
|
||||||
CronJobEntity job = job();
|
CronJobEntity job = job();
|
||||||
CronJobRunEntity run = new CronJobRunEntity();
|
CronJobRunEntity run = new CronJobRunEntity();
|
||||||
run.setId(55L);
|
run.setId(55L);
|
||||||
@ -68,18 +74,49 @@ class CronJobOriginPropagationTest {
|
|||||||
when(lifecycle.startRun(job, "do work", "scheduled", CONVERSATION_ID))
|
when(lifecycle.startRun(job, "do work", "scheduled", CONVERSATION_ID))
|
||||||
.thenReturn(new CronJobLifecycleService.StartResult(run, MESSAGE_ID));
|
.thenReturn(new CronJobLifecycleService.StartResult(run, MESSAGE_ID));
|
||||||
when(originFactory.from(job, CONVERSATION_ID, MESSAGE_ID)).thenReturn(origin);
|
when(originFactory.from(job, CONVERSATION_ID, MESSAGE_ID)).thenReturn(origin);
|
||||||
|
when(heartbeat.begin(55L)).thenReturn(lease);
|
||||||
when(agentService.chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID), eq(origin)))
|
when(agentService.chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID), eq(origin)))
|
||||||
.thenReturn(AgentService.ChatResult.contentOnly("done"));
|
.thenReturn(AgentService.ChatResult.contentOnly("done"));
|
||||||
CronJobRunner runner = new CronJobRunner(lifecycle, agentService, originFactory, resolver,
|
CronJobRunner runner = new CronJobRunner(lifecycle, heartbeat, agentService, originFactory, resolver,
|
||||||
mock(WikiProcessingService.class), new ObjectMapper());
|
mock(WikiProcessingService.class), new ObjectMapper());
|
||||||
|
|
||||||
runner.executeJob(job);
|
runner.executeJob(job);
|
||||||
|
|
||||||
verify(originFactory).from(job, CONVERSATION_ID, MESSAGE_ID);
|
verify(originFactory).from(job, CONVERSATION_ID, MESSAGE_ID);
|
||||||
|
verify(heartbeat).begin(55L);
|
||||||
|
verify(lease).close();
|
||||||
verify(agentService).chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID), eq(origin));
|
verify(agentService).chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID), eq(origin));
|
||||||
verify(agentService, never()).chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID));
|
verify(agentService, never()).chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void runnerClosesHeartbeatWhenAgentFails() {
|
||||||
|
CronJobLifecycleService lifecycle = mock(CronJobLifecycleService.class);
|
||||||
|
CronRunHeartbeatService heartbeat = mock(CronRunHeartbeatService.class);
|
||||||
|
CronRunHeartbeatService.Lease lease = mock(CronRunHeartbeatService.Lease.class);
|
||||||
|
AgentService agentService = mock(AgentService.class);
|
||||||
|
CronChatOriginFactory originFactory = mock(CronChatOriginFactory.class);
|
||||||
|
CronConversationResolver resolver = mock(CronConversationResolver.class);
|
||||||
|
CronJobEntity job = job();
|
||||||
|
CronJobRunEntity run = new CronJobRunEntity();
|
||||||
|
run.setId(55L);
|
||||||
|
ChatOrigin origin = ChatOrigin.cron(CONVERSATION_ID, WORKSPACE_ID, null, null, null);
|
||||||
|
when(resolver.resolve(job)).thenReturn(CONVERSATION_ID);
|
||||||
|
when(lifecycle.startRun(job, "do work", "scheduled", CONVERSATION_ID))
|
||||||
|
.thenReturn(new CronJobLifecycleService.StartResult(run, MESSAGE_ID));
|
||||||
|
when(originFactory.from(job, CONVERSATION_ID, MESSAGE_ID)).thenReturn(origin);
|
||||||
|
when(heartbeat.begin(55L)).thenReturn(lease);
|
||||||
|
when(agentService.chatWithUsage(eq(AGENT_ID), anyString(), eq(CONVERSATION_ID), eq(origin)))
|
||||||
|
.thenThrow(new IllegalStateException("provider timeout"));
|
||||||
|
CronJobRunner runner = new CronJobRunner(lifecycle, heartbeat, agentService, originFactory, resolver,
|
||||||
|
mock(WikiProcessingService.class), new ObjectMapper());
|
||||||
|
|
||||||
|
runner.executeJob(job);
|
||||||
|
|
||||||
|
verify(lease).close();
|
||||||
|
verify(lifecycle).markRunFailed(eq(run), any(IllegalStateException.class));
|
||||||
|
}
|
||||||
|
|
||||||
private static CronJobEntity job() {
|
private static CronJobEntity job() {
|
||||||
CronJobEntity job = new CronJobEntity();
|
CronJobEntity job = new CronJobEntity();
|
||||||
job.setId(JOB_ID);
|
job.setId(JOB_ID);
|
||||||
|
|||||||
@ -0,0 +1,86 @@
|
|||||||
|
package vip.mate.cron.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.AbstractWrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
|
import org.junit.jupiter.api.BeforeAll;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
import vip.mate.dashboard.model.CronJobRunEntity;
|
||||||
|
import vip.mate.dashboard.repository.CronJobRunMapper;
|
||||||
|
|
||||||
|
import java.time.Clock;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.time.Instant;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.ZoneOffset;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.ScheduledExecutorService;
|
||||||
|
import java.util.concurrent.ScheduledFuture;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
import static org.mockito.ArgumentMatchers.*;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
class CronRunHeartbeatServiceTest {
|
||||||
|
|
||||||
|
@BeforeAll
|
||||||
|
static void initMpLambdaCache() {
|
||||||
|
MybatisConfiguration cfg = new MybatisConfiguration();
|
||||||
|
TableInfoHelper.initTableInfo(new MapperBuilderAssistant(cfg, ""), CronJobRunEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void begin_schedulesFencedHeartbeat_andLeaseClosesIdempotently() {
|
||||||
|
CronJobRunMapper mapper = mock(CronJobRunMapper.class);
|
||||||
|
ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
ScheduledFuture<Object> future = mock(ScheduledFuture.class);
|
||||||
|
ArgumentCaptor<Runnable> task = ArgumentCaptor.forClass(Runnable.class);
|
||||||
|
doReturn(future).when(scheduler)
|
||||||
|
.scheduleAtFixedRate(task.capture(), eq(30_000L), eq(30_000L), eq(TimeUnit.MILLISECONDS));
|
||||||
|
when(mapper.update(isNull(), any(Wrapper.class))).thenReturn(1);
|
||||||
|
Clock clock = Clock.fixed(Instant.parse("2026-09-02T08:00:00Z"), ZoneOffset.UTC);
|
||||||
|
CronRunHeartbeatService service = new CronRunHeartbeatService(
|
||||||
|
mapper, scheduler, Duration.ofSeconds(30), clock, false);
|
||||||
|
|
||||||
|
CronRunHeartbeatService.Lease lease = service.begin(42L);
|
||||||
|
task.getValue().run();
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
ArgumentCaptor<Wrapper> wrapperCaptor = ArgumentCaptor.forClass(Wrapper.class);
|
||||||
|
verify(mapper).update(isNull(), wrapperCaptor.capture());
|
||||||
|
AbstractWrapper<?, ?, ?> wrapper = (AbstractWrapper<?, ?, ?>) wrapperCaptor.getValue();
|
||||||
|
Set<Object> values = Set.copyOf(wrapper.getParamNameValuePairs().values());
|
||||||
|
String where = wrapper.getSqlSegment();
|
||||||
|
assertTrue(where.contains("id"), () -> "heartbeat must target one run: " + where);
|
||||||
|
assertTrue(where.contains("status"), () -> "heartbeat must not revive a terminal run: " + where);
|
||||||
|
assertTrue(values.contains(LocalDateTime.of(2026, 9, 2, 8, 0)),
|
||||||
|
() -> "missing fixed heartbeat timestamp in " + values);
|
||||||
|
|
||||||
|
lease.close();
|
||||||
|
lease.close();
|
||||||
|
verify(future, times(1)).cancel(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void heartbeatFailure_doesNotKillSchedulerTask() {
|
||||||
|
CronJobRunMapper mapper = mock(CronJobRunMapper.class);
|
||||||
|
ScheduledExecutorService scheduler = mock(ScheduledExecutorService.class);
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
ScheduledFuture<Object> future = mock(ScheduledFuture.class);
|
||||||
|
ArgumentCaptor<Runnable> task = ArgumentCaptor.forClass(Runnable.class);
|
||||||
|
doReturn(future).when(scheduler)
|
||||||
|
.scheduleAtFixedRate(task.capture(), anyLong(), anyLong(), any());
|
||||||
|
when(mapper.update(isNull(), any(Wrapper.class))).thenThrow(new IllegalStateException("db jitter"));
|
||||||
|
CronRunHeartbeatService service = new CronRunHeartbeatService(
|
||||||
|
mapper, scheduler, Duration.ofSeconds(30), Clock.systemUTC(), false);
|
||||||
|
|
||||||
|
service.begin(7L);
|
||||||
|
|
||||||
|
assertDoesNotThrow(() -> task.getValue().run());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user