package vip.mate.channel.web;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import reactor.core.Disposable;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Pins the orphan-run policy added for issue #587: when a run's only
* subscriber disconnects (webchat SSE timeout/error) while the agent Flux is
* still running, the run becomes invisible + unreachable and must be reclaimed
* after a grace window instead of burning tokens until the 30-min idle sweep.
*
*
Composes with the detach() fix from #587 defect 1: detach() arms the
* orphan clock when the subscriber list empties; attach()/closeSubscribers()
* clear it.
*/
class ChatStreamTrackerOrphanPolicyTest {
private static final class RecordingDisposable implements Disposable {
private final AtomicBoolean disposed = new AtomicBoolean();
private final boolean throwOnDispose;
private RecordingDisposable() {
this(false);
}
private RecordingDisposable(boolean throwOnDispose) {
this.throwOnDispose = throwOnDispose;
}
@Override
public void dispose() {
disposed.set(true);
if (throwOnDispose) {
throw new IllegalStateException("dispose failed");
}
}
@Override
public boolean isDisposed() {
return disposed.get();
}
}
private ChatStreamTracker newTracker() {
ChatStreamTracker t = new ChatStreamTracker(new ObjectMapper());
t.setIdleTimeoutMinutesForTesting(30); // keep the idle bucket out of the way
t.setOrphanGraceSecondsForTesting(2); // tight grace for unit-test speed
return t;
}
private static CompletableFuture startPausedCleanup(
ChatStreamTracker tracker,
String conversationId,
CountDownLatch claimed,
CountDownLatch release) {
tracker.setEmergencySaveCallback(conversationId, () -> {
claimed.countDown();
try {
if (!release.await(2, TimeUnit.SECONDS)) {
throw new AssertionError("cleanup release latch timed out");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new AssertionError(e);
}
});
tracker.backdateOrphanForTesting(
conversationId, System.currentTimeMillis() - 3_000L);
return CompletableFuture.runAsync(tracker::cleanupStaleRuns);
}
@Test
@DisplayName("Orphan run (no subscribers past grace) is evicted")
void orphanRunEvictedAfterGrace() {
ChatStreamTracker tracker = newTracker();
String cid = "orphan-evict";
tracker.register(cid);
tracker.incrementFlux(cid);
// No subscriber ever attached — simulate the clock by backdating, which
// is exactly what detach() would have set when the last subscriber left.
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L); // 3s > 2s grace
tracker.cleanupStaleRuns();
assertFalse(tracker.hasRunStateForTesting(cid),
"orphan run past the grace window must be evicted");
}
@Test
@DisplayName("Orphan run within grace survives")
void orphanRunWithinGraceSurvives() {
ChatStreamTracker tracker = newTracker();
String cid = "orphan-fresh";
tracker.register(cid);
tracker.incrementFlux(cid);
// Just became orphaned — 1s, within the 2s grace.
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 1_000L);
tracker.cleanupStaleRuns();
assertTrue(tracker.hasRunStateForTesting(cid),
"orphan run inside the grace window must survive — tolerates a brief disconnect");
}
@Test
@DisplayName("Orphan eviction fires emergencySaveCallback before dispose")
void orphanEvictionFiresEmergencySave() {
ChatStreamTracker tracker = newTracker();
String cid = "orphan-save";
tracker.register(cid);
tracker.incrementFlux(cid);
AtomicInteger saveCount = new AtomicInteger();
tracker.setEmergencySaveCallback(cid, saveCount::incrementAndGet);
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L);
tracker.cleanupStaleRuns();
assertEquals(1, saveCount.get(),
"partial assistant content must be flushed via the emergency " +
"save before the orphan run is disposed");
assertFalse(tracker.hasRunStateForTesting(cid));
}
@Test
@DisplayName("A done run is never treated as an orphan")
void doneRunNotOrphan() {
ChatStreamTracker tracker = newTracker();
String cid = "done-not-orphan";
tracker.register(cid);
tracker.incrementFlux(cid);
tracker.complete(cid); // mark done
// Even with an orphan clock backdated past grace, a done run must not
// hit the orphan branch (it's finalized via the done path / retention).
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L);
tracker.cleanupStaleRuns();
// done run is kept for DONE_RETENTION_MS (5 min) — still here right after.
assertTrue(tracker.hasRunStateForTesting(cid),
"a done run must not be evicted as an orphan — it's already finalized");
}
@Test
@DisplayName("A run with a live subscriber is never an orphan")
void runWithSubscriberNotOrphan() {
ChatStreamTracker tracker = newTracker();
String cid = "has-sub";
tracker.register(cid);
tracker.incrementFlux(cid);
SseEmitter em = new SseEmitter();
tracker.attach(cid, em); // attaches a subscriber -> clears orphan clock
// Backdate would-be orphan clock — but a subscriber is present, so the
// orphan branch must not fire regardless.
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L);
tracker.cleanupStaleRuns();
assertTrue(tracker.hasRunStateForTesting(cid),
"a run with a live subscriber must never be evicted as an orphan");
}
@Test
@DisplayName("detach() arms the orphan clock; re-attach clears it")
void detachArmsClockAttachClears() {
ChatStreamTracker tracker = newTracker();
String cid = "detach-rearm";
tracker.register(cid);
tracker.incrementFlux(cid);
SseEmitter em = new SseEmitter();
tracker.attach(cid, em);
// Detach the only subscriber -> clock armed, run still running.
tracker.detach(cid, em);
assertTrue(tracker.isRunning(cid),
"run must still be running after the only subscriber detaches");
// A fresh subscriber re-attaches within grace -> clock cleared, survives.
SseEmitter reattached = new SseEmitter();
assertTrue(tracker.attach(cid, reattached));
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L); // would-be past grace
tracker.cleanupStaleRuns();
assertTrue(tracker.hasRunStateForTesting(cid),
"re-attach clears the orphan clock even if it was backdated");
}
@Test
@DisplayName("Attach is rejected after cleanup atomically claims an orphan")
void attachRejectedAfterEvictionClaim() throws Exception {
ChatStreamTracker tracker = newTracker();
String cid = "orphan-claim";
tracker.register(cid);
tracker.incrementFlux(cid);
CountDownLatch claimed = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
CompletableFuture cleanup = startPausedCleanup(tracker, cid, claimed, release);
assertTrue(claimed.await(1, TimeUnit.SECONDS));
try {
assertFalse(tracker.attach(cid, new SseEmitter()),
"attach must not report success after eviction is claimed");
} finally {
release.countDown();
cleanup.get(2, TimeUnit.SECONDS);
}
assertFalse(tracker.hasRunStateForTesting(cid));
}
@Test
@DisplayName("Old cleanup cannot remove or close a replacement run")
void claimedCleanupDoesNotTouchReplacementRun() throws Exception {
ChatStreamTracker tracker = newTracker();
String cid = "orphan-replacement";
tracker.register(cid);
tracker.incrementFlux(cid);
CountDownLatch claimed = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
CompletableFuture cleanup = startPausedCleanup(tracker, cid, claimed, release);
assertTrue(claimed.await(1, TimeUnit.SECONDS));
SseEmitter replacementEmitter = new SseEmitter();
try {
tracker.register(cid);
tracker.incrementFlux(cid);
assertTrue(tracker.attach(cid, replacementEmitter));
assertTrue(tracker.hasHeartbeatForTesting(cid));
} finally {
release.countDown();
cleanup.get(2, TimeUnit.SECONDS);
}
assertTrue(tracker.hasRunStateForTesting(cid));
assertTrue(tracker.isRunning(cid));
assertTrue(tracker.hasHeartbeatForTesting(cid));
assertDoesNotThrow(() -> replacementEmitter.send(
SseEmitter.event().name("probe").data("still-open")));
}
@Test
@DisplayName("A stale emitter callback cannot orphan a replacement run")
void staleDetachDoesNotArmReplacementOrphanClock() {
ChatStreamTracker tracker = newTracker();
String cid = "stale-detach";
tracker.register(cid);
tracker.incrementFlux(cid);
SseEmitter oldEmitter = new SseEmitter();
assertTrue(tracker.attach(cid, oldEmitter));
tracker.complete(cid);
tracker.register(cid);
tracker.incrementFlux(cid);
// A delayed onCompletion/onTimeout callback belongs to the prior
// generation. It must not arm the fresh state's orphan clock merely
// because that new state has not attached its own emitter yet.
tracker.detach(cid, oldEmitter);
tracker.setOrphanGraceSecondsForTesting(-1);
tracker.cleanupStaleRuns();
assertTrue(tracker.hasRunStateForTesting(cid));
assertTrue(tracker.isRunning(cid));
}
@Test
@DisplayName("Register atomically refreshes a reused run before cleanup can claim it")
void registerRefreshesReusedRunLifecycle() {
ChatStreamTracker tracker = newTracker();
String cid = "register-refresh";
tracker.register(cid);
tracker.incrementFlux(cid);
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L);
ChatStreamTracker.RunHandle handle = tracker.register(cid);
tracker.cleanupStaleRuns();
assertTrue(tracker.hasRunStateForTesting(cid));
assertTrue(tracker.attach(handle, new SseEmitter()));
}
@Test
@DisplayName("Late callbacks from an old handle cannot mutate a replacement run")
void oldHandleCannotMutateReplacementRun() throws Exception {
ChatStreamTracker tracker = newTracker();
String cid = "old-handle";
ChatStreamTracker.RunHandle oldHandle = tracker.register(cid);
tracker.incrementFlux(cid);
CountDownLatch claimed = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
CompletableFuture cleanup = startPausedCleanup(tracker, cid, claimed, release);
assertTrue(claimed.await(1, TimeUnit.SECONDS));
ChatStreamTracker.RunHandle replacementHandle = tracker.register(cid);
SseEmitter replacementEmitter = new SseEmitter();
try {
assertTrue(tracker.attach(replacementHandle, replacementEmitter));
} finally {
release.countDown();
cleanup.get(2, TimeUnit.SECONDS);
}
int bufferBefore = tracker.bufferSizeForTesting(cid);
assertFalse(tracker.attach(oldHandle, new SseEmitter()));
tracker.broadcast(oldHandle, "content_delta", "{\"text\":\"stale\"}");
tracker.closeSubscribers(oldHandle);
tracker.complete(oldHandle);
assertEquals(bufferBefore, tracker.bufferSizeForTesting(cid));
assertTrue(tracker.isRunning(cid));
assertTrue(tracker.hasHeartbeatForTesting(cid));
assertDoesNotThrow(() -> replacementEmitter.send(
SseEmitter.event().name("probe").data("still-open")));
}
@Test
@DisplayName("Cleanup and stale handles cannot touch a replacement disposable")
void cleanupDisposesOnlyClaimedStateDisposable() throws Exception {
ChatStreamTracker tracker = newTracker();
String cid = "disposable-generation";
ChatStreamTracker.RunHandle oldHandle = tracker.register(cid);
tracker.incrementFlux(cid);
RecordingDisposable oldDisposable = new RecordingDisposable();
tracker.setDisposable(oldHandle, oldDisposable);
CountDownLatch claimed = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
CompletableFuture cleanup = startPausedCleanup(tracker, cid, claimed, release);
assertTrue(claimed.await(1, TimeUnit.SECONDS));
ChatStreamTracker.RunHandle replacementHandle = tracker.register(cid);
RecordingDisposable replacementDisposable = new RecordingDisposable();
RecordingDisposable staleLateDisposable = new RecordingDisposable();
AtomicInteger replacementSaveCount = new AtomicInteger();
AtomicInteger staleSaveCount = new AtomicInteger();
tracker.setDisposable(replacementHandle, replacementDisposable);
tracker.setDisposable(oldHandle, staleLateDisposable);
tracker.setEmergencySaveCallback(replacementHandle, replacementSaveCount::incrementAndGet);
tracker.setEmergencySaveCallback(oldHandle, staleSaveCount::incrementAndGet);
release.countDown();
cleanup.get(2, TimeUnit.SECONDS);
assertTrue(oldDisposable.isDisposed());
assertFalse(replacementDisposable.isDisposed());
assertFalse(staleLateDisposable.isDisposed());
assertTrue(tracker.hasRunStateForTesting(cid));
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L);
tracker.cleanupStaleRuns();
assertEquals(1, replacementSaveCount.get());
assertEquals(0, staleSaveCount.get());
assertTrue(replacementDisposable.isDisposed());
assertFalse(staleLateDisposable.isDisposed());
}
@Test
@DisplayName("A throwing disposable cannot leave an evicting tombstone mapped")
void throwingDisposableStillRemovesClaimedState() {
ChatStreamTracker tracker = newTracker();
String cid = "throwing-disposable";
ChatStreamTracker.RunHandle handle = tracker.register(cid);
tracker.incrementFlux(cid);
tracker.setDisposable(handle, new RecordingDisposable(true));
tracker.backdateOrphanForTesting(cid, System.currentTimeMillis() - 3_000L);
assertDoesNotThrow(tracker::cleanupStaleRuns);
assertFalse(tracker.hasRunStateForTesting(cid));
}
@Test
@DisplayName("Broadcast send failure arms orphan cleanup when the last subscriber is removed")
@SuppressWarnings("unchecked")
void sendFailureArmsOrphanCleanup() throws Exception {
ChatStreamTracker tracker = newTracker();
String cid = "send-failure-orphan";
ChatStreamTracker.RunHandle handle = tracker.register(cid);
tracker.incrementFlux(cid);
SseEmitter failingEmitter = new SseEmitter() {
@Override
public void send(SseEventBuilder builder) throws IOException {
throw new IOException("client disconnected");
}
};
assertTrue(tracker.attach(handle, failingEmitter));
tracker.broadcast(handle, "content_delta", "{\"text\":\"still-running\"}");
Field runsField = ChatStreamTracker.class.getDeclaredField("runs");
runsField.setAccessible(true);
Map runs =
(Map) runsField.get(tracker);
ChatStreamTracker.RunState state = runs.get(cid);
synchronized (state.lock) {
assertNotNull(state.subscribersZeroSince,
"removing the final dead subscriber must arm the orphan clock");
state.subscribersZeroSince = System.currentTimeMillis() - 3_000L;
}
tracker.cleanupStaleRuns();
assertFalse(tracker.hasRunStateForTesting(cid));
}
@Test
@DisplayName("Exact-handle detach arms orphan cleanup before an emitter was attached")
@SuppressWarnings("unchecked")
void exactDetachArmsOrphanWithoutRemovingEmitter() throws Exception {
ChatStreamTracker tracker = newTracker();
String cid = "pre-attach-exact-detach";
ChatStreamTracker.RunHandle handle = tracker.register(cid);
tracker.incrementFlux(cid);
tracker.detach(handle, new SseEmitter());
Field runsField = ChatStreamTracker.class.getDeclaredField("runs");
runsField.setAccessible(true);
Map runs =
(Map) runsField.get(tracker);
ChatStreamTracker.RunState state = runs.get(cid);
synchronized (state.lock) {
assertNotNull(state.subscribersZeroSince,
"an exact disconnect must arm even when attach never added the emitter");
state.subscribersZeroSince = System.currentTimeMillis() - 3_000L;
}
tracker.cleanupStaleRuns();
assertFalse(tracker.hasRunStateForTesting(cid));
}
}