diff --git a/mateclaw-server/src/main/java/vip/mate/channel/media/GeneratedFileScrubber.java b/mateclaw-server/src/main/java/vip/mate/channel/media/GeneratedFileScrubber.java index 6caefcb8..2a916c14 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/media/GeneratedFileScrubber.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/media/GeneratedFileScrubber.java @@ -23,8 +23,9 @@ import java.util.regex.Matcher; * a render tool. {@link GeneratedFileCache#put} logs every real * put, so its absence here is proof the file was never generated * this turn. - *
  • The 10-min cache entry expired before the IM client got around - * to clicking, or was wiped on JVM restart.
    • + *
  • The persisted entry was swept after its retention window + * ({@link GeneratedFileCache#TTL}) elapsed before the IM client + * got around to clicking.
    • * * Without this rewrite, IM clients tap a markdown link that returns * 404, save the HTML 404 body as the requested file extension, then diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java index 35ad1756..6fa7d43f 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java @@ -1,34 +1,66 @@ package vip.mate.tool.document; import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Duration; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Stream; /** - * In-memory cache of bytes produced by tools (e.g. {@code DocxRenderTool}) and - * served by {@link GeneratedFileController}. Entries expire after {@link #TTL} - * and are evicted lazily on every {@link #put} call. + * Store of bytes produced by tools (e.g. {@code DocxRenderTool}) and served by + * {@link GeneratedFileController}. Each entry is written to disk under + * {@link #DEFAULT_STORAGE_DIR} and mirrored in an in-memory map for fast reads. * - *

    The cache is process-local and intentionally not persisted: a JVM restart - * invalidates all outstanding download links. The download URL embeds a random - * {@link UUID}, which acts as the only access credential. + *

    Persistence is what makes download links durable: the bytes survive both + * cache eviction and a JVM restart, so a link a user clicks minutes — or days — + * after generation still resolves instead of 404ing. Entries are retained for + * {@link #TTL} and a scheduled sweep removes expired files. The download URL + * embeds a random {@link UUID}, which acts as the only access credential. */ @Slf4j @Component public class GeneratedFileCache { - public static final Duration TTL = Duration.ofMinutes(10); + /** How long a generated file remains downloadable after creation. */ + public static final Duration TTL = Duration.ofDays(7); + + /** Default on-disk location for persisted generated files. */ + public static final Path DEFAULT_STORAGE_DIR = Paths.get("data", "generated-files"); + + /** How often the expired-file sweep runs (6 hours). Must be a compile-time + * constant for use in {@link Scheduled#fixedDelay()}. */ + private static final long CLEANUP_INTERVAL_MS = 6L * 60 * 60 * 1000; + + /** Guards path resolution: only server-issued UUID-shaped ids are accepted. */ + private static final Pattern ID_RE = Pattern.compile("[a-zA-Z0-9-]{1,64}"); + + private static final String META_SUFFIX = ".meta"; /** - * URL pattern for in-memory generated files served by - * {@code GeneratedFileController}. Public so channel adapters and graph - * nodes share a single source of truth. + * Upper bound on bytes held in memory. Disk is the source of truth and + * retains entries for {@link #TTL}; this map is only a hot-read cache, so + * capping it keeps heap bounded regardless of how many files are produced + * within the retention window. A miss simply reloads from disk. + */ + private static final int MAX_MEMORY_ENTRIES = 256; + + /** + * URL pattern for generated files served by {@code GeneratedFileController}. + * Public so channel adapters and graph nodes share a single source of truth. */ public static final Pattern GENERATED_URL_PATTERN = Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)"); @@ -41,7 +73,33 @@ public class GeneratedFileCache { public static final String MISSING_REFERENCE_NOTICE = "⚠️ 文件未真正生成（模型未调用文档生成工具），请重新发送请求"; - private final ConcurrentHashMap entries = new ConcurrentHashMap<>(); + private final Path storageDir; + + /** + * Access-ordered LRU bounded to {@link #MAX_MEMORY_ENTRIES}: the eldest + * entry is dropped from memory once the cap is exceeded (the persisted + * file stays on disk and is reloaded on the next read). + */ + private final Map entries = Collections.synchronizedMap( + new LinkedHashMap<>(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_MEMORY_ENTRIES; + } + }); + + public GeneratedFileCache() { + this(DEFAULT_STORAGE_DIR); + } + + public GeneratedFileCache(Path storageDir) { + this.storageDir = storageDir.normalize(); + try { + Files.createDirectories(this.storageDir); + } catch (IOException e) { + log.warn("Could not create generated-files dir {}: {}", this.storageDir, e.toString()); + } + } public record Entry(byte[] bytes, String filename, String mimeType, long expireAt) { @@ -56,41 +114,133 @@ public class GeneratedFileCache { * {@code /api/v1/files/generated/{id}}. */ public String put(byte[] bytes, String filename, String mimeType) { - evictExpired(); String id = UUID.randomUUID().toString(); long expireAt = System.currentTimeMillis() + TTL.toMillis(); - entries.put(id, new Entry(bytes, filename, mimeType, expireAt)); - log.debug("Cached generated file id={} filename={} bytes={}", id, filename, bytes.length); + Entry entry = new Entry(bytes, filename, mimeType, expireAt); + entries.put(id, entry); + persist(id, entry); + log.debug("Cached generated file id={} filename={} bytes={}", id, filename, + bytes != null ? bytes.length : 0); return id; } /** - * Look up an entry. Returns {@link Optional#empty()} if missing or expired - * (expired entries are removed as a side-effect). + * Look up an entry. Returns {@link Optional#empty()} if missing or expired. + * Falls back to disk on an in-memory miss so links survive eviction and + * JVM restarts; expired entries are removed as a side-effect. */ public Optional get(String id) { + if (id == null || !ID_RE.matcher(id).matches()) { + return Optional.empty(); + } Entry entry = entries.get(id); + if (entry == null) { + entry = loadFromDisk(id); + if (entry != null) { + entries.put(id, entry); + } + } if (entry == null) { return Optional.empty(); } if (entry.expired()) { - entries.remove(id, entry); + evict(id); return Optional.empty(); } return Optional.of(entry); } - private void evictExpired() { + private void persist(String id, Entry entry) { + if (entry.bytes() == null) { + return; + } + try { + Files.write(storageDir.resolve(id), entry.bytes()); + // expireAt \t mimeType \t base64(filename) — filename is base64-encoded + // so arbitrary unicode / separators round-trip without escaping. + String meta = entry.expireAt() + + "\t" + (entry.mimeType() == null ? "" : entry.mimeType()) + + "\t" + Base64.getEncoder().encodeToString( + (entry.filename() == null ? "" : entry.filename()).getBytes(StandardCharsets.UTF_8)); + Files.writeString(storageDir.resolve(id + META_SUFFIX), meta); + } catch (IOException e) { + // Best-effort: an in-memory entry still serves the current process. + log.warn("Could not persist generated file id={}: {}", id, e.toString()); + } + } + + private Entry loadFromDisk(String id) { + Path bin = storageDir.resolve(id).normalize(); + Path meta = storageDir.resolve(id + META_SUFFIX).normalize(); + // Containment guard — id is already validated, this is defence in depth. + if (!bin.startsWith(storageDir) || !Files.isRegularFile(bin) || !Files.isRegularFile(meta)) { + return null; + } + try { + String[] parts = Files.readString(meta).split("\t", 3); + long expireAt = Long.parseLong(parts[0].trim()); + String mimeType = parts.length > 1 && !parts[1].isEmpty() ? parts[1] : null; + String filename = parts.length > 2 && !parts[2].isEmpty() + ? new String(Base64.getDecoder().decode(parts[2]), StandardCharsets.UTF_8) + : id; + byte[] bytes = Files.readAllBytes(bin); + return new Entry(bytes, filename, mimeType, expireAt); + } catch (Exception e) { + log.warn("Could not load generated file id={}: {}", id, e.toString()); + return null; + } + } + + private void evict(String id) { + entries.remove(id); + try { + Files.deleteIfExists(storageDir.resolve(id)); + Files.deleteIfExists(storageDir.resolve(id + META_SUFFIX)); + } catch (IOException e) { + log.debug("Could not delete generated file id={}: {}", id, e.toString()); + } + } + + /** + * Drop expired entries from memory and disk. Runs on a fixed delay; also + * sweeps orphaned files left by an unclean shutdown. + */ + @Scheduled(fixedDelay = CLEANUP_INTERVAL_MS, initialDelay = CLEANUP_INTERVAL_MS) + public void cleanupExpired() { long now = System.currentTimeMillis(); - entries.entrySet().removeIf(e -> e.getValue().expireAt() <= now); + // entrySet() of a synchronizedMap must be iterated while holding its lock. + synchronized (entries) { + entries.entrySet().removeIf(e -> e.getValue().expireAt() <= now); + } + if (!Files.isDirectory(storageDir)) { + return; + } + try (Stream files = Files.list(storageDir)) { + files.filter(p -> p.getFileName().toString().endsWith(META_SUFFIX)) + .forEach(metaPath -> { + String name = metaPath.getFileName().toString(); + String id = name.substring(0, name.length() - META_SUFFIX.length()); + try { + long expireAt = Long.parseLong( + Files.readString(metaPath).split("\t", 2)[0].trim()); + if (expireAt <= now) { + evict(id); + } + } catch (Exception e) { + log.debug("Skipping unreadable meta {}: {}", name, e.toString()); + } + }); + } catch (IOException e) { + log.warn("Generated-files cleanup sweep failed: {}", e.toString()); + } } /** * Replace any {@code /api/v1/files/generated/{id}} URL in {@code text} - * whose id is NOT present (or has expired) in this cache with - * {@link #MISSING_REFERENCE_NOTICE}. URLs whose ids ARE in the cache are - * left intact so downstream channel adapters can still rewrite them - * into native attachments. + * whose id is NOT present (or has expired) with + * {@link #MISSING_REFERENCE_NOTICE}. URLs whose ids ARE live are left + * intact so downstream channel adapters can still rewrite them into + * native attachments. * *

    Cache misses are nearly always LLM hallucinations — the model * emitted a UUID-shaped string without ever calling a render tool. @@ -107,8 +257,7 @@ public class GeneratedFileCache { m.reset(); while (m.find()) { String id = m.group(1); - Entry entry = entries.get(id); - boolean live = entry != null && !entry.expired(); + boolean live = get(id).isPresent(); String replacement = live ? m.group(0) : MISSING_REFERENCE_NOTICE; m.appendReplacement(out, Matcher.quoteReplacement(replacement)); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java index a3eb8902..f8378cec 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java @@ -23,7 +23,8 @@ public final class GeneratedFileLink { public static String resultZh(byte[] bytes, String displayName, String mimeType, GeneratedFileCache cache, String typeLabel) { String url = stash(bytes, displayName, mimeType, cache); - return typeLabel + "已生成：[" + displayName + "](" + url + ")（链接 10 分钟内有效）。\n" + return typeLabel + "已生成：[" + displayName + "](" + url + ")（链接 " + + GeneratedFileCache.TTL.toDays() + " 天内有效）。\n" + "重要：回答用户时**必须**使用上述 markdown 链接格式 [" + displayName + "](" + url + ")，" + "保持相对路径原样，**不要**用反引号包裹路径，也**不要**添加任何 https://、http:// 域名前缀。"; } @@ -44,7 +45,8 @@ public final class GeneratedFileLink { String prefix = sourceFileCount > 1 ? typeLabel + " generated from " + sourceFileCount + " files" : typeLabel + " generated"; - return prefix + ": [" + displayName + "](" + url + ") (link valid for 10 minutes).\n" + return prefix + ": [" + displayName + "](" + url + ") (link valid for " + + GeneratedFileCache.TTL.toDays() + " days).\n" + "IMPORTANT: when replying to the user you **must** keep the markdown link form [" + displayName + "](" + url + ") above. Keep the relative path verbatim — do **not** " + "wrap it in backticks and do **not** prepend any https://, http:// or domain " diff --git a/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCachePersistenceTest.java b/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCachePersistenceTest.java new file mode 100644 index 00000000..f929f1d9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCachePersistenceTest.java @@ -0,0 +1,83 @@ +package vip.mate.tool.document; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin the persistence contract that keeps download links durable: bytes are + * written to disk so a link still resolves after the in-memory entry is gone + * or the JVM has restarted. A regression here reintroduces the + * "File not found or expired" page that a user hits minutes after generating + * a document. + */ +class GeneratedFileCachePersistenceTest { + + @Test + @DisplayName("a link survives a 'restart' — a fresh cache over the same dir still serves it") + void survivesRestart(@TempDir Path dir) { + GeneratedFileCache first = new GeneratedFileCache(dir); + byte[] bytes = "report-body".getBytes(StandardCharsets.UTF_8); + String id = first.put(bytes, "季度报表.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + + // Simulate a JVM restart: a brand-new instance with an empty memory map, + // pointing at the same storage directory. + GeneratedFileCache afterRestart = new GeneratedFileCache(dir); + GeneratedFileCache.Entry entry = afterRestart.get(id).orElse(null); + + assertNotNull(entry, "persisted entry must be reloaded from disk after restart"); + assertArrayEquals(bytes, entry.bytes(), "reloaded bytes must match the original"); + assertEquals("季度报表.docx", entry.filename(), "unicode filename must round-trip"); + assertEquals("application/vnd.openxmlformats-officedocument.wordprocessingml.document", + entry.mimeType()); + } + + @Test + @DisplayName("unknown id returns empty") + void unknownIdEmpty(@TempDir Path dir) { + GeneratedFileCache cache = new GeneratedFileCache(dir); + assertTrue(cache.get("00000000-0000-0000-0000-000000000000").isEmpty()); + } + + @Test + @DisplayName("malformed / path-traversal ids are rejected without touching disk") + void traversalRejected(@TempDir Path dir) { + GeneratedFileCache cache = new GeneratedFileCache(dir); + assertTrue(cache.get("../secret").isEmpty()); + assertTrue(cache.get("a/b").isEmpty()); + assertTrue(cache.get("").isEmpty()); + assertTrue(cache.get(null).isEmpty()); + } + + @Test + @DisplayName("memory LRU eviction never loses downloadability — old ids reload from disk") + void lruEvictionFallsBackToDisk(@TempDir Path dir) { + GeneratedFileCache cache = new GeneratedFileCache(dir); + // Far exceed the in-memory cap so the first id is evicted from memory. + String firstId = cache.put("first".getBytes(StandardCharsets.UTF_8), "first.txt", "text/plain"); + for (int i = 0; i < 400; i++) { + cache.put(("f" + i).getBytes(StandardCharsets.UTF_8), "f" + i + ".txt", "text/plain"); + } + GeneratedFileCache.Entry entry = cache.get(firstId).orElse(null); + assertNotNull(entry, "an id evicted from the memory cache must still resolve from disk"); + assertArrayEquals("first".getBytes(StandardCharsets.UTF_8), entry.bytes()); + } + + @Test + @DisplayName("scrub treats a persisted-but-evicted id as live (reloads from disk)") + void scrubReloadsPersisted(@TempDir Path dir) { + GeneratedFileCache first = new GeneratedFileCache(dir); + String id = first.put("x".getBytes(StandardCharsets.UTF_8), "a.pdf", "application/pdf"); + + GeneratedFileCache afterRestart = new GeneratedFileCache(dir); + String text = "下载: /api/v1/files/generated/" + id; + assertEquals(text, afterRestart.scrubMissingReferences(text), + "a still-persisted link must not be scrubbed as missing after restart"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java b/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java index 0aac0356..330622d3 100644 --- a/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java +++ b/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java @@ -3,6 +3,9 @@ package vip.mate.tool.document; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; @@ -21,8 +24,9 @@ class GeneratedFileCacheScrubTest { private GeneratedFileCache cache; @BeforeEach - void setUp() { - cache = new GeneratedFileCache(); + void setUp(@TempDir Path tempDir) { + // Hermetic storage so put() does not litter the real data/ dir. + cache = new GeneratedFileCache(tempDir); } @Test