mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(tool): persist generated files to disk so download links survive restart and 10-min window (#243)
This commit is contained in:
matevip
parent
9073c94de1
commit
40ce1c67ac
@ -23,8 +23,9 @@ import java.util.regex.Matcher;
|
||||
* a render tool. {@link GeneratedFileCache#put} logs every real
|
||||
* put, so its absence here is proof the file was never generated
|
||||
* this turn.</li>
|
||||
* <li>The 10-min cache entry expired before the IM client got around
|
||||
* to clicking, or was wiped on JVM restart.</li>
|
||||
* <li>The persisted entry was swept after its retention window
|
||||
* ({@link GeneratedFileCache#TTL}) elapsed before the IM client
|
||||
* got around to clicking.</li>
|
||||
* </ol>
|
||||
* Without this rewrite, IM clients tap a markdown link that returns
|
||||
* 404, save the HTML 404 body as the requested file extension, then
|
||||
|
||||
@ -1,34 +1,66 @@
|
||||
package vip.mate.tool.document;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.Duration;
|
||||
import java.util.Base64;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* In-memory cache of bytes produced by tools (e.g. {@code DocxRenderTool}) and
|
||||
* served by {@link GeneratedFileController}. Entries expire after {@link #TTL}
|
||||
* and are evicted lazily on every {@link #put} call.
|
||||
* Store of bytes produced by tools (e.g. {@code DocxRenderTool}) and served by
|
||||
* {@link GeneratedFileController}. Each entry is written to disk under
|
||||
* {@link #DEFAULT_STORAGE_DIR} and mirrored in an in-memory map for fast reads.
|
||||
*
|
||||
* <p>The cache is process-local and intentionally not persisted: a JVM restart
|
||||
* invalidates all outstanding download links. The download URL embeds a random
|
||||
* {@link UUID}, which acts as the only access credential.
|
||||
* <p>Persistence is what makes download links durable: the bytes survive both
|
||||
* cache eviction and a JVM restart, so a link a user clicks minutes — or days —
|
||||
* after generation still resolves instead of 404ing. Entries are retained for
|
||||
* {@link #TTL} and a scheduled sweep removes expired files. The download URL
|
||||
* embeds a random {@link UUID}, which acts as the only access credential.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class GeneratedFileCache {
|
||||
|
||||
public static final Duration TTL = Duration.ofMinutes(10);
|
||||
/** How long a generated file remains downloadable after creation. */
|
||||
public static final Duration TTL = Duration.ofDays(7);
|
||||
|
||||
/** Default on-disk location for persisted generated files. */
|
||||
public static final Path DEFAULT_STORAGE_DIR = Paths.get("data", "generated-files");
|
||||
|
||||
/** How often the expired-file sweep runs (6 hours). Must be a compile-time
|
||||
* constant for use in {@link Scheduled#fixedDelay()}. */
|
||||
private static final long CLEANUP_INTERVAL_MS = 6L * 60 * 60 * 1000;
|
||||
|
||||
/** Guards path resolution: only server-issued UUID-shaped ids are accepted. */
|
||||
private static final Pattern ID_RE = Pattern.compile("[a-zA-Z0-9-]{1,64}");
|
||||
|
||||
private static final String META_SUFFIX = ".meta";
|
||||
|
||||
/**
|
||||
* URL pattern for in-memory generated files served by
|
||||
* {@code GeneratedFileController}. Public so channel adapters and graph
|
||||
* nodes share a single source of truth.
|
||||
* Upper bound on bytes held in memory. Disk is the source of truth and
|
||||
* retains entries for {@link #TTL}; this map is only a hot-read cache, so
|
||||
* capping it keeps heap bounded regardless of how many files are produced
|
||||
* within the retention window. A miss simply reloads from disk.
|
||||
*/
|
||||
private static final int MAX_MEMORY_ENTRIES = 256;
|
||||
|
||||
/**
|
||||
* URL pattern for generated files served by {@code GeneratedFileController}.
|
||||
* Public so channel adapters and graph nodes share a single source of truth.
|
||||
*/
|
||||
public static final Pattern GENERATED_URL_PATTERN =
|
||||
Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)");
|
||||
@ -41,7 +73,33 @@ public class GeneratedFileCache {
|
||||
public static final String MISSING_REFERENCE_NOTICE =
|
||||
"⚠️ 文件未真正生成（模型未调用文档生成工具），请重新发送请求";
|
||||
|
||||
private final ConcurrentHashMap<String, Entry> entries = new ConcurrentHashMap<>();
|
||||
private final Path storageDir;
|
||||
|
||||
/**
|
||||
* Access-ordered LRU bounded to {@link #MAX_MEMORY_ENTRIES}: the eldest
|
||||
* entry is dropped from memory once the cap is exceeded (the persisted
|
||||
* file stays on disk and is reloaded on the next read).
|
||||
*/
|
||||
private final Map<String, Entry> entries = Collections.synchronizedMap(
|
||||
new LinkedHashMap<>(16, 0.75f, true) {
|
||||
@Override
|
||||
protected boolean removeEldestEntry(Map.Entry<String, Entry> eldest) {
|
||||
return size() > MAX_MEMORY_ENTRIES;
|
||||
}
|
||||
});
|
||||
|
||||
public GeneratedFileCache() {
|
||||
this(DEFAULT_STORAGE_DIR);
|
||||
}
|
||||
|
||||
public GeneratedFileCache(Path storageDir) {
|
||||
this.storageDir = storageDir.normalize();
|
||||
try {
|
||||
Files.createDirectories(this.storageDir);
|
||||
} catch (IOException e) {
|
||||
log.warn("Could not create generated-files dir {}: {}", this.storageDir, e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
public record Entry(byte[] bytes, String filename, String mimeType, long expireAt) {
|
||||
|
||||
@ -56,41 +114,133 @@ public class GeneratedFileCache {
|
||||
* {@code /api/v1/files/generated/{id}}.
|
||||
*/
|
||||
public String put(byte[] bytes, String filename, String mimeType) {
|
||||
evictExpired();
|
||||
String id = UUID.randomUUID().toString();
|
||||
long expireAt = System.currentTimeMillis() + TTL.toMillis();
|
||||
entries.put(id, new Entry(bytes, filename, mimeType, expireAt));
|
||||
log.debug("Cached generated file id={} filename={} bytes={}", id, filename, bytes.length);
|
||||
Entry entry = new Entry(bytes, filename, mimeType, expireAt);
|
||||
entries.put(id, entry);
|
||||
persist(id, entry);
|
||||
log.debug("Cached generated file id={} filename={} bytes={}", id, filename,
|
||||
bytes != null ? bytes.length : 0);
|
||||
return id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up an entry. Returns {@link Optional#empty()} if missing or expired
|
||||
* (expired entries are removed as a side-effect).
|
||||
* Look up an entry. Returns {@link Optional#empty()} if missing or expired.
|
||||
* Falls back to disk on an in-memory miss so links survive eviction and
|
||||
* JVM restarts; expired entries are removed as a side-effect.
|
||||
*/
|
||||
public Optional<Entry> get(String id) {
|
||||
if (id == null || !ID_RE.matcher(id).matches()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
Entry entry = entries.get(id);
|
||||
if (entry == null) {
|
||||
entry = loadFromDisk(id);
|
||||
if (entry != null) {
|
||||
entries.put(id, entry);
|
||||
}
|
||||
}
|
||||
if (entry == null) {
|
||||
return Optional.empty();
|
||||
}
|
||||
if (entry.expired()) {
|
||||
entries.remove(id, entry);
|
||||
evict(id);
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(entry);
|
||||
}
|
||||
|
||||
private void evictExpired() {
|
||||
private void persist(String id, Entry entry) {
|
||||
if (entry.bytes() == null) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.write(storageDir.resolve(id), entry.bytes());
|
||||
// expireAt \t mimeType \t base64(filename) — filename is base64-encoded
|
||||
// so arbitrary unicode / separators round-trip without escaping.
|
||||
String meta = entry.expireAt()
|
||||
+ "\t" + (entry.mimeType() == null ? "" : entry.mimeType())
|
||||
+ "\t" + Base64.getEncoder().encodeToString(
|
||||
(entry.filename() == null ? "" : entry.filename()).getBytes(StandardCharsets.UTF_8));
|
||||
Files.writeString(storageDir.resolve(id + META_SUFFIX), meta);
|
||||
} catch (IOException e) {
|
||||
// Best-effort: an in-memory entry still serves the current process.
|
||||
log.warn("Could not persist generated file id={}: {}", id, e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private Entry loadFromDisk(String id) {
|
||||
Path bin = storageDir.resolve(id).normalize();
|
||||
Path meta = storageDir.resolve(id + META_SUFFIX).normalize();
|
||||
// Containment guard — id is already validated, this is defence in depth.
|
||||
if (!bin.startsWith(storageDir) || !Files.isRegularFile(bin) || !Files.isRegularFile(meta)) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
String[] parts = Files.readString(meta).split("\t", 3);
|
||||
long expireAt = Long.parseLong(parts[0].trim());
|
||||
String mimeType = parts.length > 1 && !parts[1].isEmpty() ? parts[1] : null;
|
||||
String filename = parts.length > 2 && !parts[2].isEmpty()
|
||||
? new String(Base64.getDecoder().decode(parts[2]), StandardCharsets.UTF_8)
|
||||
: id;
|
||||
byte[] bytes = Files.readAllBytes(bin);
|
||||
return new Entry(bytes, filename, mimeType, expireAt);
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not load generated file id={}: {}", id, e.toString());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void evict(String id) {
|
||||
entries.remove(id);
|
||||
try {
|
||||
Files.deleteIfExists(storageDir.resolve(id));
|
||||
Files.deleteIfExists(storageDir.resolve(id + META_SUFFIX));
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not delete generated file id={}: {}", id, e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop expired entries from memory and disk. Runs on a fixed delay; also
|
||||
* sweeps orphaned files left by an unclean shutdown.
|
||||
*/
|
||||
@Scheduled(fixedDelay = CLEANUP_INTERVAL_MS, initialDelay = CLEANUP_INTERVAL_MS)
|
||||
public void cleanupExpired() {
|
||||
long now = System.currentTimeMillis();
|
||||
entries.entrySet().removeIf(e -> e.getValue().expireAt() <= now);
|
||||
// entrySet() of a synchronizedMap must be iterated while holding its lock.
|
||||
synchronized (entries) {
|
||||
entries.entrySet().removeIf(e -> e.getValue().expireAt() <= now);
|
||||
}
|
||||
if (!Files.isDirectory(storageDir)) {
|
||||
return;
|
||||
}
|
||||
try (Stream<Path> files = Files.list(storageDir)) {
|
||||
files.filter(p -> p.getFileName().toString().endsWith(META_SUFFIX))
|
||||
.forEach(metaPath -> {
|
||||
String name = metaPath.getFileName().toString();
|
||||
String id = name.substring(0, name.length() - META_SUFFIX.length());
|
||||
try {
|
||||
long expireAt = Long.parseLong(
|
||||
Files.readString(metaPath).split("\t", 2)[0].trim());
|
||||
if (expireAt <= now) {
|
||||
evict(id);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Skipping unreadable meta {}: {}", name, e.toString());
|
||||
}
|
||||
});
|
||||
} catch (IOException e) {
|
||||
log.warn("Generated-files cleanup sweep failed: {}", e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace any {@code /api/v1/files/generated/{id}} URL in {@code text}
|
||||
* whose id is NOT present (or has expired) in this cache with
|
||||
* {@link #MISSING_REFERENCE_NOTICE}. URLs whose ids ARE in the cache are
|
||||
* left intact so downstream channel adapters can still rewrite them
|
||||
* into native attachments.
|
||||
* whose id is NOT present (or has expired) with
|
||||
* {@link #MISSING_REFERENCE_NOTICE}. URLs whose ids ARE live are left
|
||||
* intact so downstream channel adapters can still rewrite them into
|
||||
* native attachments.
|
||||
*
|
||||
* <p>Cache misses are nearly always LLM hallucinations — the model
|
||||
* emitted a UUID-shaped string without ever calling a render tool.
|
||||
@ -107,8 +257,7 @@ public class GeneratedFileCache {
|
||||
m.reset();
|
||||
while (m.find()) {
|
||||
String id = m.group(1);
|
||||
Entry entry = entries.get(id);
|
||||
boolean live = entry != null && !entry.expired();
|
||||
boolean live = get(id).isPresent();
|
||||
String replacement = live ? m.group(0) : MISSING_REFERENCE_NOTICE;
|
||||
m.appendReplacement(out, Matcher.quoteReplacement(replacement));
|
||||
}
|
||||
|
||||
@ -23,7 +23,8 @@ public final class GeneratedFileLink {
|
||||
public static String resultZh(byte[] bytes, String displayName, String mimeType,
|
||||
GeneratedFileCache cache, String typeLabel) {
|
||||
String url = stash(bytes, displayName, mimeType, cache);
|
||||
return typeLabel + "已生成：[" + displayName + "](" + url + ")（链接 10 分钟内有效）。\n"
|
||||
return typeLabel + "已生成：[" + displayName + "](" + url + ")（链接 "
|
||||
+ GeneratedFileCache.TTL.toDays() + " 天内有效）。\n"
|
||||
+ "重要：回答用户时**必须**使用上述 markdown 链接格式 [" + displayName + "](" + url + ")，"
|
||||
+ "保持相对路径原样，**不要**用反引号包裹路径，也**不要**添加任何 https://、http:// 域名前缀。";
|
||||
}
|
||||
@ -44,7 +45,8 @@ public final class GeneratedFileLink {
|
||||
String prefix = sourceFileCount > 1
|
||||
? typeLabel + " generated from " + sourceFileCount + " files"
|
||||
: typeLabel + " generated";
|
||||
return prefix + ": [" + displayName + "](" + url + ") (link valid for 10 minutes).\n"
|
||||
return prefix + ": [" + displayName + "](" + url + ") (link valid for "
|
||||
+ GeneratedFileCache.TTL.toDays() + " days).\n"
|
||||
+ "IMPORTANT: when replying to the user you **must** keep the markdown link form ["
|
||||
+ displayName + "](" + url + ") above. Keep the relative path verbatim — do **not** "
|
||||
+ "wrap it in backticks and do **not** prepend any https://, http:// or domain "
|
||||
|
||||
@ -0,0 +1,83 @@
|
||||
package vip.mate.tool.document;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Pin the persistence contract that keeps download links durable: bytes are
|
||||
* written to disk so a link still resolves after the in-memory entry is gone
|
||||
* or the JVM has restarted. A regression here reintroduces the
|
||||
* "File not found or expired" page that a user hits minutes after generating
|
||||
* a document.
|
||||
*/
|
||||
class GeneratedFileCachePersistenceTest {
|
||||
|
||||
@Test
|
||||
@DisplayName("a link survives a 'restart' — a fresh cache over the same dir still serves it")
|
||||
void survivesRestart(@TempDir Path dir) {
|
||||
GeneratedFileCache first = new GeneratedFileCache(dir);
|
||||
byte[] bytes = "report-body".getBytes(StandardCharsets.UTF_8);
|
||||
String id = first.put(bytes, "季度报表.docx",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document");
|
||||
|
||||
// Simulate a JVM restart: a brand-new instance with an empty memory map,
|
||||
// pointing at the same storage directory.
|
||||
GeneratedFileCache afterRestart = new GeneratedFileCache(dir);
|
||||
GeneratedFileCache.Entry entry = afterRestart.get(id).orElse(null);
|
||||
|
||||
assertNotNull(entry, "persisted entry must be reloaded from disk after restart");
|
||||
assertArrayEquals(bytes, entry.bytes(), "reloaded bytes must match the original");
|
||||
assertEquals("季度报表.docx", entry.filename(), "unicode filename must round-trip");
|
||||
assertEquals("application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
entry.mimeType());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("unknown id returns empty")
|
||||
void unknownIdEmpty(@TempDir Path dir) {
|
||||
GeneratedFileCache cache = new GeneratedFileCache(dir);
|
||||
assertTrue(cache.get("00000000-0000-0000-0000-000000000000").isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("malformed / path-traversal ids are rejected without touching disk")
|
||||
void traversalRejected(@TempDir Path dir) {
|
||||
GeneratedFileCache cache = new GeneratedFileCache(dir);
|
||||
assertTrue(cache.get("../secret").isEmpty());
|
||||
assertTrue(cache.get("a/b").isEmpty());
|
||||
assertTrue(cache.get("").isEmpty());
|
||||
assertTrue(cache.get(null).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("memory LRU eviction never loses downloadability — old ids reload from disk")
|
||||
void lruEvictionFallsBackToDisk(@TempDir Path dir) {
|
||||
GeneratedFileCache cache = new GeneratedFileCache(dir);
|
||||
// Far exceed the in-memory cap so the first id is evicted from memory.
|
||||
String firstId = cache.put("first".getBytes(StandardCharsets.UTF_8), "first.txt", "text/plain");
|
||||
for (int i = 0; i < 400; i++) {
|
||||
cache.put(("f" + i).getBytes(StandardCharsets.UTF_8), "f" + i + ".txt", "text/plain");
|
||||
}
|
||||
GeneratedFileCache.Entry entry = cache.get(firstId).orElse(null);
|
||||
assertNotNull(entry, "an id evicted from the memory cache must still resolve from disk");
|
||||
assertArrayEquals("first".getBytes(StandardCharsets.UTF_8), entry.bytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("scrub treats a persisted-but-evicted id as live (reloads from disk)")
|
||||
void scrubReloadsPersisted(@TempDir Path dir) {
|
||||
GeneratedFileCache first = new GeneratedFileCache(dir);
|
||||
String id = first.put("x".getBytes(StandardCharsets.UTF_8), "a.pdf", "application/pdf");
|
||||
|
||||
GeneratedFileCache afterRestart = new GeneratedFileCache(dir);
|
||||
String text = "下载: /api/v1/files/generated/" + id;
|
||||
assertEquals(text, afterRestart.scrubMissingReferences(text),
|
||||
"a still-persisted link must not be scrubbed as missing after restart");
|
||||
}
|
||||
}
|
||||
@ -3,6 +3,9 @@ package vip.mate.tool.document;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
@ -21,8 +24,9 @@ class GeneratedFileCacheScrubTest {
|
||||
private GeneratedFileCache cache;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
cache = new GeneratedFileCache();
|
||||
void setUp(@TempDir Path tempDir) {
|
||||
// Hermetic storage so put() does not litter the real data/ dir.
|
||||
cache = new GeneratedFileCache(tempDir);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Loading…
Reference in New Issue
Block a user