mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
mate_memory_recall.filename is VARCHAR(256), but the snippet-level recall tracker assembles the key as `path + '#' + H2-heading-slug`. When the LLM writes an over-long daily-note heading (the summarize prompt placed no length cap on the `##` title), the CJK-preserving slug pushes the filename past the column, and writes fail with Data too long / string too long. Three layers of defence, root cause + hard caps: 1. prompt (source) — summarize-system.txt now asks for short (≤30 chars) `##` titles; details go in the body, not the heading. 2. slug cap (close to source) — MemoryRecallTracker.sanitizeSectionKey caps the slug at MAX_SECTION_SLUG=200, leaving path+'#' well under 256. 3. write-side cap (catches every path) — MemoryRecallService.recordRecall truncates filename to MAX_FILENAME_LENGTH=255 at the entry point, so the select/insert/update branches share one value and the dup-key concurrency fallback still matches. Covers trackActiveRetrieval too, which bypasses sanitizeSectionKey. Tests: MemoryRecallFilenameTruncationTest covers both caps (over-long CJK heading, normal heading untouched, ascii slug, date prefix survives) plus an end-to-end assertion that the stored value fits VARCHAR(256). Existing memory-suite unit tests still green.
This commit is contained in:
parent
fa4e7018a0
commit
3ac73623ee
@ -35,6 +35,22 @@ public class MemoryRecallService {
|
||||
|
||||
private static final int MAX_QUERY_HASHES = 32;
|
||||
|
||||
/** mate_memory_recall.filename is VARCHAR(256). Bound the value here so an
|
||||
* over-long section key (path + '#' + H2 heading slug, see #461) can never
|
||||
* blow past the column. Truncating at the entry point keeps the select /
|
||||
* insert / update branches below operating on the same value, so the
|
||||
* dup-key concurrency fallback still matches. */
|
||||
static final int MAX_FILENAME_LENGTH = 255;
|
||||
|
||||
/** Truncate {@code filename} to {@link #MAX_FILENAME_LENGTH}. Package-private
|
||||
* for direct unit testing. CJK chars live in the BMP, so {@code substring}
|
||||
* cannot split a surrogate pair. */
|
||||
static String truncateFilename(String filename) {
|
||||
return filename.length() > MAX_FILENAME_LENGTH
|
||||
? filename.substring(0, MAX_FILENAME_LENGTH)
|
||||
: filename;
|
||||
}
|
||||
|
||||
/**
|
||||
* 记录一次文件召回
|
||||
*/
|
||||
@ -42,6 +58,9 @@ public class MemoryRecallService {
|
||||
if (agentId == null || filename == null || filename.isBlank()) {
|
||||
return;
|
||||
}
|
||||
// 写库前硬截断:覆盖所有调用路径(含 trackActiveRetrieval 透传的外部 filename),
|
||||
// 防 filename 突破 VARCHAR(256) 导致写入失败(#461)
|
||||
filename = truncateFilename(filename);
|
||||
|
||||
// snippet preview 只取前 200 字符(避免对大文件做完整 SHA-256)
|
||||
String preview = snippetText != null && snippetText.length() > 200
|
||||
|
||||
@ -126,12 +126,20 @@ public class MemoryRecallTracker {
|
||||
return count;
|
||||
}
|
||||
|
||||
private String sanitizeSectionKey(String heading) {
|
||||
/** Max length of a section slug — keeps the full key (path + '#' + slug)
|
||||
* well under the mate_memory_recall.filename VARCHAR(256) ceiling even
|
||||
* when an LLM writes an over-long H2 heading. CJK chars live in the BMP,
|
||||
* so {@code substring} can never split a surrogate pair here. */
|
||||
static final int MAX_SECTION_SLUG = 200;
|
||||
|
||||
/** Package-private for direct unit testing of the slug/truncation logic. */
|
||||
static String sanitizeSectionKey(String heading) {
|
||||
// "## Some Title" -> "some-title"
|
||||
return heading.replaceFirst("^#+\\s*", "")
|
||||
String slug = heading.replaceFirst("^#+\\s*", "")
|
||||
.toLowerCase()
|
||||
.replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", "-")
|
||||
.replaceAll("^-|-$", "");
|
||||
return slug.length() > MAX_SECTION_SLUG ? slug.substring(0, MAX_SECTION_SLUG) : slug;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -42,7 +42,7 @@ MEMORY.md 与 PROFILE.md 会被**无条件注入每一次对话的系统提示**
|
||||
|
||||
字段说明:
|
||||
- `should_update`: 布尔值,是否有任何需要更新的内容。如果为 false,其余字段应为 null
|
||||
- `daily_entry`: 字符串或 null。要追加到今日 daily note 的内容(markdown 格式,以时间戳开头如 "## HH:mm ...")
|
||||
- `daily_entry`: 字符串或 null。要追加到今日 daily note 的内容(markdown 格式,以时间戳开头如 "## HH:mm 简要事件标题")。**二级标题(##)保持简短(不超过 30 字),只概括事件主题;事件细节、数字、过程写进标题下方的正文,不要堆进标题——过长的标题会导致下游索引截断。**
|
||||
- `memory_update`: 字符串或 null。MEMORY.md 的完整新内容(已合并现有内容,不是增量)。仅当有需要新增或修改的**跨项目稳定**信息时才填写
|
||||
- `profile_update`: 字符串或 null。PROFILE.md 的完整新内容(已合并现有内容,不是增量)。仅当用户身份/偏好有显著变化时才填写
|
||||
- `structured_entries`: 数组或 null。把适合按条目检索的**具体事实**路由到结构化记忆,每个元素形如 `{"type": "...", "key": "...", "content": "..."}`:
|
||||
|
||||
@ -0,0 +1,111 @@
|
||||
package vip.mate.memory.service;
|
||||
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Nested;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* Guards the {@code mate_memory_recall.filename} VARCHAR(256) ceiling against
|
||||
* over-long section keys (file path + '#' + a long H2 heading slug). See #461.
|
||||
* <p>
|
||||
* Two boundaries are covered as pure functions, no Spring context needed:
|
||||
* <ul>
|
||||
* <li>{@link MemoryRecallTracker#sanitizeSectionKey} — slug-side cap</li>
|
||||
* <li>{@link MemoryRecallService#truncateFilename} — write-side cap</li>
|
||||
* </ul>
|
||||
*/
|
||||
class MemoryRecallFilenameTruncationTest {
|
||||
|
||||
/** Repeated CJK filler so a heading can be grown past any threshold. */
|
||||
private static final String CN = "用户要求设置每日财经早报定时任务,每天早上推送汇总报告到指定群组";
|
||||
|
||||
@Nested
|
||||
@DisplayName("sanitizeSectionKey — slug-side cap")
|
||||
class SanitizeSectionKey {
|
||||
|
||||
@Test
|
||||
@DisplayName("normal CJK heading is slugified untouched (no false truncation)")
|
||||
void normalCjkHeadingPreserved() {
|
||||
String slug = MemoryRecallTracker.sanitizeSectionKey("## 08:30 用户设置每日财经早报定时任务");
|
||||
// "## " stripped, ':' and spaces → '-', CJK kept; "08-30-用户设置每日财经早报定时任务"
|
||||
assertEquals("08-30-用户设置每日财经早报定时任务", slug);
|
||||
assertTrue(slug.length() <= MemoryRecallTracker.MAX_SECTION_SLUG);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("over-long CJK heading slug is capped at MAX_SECTION_SLUG and never throws")
|
||||
void overLongCjkHeadingCapped() {
|
||||
StringBuilder heading = new StringBuilder("## ");
|
||||
while (heading.length() < MemoryRecallTracker.MAX_SECTION_SLUG + 200) {
|
||||
heading.append(CN);
|
||||
}
|
||||
String slug = assertDoesNotThrow(() -> MemoryRecallTracker.sanitizeSectionKey(heading.toString()));
|
||||
assertTrue(slug.length() <= MemoryRecallTracker.MAX_SECTION_SLUG,
|
||||
"slug must not exceed MAX_SECTION_SLUG, was " + slug.length());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ascii-only heading collapses runs of non-alnum to a single '-'")
|
||||
void asciiHeadingSlugified() {
|
||||
assertEquals("some-title-here", MemoryRecallTracker.sanitizeSectionKey("## Some Title Here"));
|
||||
}
|
||||
}
|
||||
|
||||
@Nested
|
||||
@DisplayName("truncateFilename — write-side cap")
|
||||
class TruncateFilename {
|
||||
|
||||
@Test
|
||||
@DisplayName("filename at/below the cap is returned unchanged")
|
||||
void underCapUnchanged() {
|
||||
String filename = "memory/2026-06-05.md#08-30-用户设置每日财经早报定时任务";
|
||||
assertTrue(filename.length() <= MemoryRecallService.MAX_FILENAME_LENGTH);
|
||||
assertSame(filename, MemoryRecallService.truncateFilename(filename),
|
||||
"under-cap values must pass through without copying");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("filename over the cap is truncated to MAX_FILENAME_LENGTH")
|
||||
void overCapTruncated() {
|
||||
StringBuilder filename = new StringBuilder("memory/2026-06-05.md#");
|
||||
while (filename.length() < MemoryRecallService.MAX_FILENAME_LENGTH + 100) {
|
||||
filename.append(CN);
|
||||
}
|
||||
String out = MemoryRecallService.truncateFilename(filename.toString());
|
||||
assertEquals(MemoryRecallService.MAX_FILENAME_LENGTH, out.length(),
|
||||
"over-cap value must be exactly MAX_FILENAME_LENGTH");
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("truncation keeps the date prefix intact (computeFreshness still parses it)")
|
||||
void datePrefixPreserved() {
|
||||
StringBuilder filename = new StringBuilder("memory/2026-06-05.md#");
|
||||
while (filename.length() < MemoryRecallService.MAX_FILENAME_LENGTH + 100) {
|
||||
filename.append(CN);
|
||||
}
|
||||
String out = MemoryRecallService.truncateFilename(filename.toString());
|
||||
// The leading path/date — the only part computeFreshness uses — survives.
|
||||
assertTrue(out.startsWith("memory/2026-06-05.md"), "date prefix must survive truncation");
|
||||
int hash = out.indexOf('#');
|
||||
assertTrue(hash > 0 && hash < out.length(), "section anchor must still be present");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("full daily-note section key stays under the DB column ceiling end-to-end")
|
||||
void endToEndWithinColumnCeiling() {
|
||||
// Reproduce MemoryRecallTracker's key assembly on an over-long heading.
|
||||
String dailyFile = "memory/2026-06-05.md";
|
||||
StringBuilder heading = new StringBuilder("## 08:30 ");
|
||||
while (heading.length() < MemoryRecallTracker.MAX_SECTION_SLUG + 300) {
|
||||
heading.append(CN);
|
||||
}
|
||||
String sectionKey = dailyFile + "#" + MemoryRecallTracker.sanitizeSectionKey(heading.toString());
|
||||
// Even after the write-side fallback, the stored value must fit VARCHAR(256).
|
||||
String stored = MemoryRecallService.truncateFilename(sectionKey);
|
||||
assertTrue(stored.length() <= 255,
|
||||
"stored filename must fit VARCHAR(256), was " + stored.length());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user