mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 11:58:34 +08:00
mate_memory_recall.filename is VARCHAR(256), but the snippet-level recall tracker assembles the key as `path + '#' + H2-heading-slug`. When the LLM writes an over-long daily-note heading (the summarize prompt placed no length cap on the `##` title), the CJK-preserving slug pushes the filename past the column, and writes fail with Data too long / string too long. Three layers of defence, root cause + hard caps: 1. prompt (source) — summarize-system.txt now asks for short (≤30 chars) `##` titles; details go in the body, not the heading. 2. slug cap (close to source) — MemoryRecallTracker.sanitizeSectionKey caps the slug at MAX_SECTION_SLUG=200, leaving path+'#' well under 256. 3. write-side cap (catches every path) — MemoryRecallService.recordRecall truncates filename to MAX_FILENAME_LENGTH=255 at the entry point, so the select/insert/update branches share one value and the dup-key concurrency fallback still matches. Covers trackActiveRetrieval too, which bypasses sanitizeSectionKey. Tests: MemoryRecallFilenameTruncationTest covers both caps (over-long CJK heading, normal heading untouched, ascii slug, date prefix survives) plus an end-to-end assertion that the stored value fits VARCHAR(256). Existing memory-suite unit tests still green.
This commit is contained in:
parent
fa4e7018a0
commit
3ac73623ee
@ -35,6 +35,22 @@ public class MemoryRecallService {
|
|||||||
|
|
||||||
private static final int MAX_QUERY_HASHES = 32;
|
private static final int MAX_QUERY_HASHES = 32;
|
||||||
|
|
||||||
|
/** mate_memory_recall.filename is VARCHAR(256). Bound the value here so an
|
||||||
|
* over-long section key (path + '#' + H2 heading slug, see #461) can never
|
||||||
|
* blow past the column. Truncating at the entry point keeps the select /
|
||||||
|
* insert / update branches below operating on the same value, so the
|
||||||
|
* dup-key concurrency fallback still matches. */
|
||||||
|
static final int MAX_FILENAME_LENGTH = 255;
|
||||||
|
|
||||||
|
/** Truncate {@code filename} to {@link #MAX_FILENAME_LENGTH}. Package-private
|
||||||
|
* for direct unit testing. CJK chars live in the BMP, so {@code substring}
|
||||||
|
* cannot split a surrogate pair. */
|
||||||
|
static String truncateFilename(String filename) {
|
||||||
|
return filename.length() > MAX_FILENAME_LENGTH
|
||||||
|
? filename.substring(0, MAX_FILENAME_LENGTH)
|
||||||
|
: filename;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 记录一次文件召回
|
* 记录一次文件召回
|
||||||
*/
|
*/
|
||||||
@ -42,6 +58,9 @@ public class MemoryRecallService {
|
|||||||
if (agentId == null || filename == null || filename.isBlank()) {
|
if (agentId == null || filename == null || filename.isBlank()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// 写库前硬截断:覆盖所有调用路径(含 trackActiveRetrieval 透传的外部 filename),
|
||||||
|
// 防 filename 突破 VARCHAR(256) 导致写入失败(#461)
|
||||||
|
filename = truncateFilename(filename);
|
||||||
|
|
||||||
// snippet preview 只取前 200 字符(避免对大文件做完整 SHA-256)
|
// snippet preview 只取前 200 字符(避免对大文件做完整 SHA-256)
|
||||||
String preview = snippetText != null && snippetText.length() > 200
|
String preview = snippetText != null && snippetText.length() > 200
|
||||||
|
|||||||
@ -126,12 +126,20 @@ public class MemoryRecallTracker {
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String sanitizeSectionKey(String heading) {
|
/** Max length of a section slug — keeps the full key (path + '#' + slug)
|
||||||
|
* well under the mate_memory_recall.filename VARCHAR(256) ceiling even
|
||||||
|
* when an LLM writes an over-long H2 heading. CJK chars live in the BMP,
|
||||||
|
* so {@code substring} can never split a surrogate pair here. */
|
||||||
|
static final int MAX_SECTION_SLUG = 200;
|
||||||
|
|
||||||
|
/** Package-private for direct unit testing of the slug/truncation logic. */
|
||||||
|
static String sanitizeSectionKey(String heading) {
|
||||||
// "## Some Title" -> "some-title"
|
// "## Some Title" -> "some-title"
|
||||||
return heading.replaceFirst("^#+\\s*", "")
|
String slug = heading.replaceFirst("^#+\\s*", "")
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
.replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", "-")
|
.replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", "-")
|
||||||
.replaceAll("^-|-$", "");
|
.replaceAll("^-|-$", "");
|
||||||
|
return slug.length() > MAX_SECTION_SLUG ? slug.substring(0, MAX_SECTION_SLUG) : slug;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@ -42,7 +42,7 @@ MEMORY.md 与 PROFILE.md 会被**无条件注入每一次对话的系统提示**
|
|||||||
|
|
||||||
字段说明:
|
字段说明:
|
||||||
- `should_update`: 布尔值,是否有任何需要更新的内容。如果为 false,其余字段应为 null
|
- `should_update`: 布尔值,是否有任何需要更新的内容。如果为 false,其余字段应为 null
|
||||||
- `daily_entry`: 字符串或 null。要追加到今日 daily note 的内容(markdown 格式,以时间戳开头如 "## HH:mm ...")
|
- `daily_entry`: 字符串或 null。要追加到今日 daily note 的内容(markdown 格式,以时间戳开头如 "## HH:mm 简要事件标题")。**二级标题(##)保持简短(不超过 30 字),只概括事件主题;事件细节、数字、过程写进标题下方的正文,不要堆进标题——过长的标题会导致下游索引截断。**
|
||||||
- `memory_update`: 字符串或 null。MEMORY.md 的完整新内容(已合并现有内容,不是增量)。仅当有需要新增或修改的**跨项目稳定**信息时才填写
|
- `memory_update`: 字符串或 null。MEMORY.md 的完整新内容(已合并现有内容,不是增量)。仅当有需要新增或修改的**跨项目稳定**信息时才填写
|
||||||
- `profile_update`: 字符串或 null。PROFILE.md 的完整新内容(已合并现有内容,不是增量)。仅当用户身份/偏好有显著变化时才填写
|
- `profile_update`: 字符串或 null。PROFILE.md 的完整新内容(已合并现有内容,不是增量)。仅当用户身份/偏好有显著变化时才填写
|
||||||
- `structured_entries`: 数组或 null。把适合按条目检索的**具体事实**路由到结构化记忆,每个元素形如 `{"type": "...", "key": "...", "content": "..."}`:
|
- `structured_entries`: 数组或 null。把适合按条目检索的**具体事实**路由到结构化记忆,每个元素形如 `{"type": "...", "key": "...", "content": "..."}`:
|
||||||
|
|||||||
@ -0,0 +1,111 @@
|
|||||||
|
package vip.mate.memory.service;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.DisplayName;
|
||||||
|
import org.junit.jupiter.api.Nested;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Guards the {@code mate_memory_recall.filename} VARCHAR(256) ceiling against
|
||||||
|
* over-long section keys (file path + '#' + a long H2 heading slug). See #461.
|
||||||
|
* <p>
|
||||||
|
* Two boundaries are covered as pure functions, no Spring context needed:
|
||||||
|
* <ul>
|
||||||
|
* <li>{@link MemoryRecallTracker#sanitizeSectionKey} — slug-side cap</li>
|
||||||
|
* <li>{@link MemoryRecallService#truncateFilename} — write-side cap</li>
|
||||||
|
* </ul>
|
||||||
|
*/
|
||||||
|
class MemoryRecallFilenameTruncationTest {
|
||||||
|
|
||||||
|
/** Repeated CJK filler so a heading can be grown past any threshold. */
|
||||||
|
private static final String CN = "用户要求设置每日财经早报定时任务,每天早上推送汇总报告到指定群组";
|
||||||
|
|
||||||
|
@Nested
|
||||||
|
@DisplayName("sanitizeSectionKey — slug-side cap")
|
||||||
|
class SanitizeSectionKey {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("normal CJK heading is slugified untouched (no false truncation)")
|
||||||
|
void normalCjkHeadingPreserved() {
|
||||||
|
String slug = MemoryRecallTracker.sanitizeSectionKey("## 08:30 用户设置每日财经早报定时任务");
|
||||||
|
// "## " stripped, ':' and spaces → '-', CJK kept; "08-30-用户设置每日财经早报定时任务"
|
||||||
|
assertEquals("08-30-用户设置每日财经早报定时任务", slug);
|
||||||
|
assertTrue(slug.length() <= MemoryRecallTracker.MAX_SECTION_SLUG);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("over-long CJK heading slug is capped at MAX_SECTION_SLUG and never throws")
|
||||||
|
void overLongCjkHeadingCapped() {
|
||||||
|
StringBuilder heading = new StringBuilder("## ");
|
||||||
|
while (heading.length() < MemoryRecallTracker.MAX_SECTION_SLUG + 200) {
|
||||||
|
heading.append(CN);
|
||||||
|
}
|
||||||
|
String slug = assertDoesNotThrow(() -> MemoryRecallTracker.sanitizeSectionKey(heading.toString()));
|
||||||
|
assertTrue(slug.length() <= MemoryRecallTracker.MAX_SECTION_SLUG,
|
||||||
|
"slug must not exceed MAX_SECTION_SLUG, was " + slug.length());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("ascii-only heading collapses runs of non-alnum to a single '-'")
|
||||||
|
void asciiHeadingSlugified() {
|
||||||
|
assertEquals("some-title-here", MemoryRecallTracker.sanitizeSectionKey("## Some Title Here"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nested
|
||||||
|
@DisplayName("truncateFilename — write-side cap")
|
||||||
|
class TruncateFilename {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("filename at/below the cap is returned unchanged")
|
||||||
|
void underCapUnchanged() {
|
||||||
|
String filename = "memory/2026-06-05.md#08-30-用户设置每日财经早报定时任务";
|
||||||
|
assertTrue(filename.length() <= MemoryRecallService.MAX_FILENAME_LENGTH);
|
||||||
|
assertSame(filename, MemoryRecallService.truncateFilename(filename),
|
||||||
|
"under-cap values must pass through without copying");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("filename over the cap is truncated to MAX_FILENAME_LENGTH")
|
||||||
|
void overCapTruncated() {
|
||||||
|
StringBuilder filename = new StringBuilder("memory/2026-06-05.md#");
|
||||||
|
while (filename.length() < MemoryRecallService.MAX_FILENAME_LENGTH + 100) {
|
||||||
|
filename.append(CN);
|
||||||
|
}
|
||||||
|
String out = MemoryRecallService.truncateFilename(filename.toString());
|
||||||
|
assertEquals(MemoryRecallService.MAX_FILENAME_LENGTH, out.length(),
|
||||||
|
"over-cap value must be exactly MAX_FILENAME_LENGTH");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("truncation keeps the date prefix intact (computeFreshness still parses it)")
|
||||||
|
void datePrefixPreserved() {
|
||||||
|
StringBuilder filename = new StringBuilder("memory/2026-06-05.md#");
|
||||||
|
while (filename.length() < MemoryRecallService.MAX_FILENAME_LENGTH + 100) {
|
||||||
|
filename.append(CN);
|
||||||
|
}
|
||||||
|
String out = MemoryRecallService.truncateFilename(filename.toString());
|
||||||
|
// The leading path/date — the only part computeFreshness uses — survives.
|
||||||
|
assertTrue(out.startsWith("memory/2026-06-05.md"), "date prefix must survive truncation");
|
||||||
|
int hash = out.indexOf('#');
|
||||||
|
assertTrue(hash > 0 && hash < out.length(), "section anchor must still be present");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("full daily-note section key stays under the DB column ceiling end-to-end")
|
||||||
|
void endToEndWithinColumnCeiling() {
|
||||||
|
// Reproduce MemoryRecallTracker's key assembly on an over-long heading.
|
||||||
|
String dailyFile = "memory/2026-06-05.md";
|
||||||
|
StringBuilder heading = new StringBuilder("## 08:30 ");
|
||||||
|
while (heading.length() < MemoryRecallTracker.MAX_SECTION_SLUG + 300) {
|
||||||
|
heading.append(CN);
|
||||||
|
}
|
||||||
|
String sectionKey = dailyFile + "#" + MemoryRecallTracker.sanitizeSectionKey(heading.toString());
|
||||||
|
// Even after the write-side fallback, the stored value must fit VARCHAR(256).
|
||||||
|
String stored = MemoryRecallService.truncateFilename(sectionKey);
|
||||||
|
assertTrue(stored.length() <= 255,
|
||||||
|
"stored filename must fit VARCHAR(256), was " + stored.length());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user