From a27084f9cf2727d470fe97e11b3e1c0b0ef0be8c Mon Sep 17 00:00:00 2001 From: matevip Date: Fri, 29 May 2026 16:05:10 +0800 Subject: [PATCH] feat(memory): recall project facts via query-conditioned prefetch --- .../provider/StructuredMemoryProvider.java | 22 ++- .../service/StructuredMemoryService.java | 180 +++++++++++++++++- .../vip/mate/memory/spi/MemoryManager.java | 9 +- .../src/main/resources/db/data-en.sql | 6 +- .../src/main/resources/db/data-mysql-en.sql | 6 +- .../src/main/resources/db/data-mysql-zh.sql | 6 +- .../src/main/resources/db/data-zh.sql | 6 +- ...ory_consolidation_cron_tier_discipline.sql | 13 ++ ...ory_consolidation_cron_tier_discipline.sql | 13 ++ .../prompts/memory/summarize-system.txt | 11 +- .../service/StructuredMemoryPrefetchTest.java | 113 +++++++++++ 11 files changed, 366 insertions(+), 19 deletions(-) create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V132__memory_consolidation_cron_tier_discipline.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V132__memory_consolidation_cron_tier_discipline.sql create mode 100644 mateclaw-server/src/test/java/vip/mate/memory/service/StructuredMemoryPrefetchTest.java diff --git a/mateclaw-server/src/main/java/vip/mate/memory/provider/StructuredMemoryProvider.java b/mateclaw-server/src/main/java/vip/mate/memory/provider/StructuredMemoryProvider.java index 7dcc1c64..09f75cf6 100644 --- a/mateclaw-server/src/main/java/vip/mate/memory/provider/StructuredMemoryProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/memory/provider/StructuredMemoryProvider.java @@ -35,8 +35,8 @@ public class StructuredMemoryProvider implements MemoryProvider { } /** - * Returns typed memory entries formatted as a Markdown block - * for system prompt injection. + * Returns the stable, low-volume typed entries (user profile, feedback) + * for unconditional system prompt injection. */ @Override public String systemPromptBlock(Long agentId) { @@ -49,6 +49,24 @@ public class StructuredMemoryProvider implements MemoryProvider { } } + /** + * Returns growing/specific typed entries (project facts, reference notes) + * relevant to the current question. Surfacing these per-turn rather than + * always-on keeps them salient when asked about and avoids the model + * confusing a stored fact with similarly-shaped background knowledge. + * The returned block is fenced centrally by the memory manager. + */ + @Override + public String prefetch(Long agentId, String userQuery) { + try { + return structuredMemoryService.buildPrefetchBlock(agentId, userQuery); + } catch (Exception e) { + log.warn("[StructuredMemory] Failed to build prefetch block for agent={}: {}", + agentId, e.getMessage()); + return ""; + } + } + /** * Tools are auto-discovered by ToolRegistry component scan. */ diff --git a/mateclaw-server/src/main/java/vip/mate/memory/service/StructuredMemoryService.java b/mateclaw-server/src/main/java/vip/mate/memory/service/StructuredMemoryService.java index c5295fd8..47d79d7d 100644 --- a/mateclaw-server/src/main/java/vip/mate/memory/service/StructuredMemoryService.java +++ b/mateclaw-server/src/main/java/vip/mate/memory/service/StructuredMemoryService.java @@ -36,6 +36,64 @@ public class StructuredMemoryService { private static final Set VALID_TYPES = Set.of("user", "feedback", "project", "reference"); private static final Pattern SECTION_PATTERN = Pattern.compile("^## (.+)$", Pattern.MULTILINE); + /** + * Stable, low-volume entry types injected unconditionally into the system prompt. + * These describe the user and their durable preferences, so they stay relevant + * across every turn and keep the system prefix cacheable. + */ + private static final List SYSTEM_PROMPT_TYPES = List.of("user", "feedback"); + + /** + * Growing, easily-confused entry types (specific project facts, reference notes) + * surfaced only when the current question matches them. Always-on injection of + * these competes with general knowledge in the prompt and causes the model to + * confuse a specific stored fact with similarly-shaped background information. + */ + private static final List PREFETCH_TYPES = List.of("project", "reference"); + + /** Maximum number of entries injected by a single query-conditioned prefetch. */ + private static final int MAX_PREFETCH_ENTRIES = 6; + + /** Latin word tokens of length >= 2 used for relevance shingling. */ + private static final Pattern WORD_RE = Pattern.compile("[a-z0-9]{2,}"); + + /** Captures the ISO update date from an entry's metadata line ("> ... | Updated: YYYY-MM-DD"). */ + private static final Pattern UPDATED_RE = Pattern.compile("Updated:\\s*(\\d{4}-\\d{2}-\\d{2})"); + + /** + * Domain aliases bridging natural-language question terms to entry keys/types. + * Plain substring/shingle overlap misses cross-language matches such as the + * question term "技术栈" against the key "project_tech_stack", so each alias + * boosts entries whose key contains one of {@code keySubstrings} or whose type + * equals {@code type} when any of its {@code queryTerms} appears in the question. + */ + private static final List ALIASES = List.of( + new Alias(List.of("代号", "项目代号", "codename", "code name"), + List.of("codename", "code_name", "code"), null), + new Alias(List.of("技术栈", "技术", "技术堆栈", "tech stack", "techstack", "technology", "stack"), + List.of("tech", "stack", "技术"), null), + new Alias(List.of("偏好", "风格", "习惯", "preference", "style"), + List.of("pref", "style", "偏好", "风格"), null), + new Alias(List.of("项目", "project"), + List.of(), "project") + ); + + /** A natural-language-to-entry alias rule used by relevance scoring. */ + private record Alias(List queryTerms, List keySubstrings, String type) { + boolean matchesQuery(String query) { + return queryTerms.stream().anyMatch(query::contains); + } + + boolean matchesEntry(String entryType, String keyLower) { + boolean keyHit = keySubstrings.stream().anyMatch(keyLower::contains); + boolean typeHit = type != null && type.equals(entryType); + return keyHit || typeHit; + } + } + + /** A structured entry with its relevance score and update date for the current query. */ + private record ScoredEntry(String type, String key, String body, int score, String updated) {} + private final WorkspaceFileService workspaceFileService; private final ApplicationEventPublisher eventPublisher; @@ -146,13 +204,14 @@ public class StructuredMemoryService { /** * Build a formatted memory block for system prompt injection. - * Returns all typed entries formatted as Markdown. + * Includes only the stable, low-volume entry types ({@link #SYSTEM_PROMPT_TYPES}); + * growing/specific types are surfaced per-turn via {@link #buildPrefetchBlock}. */ public String buildMemoryBlock(Long agentId) { StringBuilder sb = new StringBuilder(); boolean hasContent = false; - for (String type : List.of("user", "feedback", "project", "reference")) { + for (String type : SYSTEM_PROMPT_TYPES) { String fileContent = readFileSafe(agentId, toFilename(type)); if (fileContent.isBlank()) continue; @@ -176,8 +235,119 @@ public class StructuredMemoryService { return sb.toString().trim(); } + /** + * Build a query-conditioned memory block for per-turn prefetch injection. + * Scores {@link #PREFETCH_TYPES} entries against the user's question and returns + * the top matches as Markdown, or an empty string when nothing is relevant. + * Keeping these entries out of the always-on system prompt avoids salience + * competition that would otherwise let the model answer from general knowledge + * instead of the specific stored fact. + */ + public String buildPrefetchBlock(Long agentId, String userQuery) { + if (userQuery == null || userQuery.isBlank()) return ""; + + List scored = recallRelevant(agentId, userQuery, PREFETCH_TYPES, MAX_PREFETCH_ENTRIES); + if (scored.isEmpty()) return ""; + + StringBuilder sb = new StringBuilder("## Relevant Structured Memory\n"); + for (ScoredEntry e : scored) { + sb.append("- **").append(e.key()).append("**: ") + .append(extractContentOnly(e.body())); + if (!e.updated().isBlank()) { + sb.append(" _(updated ").append(e.updated()).append(")_"); + } + sb.append("\n"); + } + return sb.toString().trim(); + } + // ==================== Internal ==================== + /** + * Score entries of the given types against the user query and return the + * highest-scoring matches (score > 0), best first, capped at {@code limit}. + */ + private List recallRelevant(Long agentId, String userQuery, List types, int limit) { + String q = userQuery.toLowerCase(); + Set queryShingles = shingles(q); + + List matches = new ArrayList<>(); + for (String t : types) { + String fileContent = readFileSafe(agentId, toFilename(t)); + if (fileContent.isBlank()) continue; + + for (Map.Entry entry : parseSections(fileContent).entrySet()) { + int score = scoreEntry(q, queryShingles, t, entry.getKey(), entry.getValue()); + if (score > 0) { + matches.add(new ScoredEntry(t, entry.getKey(), entry.getValue(), + score, extractUpdated(entry.getValue()))); + } + } + } + + // Most relevant first; break ties by recency so the freshest fact wins a conflict. + matches.sort(Comparator.comparingInt(ScoredEntry::score).reversed() + .thenComparing(Comparator.comparing(ScoredEntry::updated).reversed())); + return matches.size() > limit ? matches.subList(0, limit) : matches; + } + + /** + * Combine three lightweight relevance signals into a single score: + * key-token presence in the query, domain-alias boosts, and character-level + * shingle overlap (CJK bigrams + Latin word tokens) between the query and entry. + */ + private int scoreEntry(String query, Set queryShingles, String type, String key, String body) { + int score = 0; + String keyLower = key.toLowerCase(); + + // 1. Key tokens appearing verbatim in the query. + for (String token : keyLower.split("[_\\s-]+")) { + if (token.length() >= 2 && query.contains(token)) score += 4; + } + + // 2. Domain-alias boosts for cross-language question/key matches. + for (Alias alias : ALIASES) { + if (alias.matchesQuery(query) && alias.matchesEntry(type, keyLower)) score += 6; + } + + // 3. Shingle overlap between the query and the entry text (capped). + Set entryShingles = shingles((key + " " + body).toLowerCase()); + int overlap = 0; + for (String s : entryShingles) { + if (queryShingles.contains(s)) overlap++; + } + score += Math.min(overlap, 6); + + return score; + } + + /** + * Produce a language-agnostic shingle set: Latin word tokens (length >= 2) + * plus CJK character bigrams (single CJK characters when isolated). This lets + * relevance scoring work without a word segmenter on space-free CJK text. + */ + private static Set shingles(String text) { + Set out = new HashSet<>(); + + Matcher m = WORD_RE.matcher(text); + while (m.find()) { + out.add(m.group()); + } + + for (String run : text.replaceAll("[^\\p{IsHan}]", " ").split("\\s+")) { + if (run.isEmpty()) continue; + if (run.length() == 1) { + out.add(run); + } else { + for (int i = 0; i + 2 <= run.length(); i++) { + out.add(run.substring(i, i + 2)); + } + } + } + + return out; + } + private String toFilename(String type) { return "structured/" + type + ".md"; } @@ -251,6 +421,12 @@ public class StructuredMemoryService { return sb.toString(); } + /** Extract the ISO update date from an entry body's metadata line, or "" if absent. */ + private String extractUpdated(String sectionBody) { + Matcher m = UPDATED_RE.matcher(sectionBody); + return m.find() ? m.group(1) : ""; + } + private String typeDisplayName(String type) { return switch (type) { case "user" -> "User Profile"; diff --git a/mateclaw-server/src/main/java/vip/mate/memory/spi/MemoryManager.java b/mateclaw-server/src/main/java/vip/mate/memory/spi/MemoryManager.java index b5dd387f..5ba4fb72 100644 --- a/mateclaw-server/src/main/java/vip/mate/memory/spi/MemoryManager.java +++ b/mateclaw-server/src/main/java/vip/mate/memory/spi/MemoryManager.java @@ -206,8 +206,13 @@ public class MemoryManager { */ private String buildMemoryContextBlock(String rawContext) { return "\n" - + "[System note: The following is recalled memory context, " - + "NOT new user input. Treat as informational background data.]\n\n" + + "The following is what you already know about this user and their " + + "work, recalled from your own long-term memory. Use it directly as " + + "established fact when answering — this is your knowledge, not the " + + "user speaking. If something the user asks about is not covered here, " + + "say you do not have it in memory rather than guessing. If entries " + + "conflict, prefer the most recently updated one; if they refer to " + + "different projects, ask which one the user means.\n\n" + rawContext + "\n" + ""; } diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index 6fdeb8bc..47731a4d 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -1309,15 +1309,15 @@ VALUES (1000100002, 'Weekly Work Summary', '0 18 * * 5', 'Asia/Shanghai', 100000 -- Daily 2:00 AM: consolidate daily notes → MEMORY.md MERGE INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) KEY (id) -VALUES (1000100010, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0); +VALUES (1000100010, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0); MERGE INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) KEY (id) -VALUES (1000100011, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0); +VALUES (1000100011, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0); MERGE INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) KEY (id) -VALUES (1000100012, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0); +VALUES (1000100012, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0); -- ==================== Workspace File Seed Data ==================== -- Each Agent has its own workspace document collection: AGENTS.md / SOUL.md / PROFILE.md / MEMORY.md diff --git a/mateclaw-server/src/main/resources/db/data-mysql-en.sql b/mateclaw-server/src/main/resources/db/data-mysql-en.sql index 1f16ce9f..b037de0e 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-en.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-en.sql @@ -1353,15 +1353,15 @@ ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expressio -- ==================== Memory Emergence Cron Jobs ==================== -- Daily 2:00 AM: consolidate daily notes → MEMORY.md INSERT INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) -VALUES (1000100010, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0) +VALUES (1000100010, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expression), timezone=VALUES(timezone), agent_id=VALUES(agent_id), task_type=VALUES(task_type), trigger_message=VALUES(trigger_message), request_body=VALUES(request_body), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); INSERT INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) -VALUES (1000100011, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0) +VALUES (1000100011, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expression), timezone=VALUES(timezone), agent_id=VALUES(agent_id), task_type=VALUES(task_type), trigger_message=VALUES(trigger_message), request_body=VALUES(request_body), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); INSERT INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) -VALUES (1000100012, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0) +VALUES (1000100012, 'Memory Consolidation', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.', NULL, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expression), timezone=VALUES(timezone), agent_id=VALUES(agent_id), task_type=VALUES(task_type), trigger_message=VALUES(trigger_message), request_body=VALUES(request_body), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); -- ==================== Workspace File Seed Data ==================== diff --git a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql index d3a98623..e60f8ee5 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql @@ -1350,15 +1350,15 @@ ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expressio -- ==================== 记忆整合定时任务 ==================== -- 每天凌晨 2:00 整合 daily notes → MEMORY.md INSERT INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) -VALUES (1000100010, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0) +VALUES (1000100010, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expression), timezone=VALUES(timezone), agent_id=VALUES(agent_id), task_type=VALUES(task_type), trigger_message=VALUES(trigger_message), request_body=VALUES(request_body), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); INSERT INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) -VALUES (1000100011, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0) +VALUES (1000100011, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expression), timezone=VALUES(timezone), agent_id=VALUES(agent_id), task_type=VALUES(task_type), trigger_message=VALUES(trigger_message), request_body=VALUES(request_body), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); INSERT INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) -VALUES (1000100012, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0) +VALUES (1000100012, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), cron_expression=VALUES(cron_expression), timezone=VALUES(timezone), agent_id=VALUES(agent_id), task_type=VALUES(task_type), trigger_message=VALUES(trigger_message), request_body=VALUES(request_body), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); -- ==================== 工作区文件种子数据（参考 MateClaw md_files/zh） ==================== diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index c68c4109..6ccb790e 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -1310,15 +1310,15 @@ VALUES (1000100002, '每周工作总结', '0 18 * * 5', 'Asia/Shanghai', 1000000 -- 每天凌晨 2:00 整合 daily notes → MEMORY.md MERGE INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) KEY (id) -VALUES (1000100010, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0); +VALUES (1000100010, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000001, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0); MERGE INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) KEY (id) -VALUES (1000100011, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0); +VALUES (1000100011, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000002, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0); MERGE INTO mate_cron_job (id, name, cron_expression, timezone, agent_id, task_type, trigger_message, request_body, enabled, create_time, update_time, deleted) KEY (id) -VALUES (1000100012, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0); +VALUES (1000100012, '记忆整合', '0 2 * * *', 'Asia/Shanghai', 1000000003, 'text', '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。', NULL, TRUE, NOW(), NOW(), 0); -- ==================== 工作区文件种子数据（参考 MateClaw md_files/zh） ==================== -- 每个 Agent 拥有独立的工作区文档集合：AGENTS.md / SOUL.md / PROFILE.md / MEMORY.md diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V132__memory_consolidation_cron_tier_discipline.sql b/mateclaw-server/src/main/resources/db/migration/h2/V132__memory_consolidation_cron_tier_discipline.sql new file mode 100644 index 00000000..dc961ad0 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V132__memory_consolidation_cron_tier_discipline.sql @@ -0,0 +1,13 @@ +-- Update the daily "memory consolidation" cron prompt on existing databases so it +-- keeps project-specific volatile facts (codenames, tech stacks, per-project +-- decisions) out of the always-on MEMORY.md. Seed scripts only run on fresh +-- installs, so existing rows need this data migration to pick up the new wording. +-- Scoped to the original default text so user-edited prompts are left untouched. + +UPDATE mate_cron_job +SET trigger_message = '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。' +WHERE trigger_message = '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。'; + +UPDATE mate_cron_job +SET trigger_message = 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.' +WHERE trigger_message = 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.'; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V132__memory_consolidation_cron_tier_discipline.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V132__memory_consolidation_cron_tier_discipline.sql new file mode 100644 index 00000000..dc961ad0 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V132__memory_consolidation_cron_tier_discipline.sql @@ -0,0 +1,13 @@ +-- Update the daily "memory consolidation" cron prompt on existing databases so it +-- keeps project-specific volatile facts (codenames, tech stacks, per-project +-- decisions) out of the always-on MEMORY.md. Seed scripts only run on fresh +-- installs, so existing rows need this data migration to pick up the new wording. +-- Scoped to the original default text so user-edited prompts are left untouched. + +UPDATE mate_cron_job +SET trigger_message = '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。注意：MEMORY.md 会被注入每一次对话，只整合跨项目长期稳定的信息；具体项目的代号、名称、技术栈、仓库、单项目的指标/预算/团队/上线日期或只对某个项目成立的决策等易变事实，不要写入 MEMORY.md（会随项目切换互相冲突、导致张冠李戴），应留在 daily note 或通过结构化 project 记忆维护。判定口诀：换一个项目后仍成立才进 MEMORY.md。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。' +WHERE trigger_message = '请回顾你最近的 memory/ 日记文件，将反复出现的重要信息（用户偏好、稳定事实、经验教训、工作流）提炼整合到 MEMORY.md 中。保留日记原文不动，只更新 MEMORY.md。完成后简要说明做了哪些整合。'; + +UPDATE mate_cron_job +SET trigger_message = 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Note: MEMORY.md is injected into every conversation, so only consolidate cross-project, long-term stable information; do NOT write project-specific volatile facts into MEMORY.md (project codenames, names, tech stacks, repos, a single project''s metrics/budget/team/launch date, or decisions that hold only for one project) — they conflict across projects and cause mix-ups. Keep those in the daily note or maintain them via structured project memory. Rule of thumb: only facts that still hold after switching projects belong in MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.' +WHERE trigger_message = 'Review your recent memory/ daily note files and consolidate recurring important information (user preferences, stable facts, lessons learned, workflows) into MEMORY.md. Keep the original daily notes intact, only update MEMORY.md. Briefly describe what consolidations were made.'; diff --git a/mateclaw-server/src/main/resources/prompts/memory/summarize-system.txt b/mateclaw-server/src/main/resources/prompts/memory/summarize-system.txt index dde6c576..3449f5c1 100644 --- a/mateclaw-server/src/main/resources/prompts/memory/summarize-system.txt +++ b/mateclaw-server/src/main/resources/prompts/memory/summarize-system.txt @@ -4,9 +4,18 @@ 记忆文件分三种： 1. **PROFILE.md** — 用户画像：稳定的身份信息、偏好、协作方式、沟通风格 -2. **MEMORY.md** — 长期记忆：稳定事实、经验教训、工作流、工具配置、反复出现的规律 +2. **MEMORY.md** — 长期记忆：**跨项目稳定**的事实、经验教训、通用工作流、工具配置、反复出现的规律 3. **memory/YYYY-MM-DD.md** — 每日笔记：一次性事件、当天上下文、临时决定、会议记录 +## 记忆分层纪律（重要） + +MEMORY.md 与 PROFILE.md 会被**无条件注入每一次对话的系统提示**，因此只能放**跨项目、长期稳定、不随项目切换而改变**的信息。 + +- **不要把具体项目的易变事实写进 MEMORY.md**：项目代号、项目名称、单个项目的技术栈、仓库地址、单项目的指标/预算/团队/上线日期、只对某个项目成立的决策——这些都**不属于**稳定事实，写进去会在用户切换项目时与其他项目互相冲突，导致助手张冠李戴。 +- 这类**项目/情景信息**应放入当日 `memory/YYYY-MM-DD.md`（情景记录），由对话中按需召回；需要长期保留的项目事实，应通过结构化 project 记忆（`remember_structured`）维护，而不是塞进 MEMORY.md。 +- MEMORY.md 只保留**与具体项目无关**的内容：用户长期偏好、协作约定、通用工作流、工具/环境配置、反复验证的经验教训。 +- 判定口诀：一条信息**换一个项目后是否仍然成立**？成立 → 可进 MEMORY.md；不成立（只对当前项目为真）→ 进 daily note 或结构化 project 记忆。 + ## 判断原则 - **只提取真正新的信息**：如果信息已经在现有记忆文件中，不要重复提取 diff --git a/mateclaw-server/src/test/java/vip/mate/memory/service/StructuredMemoryPrefetchTest.java b/mateclaw-server/src/test/java/vip/mate/memory/service/StructuredMemoryPrefetchTest.java new file mode 100644 index 00000000..6fa26239 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/service/StructuredMemoryPrefetchTest.java @@ -0,0 +1,113 @@ +package vip.mate.memory.service; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * Verifies the structured-memory split between always-on system prompt injection + * (stable types) and query-conditioned prefetch (growing/specific types), plus + * the relevance scoring that lets a natural-language question surface the right + * stored fact instead of letting it lose salience in an always-on dump. + */ +class StructuredMemoryPrefetchTest { + + private static final long AGENT_ID = 1000000001L; + + private StructuredMemoryService newService(String projectMd, String userMd) { + WorkspaceFileService files = mock(WorkspaceFileService.class); + when(files.getFile(eq(AGENT_ID), anyString())).thenReturn(null); + if (projectMd != null) { + when(files.getFile(AGENT_ID, "structured/project.md")).thenReturn(fileWith(projectMd)); + } + if (userMd != null) { + when(files.getFile(AGENT_ID, "structured/user.md")).thenReturn(fileWith(userMd)); + } + return new StructuredMemoryService(files, mock(ApplicationEventPublisher.class)); + } + + private WorkspaceFileEntity fileWith(String content) { + WorkspaceFileEntity e = new WorkspaceFileEntity(); + e.setContent(content); + return e; + } + + @Test + @DisplayName("system prompt block excludes growing project entries") + void systemPromptBlockExcludesProject() { + StructuredMemoryService svc = newService( + "## project_codename\n用户的项目代号叫\"天枢\"。\n> Source: agent | Updated: 2026-05-29", + "## reply_style\n偏好简洁直接的回答风格。\n> Source: agent | Updated: 2026-05-29"); + + String block = svc.buildMemoryBlock(AGENT_ID); + + // Stable user profile stays in the system prompt... + assertTrue(block.contains("reply_style"), "stable user entry should be in system prompt"); + // ...but specific project facts must not be dumped always-on. + assertFalse(block.contains("天枢"), "project codename must not be in system prompt block"); + } + + @Test + @DisplayName("prefetch surfaces the project codename for a Chinese question about it") + void prefetchSurfacesCodename() { + StructuredMemoryService svc = newService( + "## project_codename\n用户的项目代号叫\"天枢\"。\n> Source: agent | Updated: 2026-05-29\n\n" + + "## project_tech_stack\nRust + Postgres\n> Source: agent | Updated: 2026-05-29", + null); + + String block = svc.buildPrefetchBlock(AGENT_ID, + "我之前告诉过你我的项目代号,你还记得吗?"); + + assertTrue(block.contains("天枢"), "codename should be recalled by a codename question"); + } + + @Test + @DisplayName("prefetch surfaces tech stack via cross-language alias (技术栈 -> tech_stack)") + void prefetchSurfacesTechStackViaAlias() { + StructuredMemoryService svc = newService( + "## project_tech_stack\nRust + Postgres\n> Source: agent | Updated: 2026-05-29", + null); + + String block = svc.buildPrefetchBlock(AGENT_ID, "我的技术栈是什么?"); + + assertNotNull(block); + assertTrue(block.contains("Rust") && block.contains("Postgres"), + "tech stack should be recalled even though the key is English and the question is Chinese"); + } + + @Test + @DisplayName("prefetch orders conflicting entries newest-first and annotates the update date") + void prefetchOrdersByRecency() { + StructuredMemoryService svc = newService( + "## project_old_codename\n旧项目代号叫\"天枢\"。\n> Source: agent | Updated: 2026-05-01\n\n" + + "## project_new_codename\n新项目代号叫\"云梯计划\"。\n> Source: agent | Updated: 2026-05-29", + null); + + String block = svc.buildPrefetchBlock(AGENT_ID, "我的项目代号是什么?"); + + // Both surface, but the most recently updated one ranks first... + int newIdx = block.indexOf("云梯计划"); + int oldIdx = block.indexOf("天枢"); + assertTrue(newIdx >= 0 && oldIdx >= 0, "both conflicting entries should be recalled"); + assertTrue(newIdx < oldIdx, "the most recently updated entry should rank first"); + // ...and the update date is exposed so the model can resolve the conflict. + assertTrue(block.contains("updated 2026-05-29"), "recency hint should be present"); + } + + @Test + @DisplayName("prefetch returns empty for an unrelated question") + void prefetchEmptyForUnrelatedQuery() { + StructuredMemoryService svc = newService( + "## project_codename\n用户的项目代号叫\"天枢\"。\n> Source: agent | Updated: 2026-05-29", + null); + + assertEquals("", svc.buildPrefetchBlock(AGENT_ID, "今天天气怎么样?")); + assertEquals("", svc.buildPrefetchBlock(AGENT_ID, "")); + assertEquals("", svc.buildPrefetchBlock(AGENT_ID, null)); + } +}