diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/CompositeEntityExtractor.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/CompositeEntityExtractor.java new file mode 100644 index 00000000..c4b17aad --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/CompositeEntityExtractor.java @@ -0,0 +1,36 @@ +package vip.mate.memory.fact.extraction; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +/** + * Composite extractor — combines pattern + LLM extractors. + * Deduplicates by sourceRef. + * + * @author MateClaw Team + */ +@Component +@RequiredArgsConstructor +public class CompositeEntityExtractor implements EntityExtractor { + + private final PatternEntityExtractor patternExtractor; + private final LlmEntityExtractor llmExtractor; + + @Override + public List extract(Long agentId, String filename, String content) { + List results = new ArrayList<>(patternExtractor.extract(agentId, filename, content)); + + // Add LLM-extracted facts that don't duplicate pattern ones + List llmFacts = llmExtractor.extract(agentId, filename, content); + for (ExtractedFact f : llmFacts) { + if (results.stream().noneMatch(r -> r.sourceRef().equals(f.sourceRef()))) { + results.add(f); + } + } + + return results; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/EntityExtractor.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/EntityExtractor.java new file mode 100644 index 00000000..92187923 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/EntityExtractor.java @@ -0,0 +1,13 @@ +package vip.mate.memory.fact.extraction; + +import java.util.List; + +/** + * Strategy interface for extracting facts from markdown content. + * + * @author MateClaw Team + */ +public interface EntityExtractor { + + List extract(Long agentId, String filename, String content); +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/ExtractedFact.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/ExtractedFact.java new file mode 100644 index 00000000..c055146a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/ExtractedFact.java @@ -0,0 +1,16 @@ +package vip.mate.memory.fact.extraction; + +/** + * A single fact extracted from canonical memory content. + * + * @author MateClaw Team + */ +public record ExtractedFact( + String sourceRef, + String category, + String subject, + String predicate, + String objectValue, + double confidence, + String extractedBy +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/LlmEntityExtractor.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/LlmEntityExtractor.java new file mode 100644 index 00000000..433e8724 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/LlmEntityExtractor.java @@ -0,0 +1,35 @@ +package vip.mate.memory.fact.extraction; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.memory.MemoryProperties; + +import java.util.List; + +/** + * LLM-based fact extractor — uses the LLM to identify entities and relationships. + * Only active when fact.llm-extraction-enabled=true; otherwise returns empty. + *

+ * Phase 3 L1: placeholder that delegates to pattern extractor. + * Full LLM implementation in Phase 3 L2+. + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class LlmEntityExtractor implements EntityExtractor { + + private final MemoryProperties properties; + + @Override + public List extract(Long agentId, String filename, String content) { + if (!properties.getFact().isLlmExtractionEnabled()) { + return List.of(); // LLM extraction disabled + } + // TODO: Phase 3 L2+ — call LLM with entity extraction prompt + log.debug("[FactExtract] LLM extraction not yet implemented, returning empty for agent={}", agentId); + return List.of(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/PatternEntityExtractor.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/PatternEntityExtractor.java new file mode 100644 index 00000000..3508328e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/PatternEntityExtractor.java @@ -0,0 +1,81 @@ +package vip.mate.memory.fact.extraction; + +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Pattern-based fact extractor — parses structured markdown sections + * into subject-predicate-object triples using regex patterns. + *

+ * Handles formats like: + * - "## key\ncontent" sections in structured/*.md + * - "- **key**: value" bullet patterns in MEMORY.md + * + * @author MateClaw Team + */ +@Component +public class PatternEntityExtractor implements EntityExtractor { + + private static final Pattern SECTION_HEADER = Pattern.compile("^## (.+)$", Pattern.MULTILINE); + private static final Pattern KV_BULLET = Pattern.compile("^- \\*\\*(.+?)\\*\\*:\\s*(.+)$", Pattern.MULTILINE); + private static final Pattern SIMPLE_BULLET = Pattern.compile("^- (.+)$", Pattern.MULTILINE); + + @Override + public List extract(Long agentId, String filename, String content) { + List facts = new ArrayList<>(); + if (content == null || content.isBlank()) return facts; + + // Determine category from filename + String category = inferCategory(filename); + + // Extract key-value bullets: - **key**: value + Matcher kvMatcher = KV_BULLET.matcher(content); + while (kvMatcher.find()) { + String key = kvMatcher.group(1).trim(); + String value = kvMatcher.group(2).trim(); + if (value.isBlank() || value.equals(":")) continue; + String sourceRef = filename + "#" + toSlug(key); + facts.add(new ExtractedFact(sourceRef, category, key, "is", value, 0.9, "pattern")); + } + + // Extract section-based facts from structured files + if (filename.startsWith("structured/")) { + String[] sections = content.split("(?=^## )", Pattern.MULTILINE); + for (String section : sections) { + Matcher headerM = SECTION_HEADER.matcher(section); + if (!headerM.find()) continue; + String heading = headerM.group(1).trim(); + String body = section.substring(headerM.end()).trim(); + if (body.isBlank()) continue; + + // Each non-empty line under a heading is a fact + String sourceRef = filename + "#" + toSlug(heading); + // Avoid duplicate if already captured as KV bullet + if (facts.stream().anyMatch(f -> f.sourceRef().equals(sourceRef))) continue; + + String firstLine = body.split("\n")[0].replaceAll("^[-*>]+\\s*", "").trim(); + if (firstLine.length() >= 5) { + facts.add(new ExtractedFact(sourceRef, category, heading, "has", firstLine, 0.8, "pattern")); + } + } + } + + return facts; + } + + private String inferCategory(String filename) { + if (filename.contains("user")) return "user_pref"; + if (filename.contains("project")) return "project"; + if (filename.contains("reference")) return "reference"; + if (filename.contains("feedback")) return "feedback"; + return "general"; + } + + private String toSlug(String s) { + return s.toLowerCase().replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", "_").replaceAll("^_|_$", ""); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactContradictionEntity.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactContradictionEntity.java new file mode 100644 index 00000000..b2e8b6c0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactContradictionEntity.java @@ -0,0 +1,42 @@ +package vip.mate.memory.fact.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Contradiction between two facts, detected during Dream consolidation. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_fact_contradiction") +public class FactContradictionEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long agentId; + + private Long factAId; + + private Long factBId; + + private String description; + + /** null | KEEP_A | KEEP_B | MERGE | IGNORE */ + private String resolution; + + private LocalDateTime resolvedAt; + + private String resolvedBy; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntity.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntity.java new file mode 100644 index 00000000..e0bcbee0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntity.java @@ -0,0 +1,60 @@ +package vip.mate.memory.fact.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Fact projection entity — read-only derived view of canonical memory. + * Derived columns (subject, predicate, object_value, confidence, trust, category) + * are rebuilt by FactProjectionBuilder. + * Accumulated columns (last_used_at, use_count) are only written by FactQueryService.bumpUseCount. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_fact") +public class FactEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long agentId; + + /** Canonical source reference, e.g. "structured/user.md#preferred_language" */ + private String sourceRef; + + /** Category: user_pref, project, tool, general */ + private String category; + + private String subject; + + private String predicate; + + @TableField("object_value") + private String objectValue; + + /** Extraction confidence [0..1] */ + private Double confidence; + + /** Trust score derived from feedback + time decay */ + private Double trust; + + // --- Accumulated columns (preserved across rebuilds) --- + + private LocalDateTime lastUsedAt; + + private Integer useCount; + + /** pattern | llm */ + private String extractedBy; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java new file mode 100644 index 00000000..2fe75991 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java @@ -0,0 +1,32 @@ +package vip.mate.memory.fact.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Entity reference for multi-hop graph queries on facts. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_fact_entity_ref") +public class FactEntityRefEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long factId; + + private String entityName; + + /** person, tool, project, concept */ + private String entityType; + + /** subject | object */ + private String role; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/projection/FactProjectionBuilder.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/projection/FactProjectionBuilder.java new file mode 100644 index 00000000..1ab6667f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/projection/FactProjectionBuilder.java @@ -0,0 +1,103 @@ +package vip.mate.memory.fact.projection; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.memory.MemoryProperties; +import vip.mate.memory.fact.extraction.CompositeEntityExtractor; +import vip.mate.memory.fact.extraction.ExtractedFact; +import vip.mate.memory.fact.repository.FactMapper; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +/** + * Rebuilds the fact projection from canonical sources. + *

+ * Derived columns are overwritten; accumulated columns (use_count, last_used_at) + * are preserved via MERGE/upsert keyed on (agent_id, source_ref). + *

+ * Only this class may write derived columns to mate_fact (core invariant). + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class FactProjectionBuilder { + + private final FactMapper factMapper; + private final WorkspaceFileService workspaceFileService; + private final CompositeEntityExtractor extractor; + private final MemoryProperties properties; + + /** + * Full rebuild for an agent. Extracts facts from all canonical sources, + * upserts derived columns, and soft-deletes stale entries. + */ + public int rebuildAll(Long agentId) { + if (!properties.getFact().isProjectionEnabled()) { + log.debug("[FactProjection] Projection disabled, skipping rebuildAll for agent={}", agentId); + return 0; + } + + List allFacts = new ArrayList<>(); + + // Extract from structured/*.md files + List files = workspaceFileService.listFiles(agentId); + for (WorkspaceFileEntity file : files) { + String filename = file.getFilename(); + if (filename == null) continue; + if (filename.startsWith("structured/") && filename.endsWith(".md")) { + WorkspaceFileEntity full = workspaceFileService.getFile(agentId, filename); + if (full != null && full.getContent() != null && !full.getContent().isBlank()) { + allFacts.addAll(extractor.extract(agentId, filename, full.getContent())); + } + } + } + + // Extract from MEMORY.md + WorkspaceFileEntity memoryFile = workspaceFileService.getFile(agentId, "MEMORY.md"); + if (memoryFile != null && memoryFile.getContent() != null && !memoryFile.getContent().isBlank()) { + allFacts.addAll(extractor.extract(agentId, "MEMORY.md", memoryFile.getContent())); + } + + // Upsert all extracted facts + LocalDateTime now = LocalDateTime.now(); + List keepRefs = new ArrayList<>(); + for (ExtractedFact fact : allFacts) { + factMapper.upsertDerivedH2(agentId, fact.sourceRef(), fact.category(), + fact.subject(), fact.predicate(), fact.objectValue(), + fact.confidence(), 0.5, fact.extractedBy(), now, now); + keepRefs.add(fact.sourceRef()); + } + + // Remove stale facts + if (!keepRefs.isEmpty()) { + factMapper.deleteByAgentIdAndSourceRefNotIn(agentId, keepRefs, now); + } + + log.info("[FactProjection] rebuildAll: agent={}, facts={}", agentId, allFacts.size()); + return allFacts.size(); + } + + /** + * Incremental rebuild for a single file change. + */ + public int rebuildOne(Long agentId, String filename, String content) { + if (!properties.getFact().isProjectionEnabled()) return 0; + + List facts = extractor.extract(agentId, filename, content); + LocalDateTime now = LocalDateTime.now(); + for (ExtractedFact fact : facts) { + factMapper.upsertDerivedH2(agentId, fact.sourceRef(), fact.category(), + fact.subject(), fact.predicate(), fact.objectValue(), + fact.confidence(), 0.5, fact.extractedBy(), now, now); + } + log.debug("[FactProjection] rebuildOne: agent={}, file={}, facts={}", agentId, filename, facts.size()); + return facts.size(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactMapper.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactMapper.java new file mode 100644 index 00000000..ff257925 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactMapper.java @@ -0,0 +1,75 @@ +package vip.mate.memory.fact.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Update; +import vip.mate.memory.fact.model.FactEntity; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * Fact mapper — limited write operations enforce the core invariant: + * - Derived columns: only FactProjectionBuilder may write + * - Accumulated columns: only bumpUseCount may write + * + * @author MateClaw Team + */ +@Mapper +public interface FactMapper extends BaseMapper { + + /** + * Upsert derived columns by (agent_id, source_ref). + * Preserves accumulated columns (last_used_at, use_count). + */ + @Update(""" + MERGE INTO mate_fact (agent_id, source_ref, category, subject, predicate, object_value, + confidence, trust, extracted_by, create_time, update_time, deleted) + KEY (agent_id, source_ref) + VALUES (#{agentId}, #{sourceRef}, #{category}, #{subject}, #{predicate}, #{objectValue}, + #{confidence}, #{trust}, #{extractedBy}, #{createTime}, #{updateTime}, 0) + """) + void upsertDerivedH2(@Param("agentId") Long agentId, + @Param("sourceRef") String sourceRef, + @Param("category") String category, + @Param("subject") String subject, + @Param("predicate") String predicate, + @Param("objectValue") String objectValue, + @Param("confidence") Double confidence, + @Param("trust") Double trust, + @Param("extractedBy") String extractedBy, + @Param("createTime") LocalDateTime createTime, + @Param("updateTime") LocalDateTime updateTime); + + /** + * Bump use_count and last_used_at for the given fact IDs. + * This is the ONLY path that writes accumulated columns. + */ + @Update(""" + + """) + void bumpUseCount(@Param("ids") List ids, @Param("now") LocalDateTime now); + + /** + * Soft-delete facts whose source_ref is no longer in the canonical set. + * Used during full rebuild to remove stale projections. + */ + @Update(""" + + """) + void deleteByAgentIdAndSourceRefNotIn(@Param("agentId") Long agentId, + @Param("keepSet") List keepSet, + @Param("now") LocalDateTime now); +} diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V29__memory_fact_projection.sql b/mateclaw-server/src/main/resources/db/migration/h2/V29__memory_fact_projection.sql new file mode 100644 index 00000000..66eaf306 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V29__memory_fact_projection.sql @@ -0,0 +1,56 @@ +-- Dream v2 Phase 3: Fact projection tables (read-only derived from canonical) +-- Ref: rfc-038 §3.3 + +CREATE TABLE IF NOT EXISTS mate_fact ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + agent_id BIGINT NOT NULL, + source_ref VARCHAR(512) NOT NULL, -- canonical source: "structured/user.md#section_key" or "MEMORY.md#heading" + category VARCHAR(64), -- user_pref, project, tool, general + subject VARCHAR(256), -- entity subject + predicate VARCHAR(256), -- relationship or property + object_value TEXT, -- entity object or value + confidence DOUBLE DEFAULT 1.0, -- extraction confidence [0..1] + trust DOUBLE DEFAULT 0.5, -- derived trust score (feedback + decay) + -- Accumulated columns (NOT overwritten by projection rebuild) + last_used_at DATETIME, + use_count INT DEFAULT 0, + -- Metadata + extracted_by VARCHAR(32) DEFAULT 'pattern', -- pattern | llm + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted TINYINT DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_fact_agent ON mate_fact(agent_id, deleted); +CREATE INDEX IF NOT EXISTS idx_fact_agent_source ON mate_fact(agent_id, source_ref); +CREATE INDEX IF NOT EXISTS idx_fact_agent_subject ON mate_fact(agent_id, subject); + +-- Entity references (for multi-hop queries) +CREATE TABLE IF NOT EXISTS mate_fact_entity_ref ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + fact_id BIGINT NOT NULL, + entity_name VARCHAR(256) NOT NULL, + entity_type VARCHAR(64), -- person, tool, project, concept + role VARCHAR(32) NOT NULL, -- subject | object + create_time DATETIME NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_fact_ref_entity ON mate_fact_entity_ref(entity_name, entity_type); +CREATE INDEX IF NOT EXISTS idx_fact_ref_fact ON mate_fact_entity_ref(fact_id); + +-- Contradiction tracking (populated by Dream contradiction detection step) +CREATE TABLE IF NOT EXISTS mate_fact_contradiction ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + agent_id BIGINT NOT NULL, + fact_a_id BIGINT NOT NULL, + fact_b_id BIGINT NOT NULL, + description TEXT, + resolution VARCHAR(32), -- null | KEEP_A | KEEP_B | MERGE | IGNORE + resolved_at DATETIME, + resolved_by VARCHAR(64), + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted TINYINT DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_contradiction_agent ON mate_fact_contradiction(agent_id, resolution); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V29__memory_fact_projection.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V29__memory_fact_projection.sql new file mode 100644 index 00000000..c10edb26 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V29__memory_fact_projection.sql @@ -0,0 +1,49 @@ +-- Dream v2 Phase 3: Fact projection tables (read-only derived from canonical) +-- Ref: rfc-038 §3.3 + +CREATE TABLE IF NOT EXISTS mate_fact ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + agent_id BIGINT NOT NULL, + source_ref VARCHAR(512) NOT NULL, + category VARCHAR(64), + subject VARCHAR(256), + predicate VARCHAR(256), + object_value TEXT, + confidence DOUBLE DEFAULT 1.0, + trust DOUBLE DEFAULT 0.5, + last_used_at DATETIME, + use_count INT DEFAULT 0, + extracted_by VARCHAR(32) DEFAULT 'pattern', + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted TINYINT DEFAULT 0, + INDEX idx_fact_agent (agent_id, deleted), + INDEX idx_fact_agent_source (agent_id, source_ref(255)), + INDEX idx_fact_agent_subject (agent_id, subject(255)) +); + +CREATE TABLE IF NOT EXISTS mate_fact_entity_ref ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + fact_id BIGINT NOT NULL, + entity_name VARCHAR(256) NOT NULL, + entity_type VARCHAR(64), + role VARCHAR(32) NOT NULL, + create_time DATETIME NOT NULL, + INDEX idx_fact_ref_entity (entity_name(128), entity_type), + INDEX idx_fact_ref_fact (fact_id) +); + +CREATE TABLE IF NOT EXISTS mate_fact_contradiction ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + agent_id BIGINT NOT NULL, + fact_a_id BIGINT NOT NULL, + fact_b_id BIGINT NOT NULL, + description TEXT, + resolution VARCHAR(32), + resolved_at DATETIME, + resolved_by VARCHAR(64), + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted TINYINT DEFAULT 0, + INDEX idx_contradiction_agent (agent_id, resolution) +);