opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-15 11:58:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(memory): dream-v2 E1 — Fact Projection foundation

Browse Source
This commit is contained in:
matevip 2026-04-21 04:58:47 +08:00
parent bf4cebb7b3
commit acc6f448db
12 changed files with 598 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/CompositeEntityExtractor.java Normal file
View File

@ -0,0 +1,36 @@
package vip.mate.memory.fact.extraction;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
/**
* Composite extractor — combines pattern + LLM extractors.
* Deduplicates by sourceRef.
*
* @author MateClaw Team
*/
@Component
@RequiredArgsConstructor
public class CompositeEntityExtractor implements EntityExtractor {
private final PatternEntityExtractor patternExtractor;
private final LlmEntityExtractor llmExtractor;
@Override
public List<ExtractedFact> extract(Long agentId, String filename, String content) {
List<ExtractedFact> results = new ArrayList<>(patternExtractor.extract(agentId, filename, content));
// Add LLM-extracted facts that don't duplicate pattern ones
List<ExtractedFact> llmFacts = llmExtractor.extract(agentId, filename, content);
for (ExtractedFact f : llmFacts) {
if (results.stream().noneMatch(r -> r.sourceRef().equals(f.sourceRef()))) {
results.add(f);
}
}
return results;
}
}

13
mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/EntityExtractor.java Normal file
View File

@ -0,0 +1,13 @@
package vip.mate.memory.fact.extraction;
import java.util.List;
/**
* Strategy interface for extracting facts from markdown content.
*
* @author MateClaw Team
*/
public interface EntityExtractor {
List<ExtractedFact> extract(Long agentId, String filename, String content);
}

16
mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/ExtractedFact.java Normal file
View File

@ -0,0 +1,16 @@
package vip.mate.memory.fact.extraction;
/**
* A single fact extracted from canonical memory content.
*
* @author MateClaw Team
*/
public record ExtractedFact(
String sourceRef,
String category,
String subject,
String predicate,
String objectValue,
double confidence,
String extractedBy
) {}

35
mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/LlmEntityExtractor.java Normal file
View File

@ -0,0 +1,35 @@
package vip.mate.memory.fact.extraction;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import vip.mate.memory.MemoryProperties;
import java.util.List;
/**
* LLM-based fact extractor — uses the LLM to identify entities and relationships.
* Only active when fact.llm-extraction-enabled=true; otherwise returns empty.
* <p>
* Phase 3 L1: placeholder that delegates to pattern extractor.
* Full LLM implementation in Phase 3 L2+.
*
* @author MateClaw Team
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class LlmEntityExtractor implements EntityExtractor {
private final MemoryProperties properties;
@Override
public List<ExtractedFact> extract(Long agentId, String filename, String content) {
if (!properties.getFact().isLlmExtractionEnabled()) {
return List.of(); // LLM extraction disabled
}
// TODO: Phase 3 L2+ — call LLM with entity extraction prompt
log.debug("[FactExtract] LLM extraction not yet implemented, returning empty for agent={}", agentId);
return List.of();
}
}

81
mateclaw-server/src/main/java/vip/mate/memory/fact/extraction/PatternEntityExtractor.java Normal file
View File

@ -0,0 +1,81 @@
package vip.mate.memory.fact.extraction;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Pattern-based fact extractor — parses structured markdown sections
* into subject-predicate-object triples using regex patterns.
* <p>
* Handles formats like:
* - "## key\ncontent" sections in structured/*.md
* - "- **key**: value" bullet patterns in MEMORY.md
*
* @author MateClaw Team
*/
@Component
public class PatternEntityExtractor implements EntityExtractor {
private static final Pattern SECTION_HEADER = Pattern.compile("^## (.+)$", Pattern.MULTILINE);
private static final Pattern KV_BULLET = Pattern.compile("^- \\*\\*(.+?)\\*\\*:\\s*(.+)$", Pattern.MULTILINE);
private static final Pattern SIMPLE_BULLET = Pattern.compile("^- (.+)$", Pattern.MULTILINE);
@Override
public List<ExtractedFact> extract(Long agentId, String filename, String content) {
List<ExtractedFact> facts = new ArrayList<>();
if (content == null || content.isBlank()) return facts;
// Determine category from filename
String category = inferCategory(filename);
// Extract key-value bullets: - **key**: value
Matcher kvMatcher = KV_BULLET.matcher(content);
while (kvMatcher.find()) {
String key = kvMatcher.group(1).trim();
String value = kvMatcher.group(2).trim();
if (value.isBlank() || value.equals(":")) continue;
String sourceRef = filename + "#" + toSlug(key);
facts.add(new ExtractedFact(sourceRef, category, key, "is", value, 0.9, "pattern"));
}
// Extract section-based facts from structured files
if (filename.startsWith("structured/")) {
String[] sections = content.split("(?=^## )", Pattern.MULTILINE);
for (String section : sections) {
Matcher headerM = SECTION_HEADER.matcher(section);
if (!headerM.find()) continue;
String heading = headerM.group(1).trim();
String body = section.substring(headerM.end()).trim();
if (body.isBlank()) continue;
// Each non-empty line under a heading is a fact
String sourceRef = filename + "#" + toSlug(heading);
// Avoid duplicate if already captured as KV bullet
if (facts.stream().anyMatch(f -> f.sourceRef().equals(sourceRef))) continue;
String firstLine = body.split("\n")[0].replaceAll("^[-*>]+\\s*", "").trim();
if (firstLine.length() >= 5) {
facts.add(new ExtractedFact(sourceRef, category, heading, "has", firstLine, 0.8, "pattern"));
}
}
}
return facts;
}
private String inferCategory(String filename) {
if (filename.contains("user")) return "user_pref";
if (filename.contains("project")) return "project";
if (filename.contains("reference")) return "reference";
if (filename.contains("feedback")) return "feedback";
return "general";
}
private String toSlug(String s) {
return s.toLowerCase().replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", "_").replaceAll("^_|_$", "");
}
}

42
mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactContradictionEntity.java Normal file
View File

@ -0,0 +1,42 @@
package vip.mate.memory.fact.model;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* Contradiction between two facts, detected during Dream consolidation.
*
* @author MateClaw Team
*/
@Data
@TableName("mate_fact_contradiction")
public class FactContradictionEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long agentId;
private Long factAId;
private Long factBId;
private String description;
/** null | KEEP_A | KEEP_B | MERGE | IGNORE */
private String resolution;
private LocalDateTime resolvedAt;
private String resolvedBy;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
private Integer deleted;
}

60
mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntity.java Normal file
View File

@ -0,0 +1,60 @@
package vip.mate.memory.fact.model;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* Fact projection entity — read-only derived view of canonical memory.
* Derived columns (subject, predicate, object_value, confidence, trust, category)
* are rebuilt by FactProjectionBuilder.
* Accumulated columns (last_used_at, use_count) are only written by FactQueryService.bumpUseCount.
*
* @author MateClaw Team
*/
@Data
@TableName("mate_fact")
public class FactEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long agentId;
/** Canonical source reference, e.g. "structured/user.md#preferred_language" */
private String sourceRef;
/** Category: user_pref, project, tool, general */
private String category;
private String subject;
private String predicate;
@TableField("object_value")
private String objectValue;
/** Extraction confidence [0..1] */
private Double confidence;
/** Trust score derived from feedback + time decay */
private Double trust;
// --- Accumulated columns (preserved across rebuilds) ---
private LocalDateTime lastUsedAt;
private Integer useCount;
/** pattern | llm */
private String extractedBy;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
private Integer deleted;
}

32
mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java Normal file
View File

@ -0,0 +1,32 @@
package vip.mate.memory.fact.model;
import com.baomidou.mybatisplus.annotation.*;
import lombok.Data;
import java.time.LocalDateTime;
/**
* Entity reference for multi-hop graph queries on facts.
*
* @author MateClaw Team
*/
@Data
@TableName("mate_fact_entity_ref")
public class FactEntityRefEntity {
@TableId(type = IdType.AUTO)
private Long id;
private Long factId;
private String entityName;
/** person, tool, project, concept */
private String entityType;
/** subject | object */
private String role;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
}

103
mateclaw-server/src/main/java/vip/mate/memory/fact/projection/FactProjectionBuilder.java Normal file
View File

@ -0,0 +1,103 @@
package vip.mate.memory.fact.projection;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import vip.mate.memory.MemoryProperties;
import vip.mate.memory.fact.extraction.CompositeEntityExtractor;
import vip.mate.memory.fact.extraction.ExtractedFact;
import vip.mate.memory.fact.repository.FactMapper;
import vip.mate.workspace.document.WorkspaceFileService;
import vip.mate.workspace.document.model.WorkspaceFileEntity;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
/**
* Rebuilds the fact projection from canonical sources.
* <p>
* Derived columns are overwritten; accumulated columns (use_count, last_used_at)
* are preserved via MERGE/upsert keyed on (agent_id, source_ref).
* <p>
* Only this class may write derived columns to mate_fact (core invariant).
*
* @author MateClaw Team
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class FactProjectionBuilder {
private final FactMapper factMapper;
private final WorkspaceFileService workspaceFileService;
private final CompositeEntityExtractor extractor;
private final MemoryProperties properties;
/**
* Full rebuild for an agent. Extracts facts from all canonical sources,
* upserts derived columns, and soft-deletes stale entries.
*/
public int rebuildAll(Long agentId) {
if (!properties.getFact().isProjectionEnabled()) {
log.debug("[FactProjection] Projection disabled, skipping rebuildAll for agent={}", agentId);
return 0;
}
List<ExtractedFact> allFacts = new ArrayList<>();
// Extract from structured/*.md files
List<WorkspaceFileEntity> files = workspaceFileService.listFiles(agentId);
for (WorkspaceFileEntity file : files) {
String filename = file.getFilename();
if (filename == null) continue;
if (filename.startsWith("structured/") && filename.endsWith(".md")) {
WorkspaceFileEntity full = workspaceFileService.getFile(agentId, filename);
if (full != null && full.getContent() != null && !full.getContent().isBlank()) {
allFacts.addAll(extractor.extract(agentId, filename, full.getContent()));
}
}
}
// Extract from MEMORY.md
WorkspaceFileEntity memoryFile = workspaceFileService.getFile(agentId, "MEMORY.md");
if (memoryFile != null && memoryFile.getContent() != null && !memoryFile.getContent().isBlank()) {
allFacts.addAll(extractor.extract(agentId, "MEMORY.md", memoryFile.getContent()));
}
// Upsert all extracted facts
LocalDateTime now = LocalDateTime.now();
List<String> keepRefs = new ArrayList<>();
for (ExtractedFact fact : allFacts) {
factMapper.upsertDerivedH2(agentId, fact.sourceRef(), fact.category(),
fact.subject(), fact.predicate(), fact.objectValue(),
fact.confidence(), 0.5, fact.extractedBy(), now, now);
keepRefs.add(fact.sourceRef());
}
// Remove stale facts
if (!keepRefs.isEmpty()) {
factMapper.deleteByAgentIdAndSourceRefNotIn(agentId, keepRefs, now);
}
log.info("[FactProjection] rebuildAll: agent={}, facts={}", agentId, allFacts.size());
return allFacts.size();
}
/**
* Incremental rebuild for a single file change.
*/
public int rebuildOne(Long agentId, String filename, String content) {
if (!properties.getFact().isProjectionEnabled()) return 0;
List<ExtractedFact> facts = extractor.extract(agentId, filename, content);
LocalDateTime now = LocalDateTime.now();
for (ExtractedFact fact : facts) {
factMapper.upsertDerivedH2(agentId, fact.sourceRef(), fact.category(),
fact.subject(), fact.predicate(), fact.objectValue(),
fact.confidence(), 0.5, fact.extractedBy(), now, now);
}
log.debug("[FactProjection] rebuildOne: agent={}, file={}, facts={}", agentId, filename, facts.size());
return facts.size();
}
}

75
mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactMapper.java Normal file
View File

@ -0,0 +1,75 @@
package vip.mate.memory.fact.repository;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import vip.mate.memory.fact.model.FactEntity;
import java.time.LocalDateTime;
import java.util.List;
/**
* Fact mapper — limited write operations enforce the core invariant:
* - Derived columns: only FactProjectionBuilder may write
* - Accumulated columns: only bumpUseCount may write
*
* @author MateClaw Team
*/
@Mapper
public interface FactMapper extends BaseMapper<FactEntity> {
/**
* Upsert derived columns by (agent_id, source_ref).
* Preserves accumulated columns (last_used_at, use_count).
*/
@Update("""
MERGE INTO mate_fact (agent_id, source_ref, category, subject, predicate, object_value,
confidence, trust, extracted_by, create_time, update_time, deleted)
KEY (agent_id, source_ref)
VALUES (#{agentId}, #{sourceRef}, #{category}, #{subject}, #{predicate}, #{objectValue},
#{confidence}, #{trust}, #{extractedBy}, #{createTime}, #{updateTime}, 0)
""")
void upsertDerivedH2(@Param("agentId") Long agentId,
@Param("sourceRef") String sourceRef,
@Param("category") String category,
@Param("subject") String subject,
@Param("predicate") String predicate,
@Param("objectValue") String objectValue,
@Param("confidence") Double confidence,
@Param("trust") Double trust,
@Param("extractedBy") String extractedBy,
@Param("createTime") LocalDateTime createTime,
@Param("updateTime") LocalDateTime updateTime);
/**
* Bump use_count and last_used_at for the given fact IDs.
* This is the ONLY path that writes accumulated columns.
*/
@Update("""
<script>
UPDATE mate_fact
SET use_count = use_count + 1, last_used_at = #{now}, update_time = #{now}
WHERE id IN
<foreach item='id' collection='ids' open='(' separator=',' close=')'>#{id}</foreach>
AND deleted = 0
</script>
""")
void bumpUseCount(@Param("ids") List<Long> ids, @Param("now") LocalDateTime now);
/**
* Soft-delete facts whose source_ref is no longer in the canonical set.
* Used during full rebuild to remove stale projections.
*/
@Update("""
<script>
UPDATE mate_fact SET deleted = 1, update_time = #{now}
WHERE agent_id = #{agentId} AND deleted = 0
AND source_ref NOT IN
<foreach item='ref' collection='keepSet' open='(' separator=',' close=')'>#{ref}</foreach>
</script>
""")
void deleteByAgentIdAndSourceRefNotIn(@Param("agentId") Long agentId,
@Param("keepSet") List<String> keepSet,
@Param("now") LocalDateTime now);
}

56
mateclaw-server/src/main/resources/db/migration/h2/V29__memory_fact_projection.sql Normal file
View File

@ -0,0 +1,56 @@
-- Dream v2 Phase 3: Fact projection tables (read-only derived from canonical)
-- Ref: rfc-038 §3.3
CREATE TABLE IF NOT EXISTS mate_fact (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
agent_id BIGINT NOT NULL,
source_ref VARCHAR(512) NOT NULL, -- canonical source: "structured/user.md#section_key" or "MEMORY.md#heading"
category VARCHAR(64), -- user_pref, project, tool, general
subject VARCHAR(256), -- entity subject
predicate VARCHAR(256), -- relationship or property
object_value TEXT, -- entity object or value
confidence DOUBLE DEFAULT 1.0, -- extraction confidence [0..1]
trust DOUBLE DEFAULT 0.5, -- derived trust score (feedback + decay)
-- Accumulated columns (NOT overwritten by projection rebuild)
last_used_at DATETIME,
use_count INT DEFAULT 0,
-- Metadata
extracted_by VARCHAR(32) DEFAULT 'pattern', -- pattern | llm
create_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
deleted TINYINT DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_fact_agent ON mate_fact(agent_id, deleted);
CREATE INDEX IF NOT EXISTS idx_fact_agent_source ON mate_fact(agent_id, source_ref);
CREATE INDEX IF NOT EXISTS idx_fact_agent_subject ON mate_fact(agent_id, subject);
-- Entity references (for multi-hop queries)
CREATE TABLE IF NOT EXISTS mate_fact_entity_ref (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
fact_id BIGINT NOT NULL,
entity_name VARCHAR(256) NOT NULL,
entity_type VARCHAR(64), -- person, tool, project, concept
role VARCHAR(32) NOT NULL, -- subject | object
create_time DATETIME NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_fact_ref_entity ON mate_fact_entity_ref(entity_name, entity_type);
CREATE INDEX IF NOT EXISTS idx_fact_ref_fact ON mate_fact_entity_ref(fact_id);
-- Contradiction tracking (populated by Dream contradiction detection step)
CREATE TABLE IF NOT EXISTS mate_fact_contradiction (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
agent_id BIGINT NOT NULL,
fact_a_id BIGINT NOT NULL,
fact_b_id BIGINT NOT NULL,
description TEXT,
resolution VARCHAR(32), -- null | KEEP_A | KEEP_B | MERGE | IGNORE
resolved_at DATETIME,
resolved_by VARCHAR(64),
create_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
deleted TINYINT DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_contradiction_agent ON mate_fact_contradiction(agent_id, resolution);

49
mateclaw-server/src/main/resources/db/migration/mysql/V29__memory_fact_projection.sql Normal file
View File

@ -0,0 +1,49 @@
-- Dream v2 Phase 3: Fact projection tables (read-only derived from canonical)
-- Ref: rfc-038 §3.3
CREATE TABLE IF NOT EXISTS mate_fact (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
agent_id BIGINT NOT NULL,
source_ref VARCHAR(512) NOT NULL,
category VARCHAR(64),
subject VARCHAR(256),
predicate VARCHAR(256),
object_value TEXT,
confidence DOUBLE DEFAULT 1.0,
trust DOUBLE DEFAULT 0.5,
last_used_at DATETIME,
use_count INT DEFAULT 0,
extracted_by VARCHAR(32) DEFAULT 'pattern',
create_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
deleted TINYINT DEFAULT 0,
INDEX idx_fact_agent (agent_id, deleted),
INDEX idx_fact_agent_source (agent_id, source_ref(255)),
INDEX idx_fact_agent_subject (agent_id, subject(255))
);
CREATE TABLE IF NOT EXISTS mate_fact_entity_ref (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
fact_id BIGINT NOT NULL,
entity_name VARCHAR(256) NOT NULL,
entity_type VARCHAR(64),
role VARCHAR(32) NOT NULL,
create_time DATETIME NOT NULL,
INDEX idx_fact_ref_entity (entity_name(128), entity_type),
INDEX idx_fact_ref_fact (fact_id)
);
CREATE TABLE IF NOT EXISTS mate_fact_contradiction (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
agent_id BIGINT NOT NULL,
fact_a_id BIGINT NOT NULL,
fact_b_id BIGINT NOT NULL,
description TEXT,
resolution VARCHAR(32),
resolved_at DATETIME,
resolved_by VARCHAR(64),
create_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
deleted TINYINT DEFAULT 0,
INDEX idx_contradiction_agent (agent_id, resolution)
);