From 6e7c137154e5d5f11d89cea8f7cc2e0978f3b9f0 Mon Sep 17 00:00:00 2001 From: matevip Date: Wed, 17 Jun 2026 14:17:54 +0800 Subject: [PATCH] feat(wiki): entity-level knowledge graph extraction (#336) Add an opt-in named-entity extraction pass so the wiki knowledge graph captures fine-grained entities (people, organizations, locations, ...) and their relations, not just page-level link relations. - new tables mate_wiki_entity / _mention / _relation (h2/mysql/kingbase) - structured LLM extraction per chunk with entity resolution (normalized-key dedup + embedding near-merge), mention/relation persistence and page linking via chunk citations - per-KB opt-in toggle (off by default); async dispatch after embedding - read API: entity list, KB graph, entity ego-graph, manual extract - UI: entity-layer toggle in the graph view + KB config toggle - replace inline fully-qualified class names with imports in WikiProcessingService Closes #336 --- .../wiki/controller/WikiEntityController.java | 76 +++ .../mate/wiki/dto/EntityExtractionResult.java | 49 ++ .../mate/wiki/dto/WikiEntityGraphView.java | 40 ++ .../vip/mate/wiki/dto/WikiEntityView.java | 24 + .../java/vip/mate/wiki/job/WikiJobStep.java | 5 +- .../java/vip/mate/wiki/job/WikiKbConfig.java | 17 + .../vip/mate/wiki/model/WikiEntityEntity.java | 72 +++ .../wiki/model/WikiEntityMentionEntity.java | 64 +++ .../wiki/model/WikiEntityRelationEntity.java | 64 +++ .../wiki/repository/WikiEntityMapper.java | 17 + .../repository/WikiEntityMentionMapper.java | 17 + .../repository/WikiEntityRelationMapper.java | 19 + .../service/WikiEntityExtractionService.java | 457 ++++++++++++++++++ .../wiki/service/WikiEntityGraphService.java | 177 +++++++ .../wiki/service/WikiProcessingService.java | 235 +++++---- .../db/migration/h2/V148__wiki_entity.sql | 56 +++ .../h2/V149__wiki_entity_mention.sql | 38 ++ .../h2/V150__wiki_entity_relation.sql | 42 ++ .../migration/kingbase/V148__wiki_entity.sql | 31 ++ .../kingbase/V149__wiki_entity_mention.sql | 25 + .../kingbase/V150__wiki_entity_relation.sql | 27 ++ .../db/migration/mysql/V148__wiki_entity.sql | 32 ++ .../mysql/V149__wiki_entity_mention.sql | 27 ++ .../mysql/V150__wiki_entity_relation.sql | 28 ++ .../WikiEntityExtractionServiceTest.java | 179 +++++++ .../service/WikiProcessingFallbackTest.java | 3 +- .../WikiProcessingServiceLazyTest.java | 3 +- mateclaw-ui/src/api/index.ts | 10 + mateclaw-ui/src/i18n/locales/en-US.ts | 14 + mateclaw-ui/src/i18n/locales/zh-CN.ts | 14 + .../src/views/Wiki/components/WikiConfig.vue | 71 +++ .../Wiki/components/WikiEntityGraphView.vue | 264 ++++++++++ .../Wiki/components/WikiGraphToolbar.vue | 27 +- .../views/Wiki/components/WikiGraphView.vue | 42 +- 34 files changed, 2164 insertions(+), 102 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiEntityController.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/dto/EntityExtractionResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityGraphView.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityView.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityEntity.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityMentionEntity.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityRelationEntity.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMapper.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMentionMapper.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityRelationMapper.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityExtractionService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityGraphService.java create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V148__wiki_entity.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V149__wiki_entity_mention.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V150__wiki_entity_relation.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/kingbase/V148__wiki_entity.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/kingbase/V149__wiki_entity_mention.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/kingbase/V150__wiki_entity_relation.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V148__wiki_entity.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V149__wiki_entity_mention.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V150__wiki_entity_relation.sql create mode 100644 mateclaw-server/src/test/java/vip/mate/wiki/service/WikiEntityExtractionServiceTest.java create mode 100644 mateclaw-ui/src/views/Wiki/components/WikiEntityGraphView.vue diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiEntityController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiEntityController.java new file mode 100644 index 00000000..fcac1042 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiEntityController.java @@ -0,0 +1,76 @@ +package vip.mate.wiki.controller; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import vip.mate.wiki.dto.WikiEntityGraphView; +import vip.mate.wiki.dto.WikiEntityView; +import vip.mate.wiki.service.WikiEntityExtractionService; +import vip.mate.wiki.service.WikiEntityGraphService; +import vip.mate.wiki.service.WikiProcessingService; + +import java.util.List; +import java.util.Map; + +/** + * Read and manual-trigger endpoints for the entity-level knowledge graph. + * + * @author MateClaw Team + */ +@Slf4j +@RestController +@RequestMapping("/api/v1/wiki") +@RequiredArgsConstructor +public class WikiEntityController { + + private final WikiEntityGraphService graphService; + private final WikiEntityExtractionService extractionService; + + /** List entities in a KB, optionally filtered by type, ranked by salience. */ + @GetMapping("/kb/{kbId}/entities") + public List listEntities(@PathVariable Long kbId, + @RequestParam(required = false) String type, + @RequestParam(defaultValue = "100") int limit) { + return graphService.listEntities(kbId, type, limit); + } + + /** Whole-KB entity graph: top entities by salience plus the edges among them. */ + @GetMapping("/kb/{kbId}/entity-graph") + public WikiEntityGraphView kbEntityGraph(@PathVariable Long kbId, + @RequestParam(defaultValue = "150") int limit) { + return graphService.graph(kbId, limit); + } + + /** Ego-graph around a single entity: neighbors, edges, and mentioning pages. */ + @GetMapping("/kb/{kbId}/entities/{entityId}/graph") + public WikiEntityGraphView entityGraph(@PathVariable Long kbId, + @PathVariable Long entityId, + @RequestParam(defaultValue = "50") int limit) { + return graphService.ego(kbId, entityId, limit); + } + + /** + * Manually trigger an entity-extraction pass for a KB. Runs on the wiki + * executor so the request returns immediately. + * + * @param force when true, re-extract chunks that already have mentions + */ + @PostMapping("/kb/{kbId}/entities/extract") + public Map extract(@PathVariable Long kbId, + @RequestParam(defaultValue = "false") boolean force) { + WikiProcessingService.WIKI_EXECUTOR.submit(() -> { + try { + int count = extractionService.extractForKb(kbId, force); + log.info("[WikiEntity] Manual extraction completed: kbId={}, entities={}", kbId, count); + } catch (Exception e) { + log.warn("[WikiEntity] Manual extraction failed for kbId={}: {}", kbId, e.getMessage()); + } + }); + return Map.of("status", "started", "kbId", kbId); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/EntityExtractionResult.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/EntityExtractionResult.java new file mode 100644 index 00000000..4a5f1794 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/EntityExtractionResult.java @@ -0,0 +1,49 @@ +package vip.mate.wiki.dto; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +/** + * Structured output schema for a single entity-extraction LLM call over one + * source chunk. Bound via {@code BeanOutputConverter} and tolerant of a model + * returning either field empty. + * + * @author MateClaw Team + */ +@Data +public class EntityExtractionResult { + + /** Named entities found in the chunk. */ + private List entities = new ArrayList<>(); + + /** Subject → predicate → object triples between the extracted entities. */ + private List relations = new ArrayList<>(); + + @Data + public static class ExtractedEntity { + /** Canonical surface form of the entity as it appears in the text. */ + private String name; + /** One of the requested types: person | organization | location | event | product | concept | other. */ + private String type; + /** Alternate names / spellings for the same entity, if any. */ + private List aliases = new ArrayList<>(); + /** One-line description grounded in the chunk. */ + private String description; + /** Short verbatim quote evidencing the entity. */ + private String evidence; + } + + @Data + public static class ExtractedRelation { + /** Subject entity name (should match an entry in {@link #entities}). */ + private String subject; + /** Relation label, e.g. "works_for", "located_in", "founded". */ + private String predicate; + /** Object entity name (should match an entry in {@link #entities}). */ + private String object; + /** Short verbatim quote evidencing the relation. */ + private String evidence; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityGraphView.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityGraphView.java new file mode 100644 index 00000000..69b1cfb8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityGraphView.java @@ -0,0 +1,40 @@ +package vip.mate.wiki.dto; + +import lombok.Data; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +/** + * Ego-graph around one entity: the center node, its neighbor entity nodes, + * the relation edges connecting them, and the wiki pages that mention the + * center entity (the bridge from the entity layer to the page layer). + * + * @author MateClaw Team + */ +@Data +public class WikiEntityGraphView { + + private WikiEntityView center; + private List nodes = new ArrayList<>(); + private List edges = new ArrayList<>(); + private List pages = new ArrayList<>(); + + @Data + public static class Edge { + private Long id; + private Long subjectEntityId; + private String predicate; + private Long objectEntityId; + private String evidence; + private BigDecimal confidence; + } + + @Data + public static class PageRef { + private Long pageId; + private String slug; + private String title; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityView.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityView.java new file mode 100644 index 00000000..1ab8db68 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiEntityView.java @@ -0,0 +1,24 @@ +package vip.mate.wiki.dto; + +import lombok.Data; + +import java.math.BigDecimal; +import java.util.List; + +/** + * API-facing projection of a canonical entity node. Excludes the raw + * embedding vector and other internal columns. + * + * @author MateClaw Team + */ +@Data +public class WikiEntityView { + private Long id; + private Long kbId; + private String canonicalName; + private String type; + private List aliases; + private String description; + private BigDecimal salience; + private Integer mentionCount; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java index 707a8a8d..eb78af8c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java @@ -1,9 +1,8 @@ package vip.mate.wiki.job; /** - * RFC-030: Logical steps within a wiki processing job, - * used for per-step model routing. + * Logical steps within a wiki processing job, used for per-step model routing. */ public enum WikiJobStep { - ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY + ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY, ENTITY_EXTRACTION } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java index e1e1a489..5230de2b 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java @@ -57,4 +57,21 @@ public class WikiKbConfig { * pageType to be granted explicitly per agent. */ private String defaultReadPolicy; + + /** + * Opt-in for entity-level knowledge graph extraction on this KB. When + * {@code true}, an extraction pass runs after ingest/embedding to pull + * named entities (person, organization, location, ...) and their + * relations from source chunks into the {@code mate_wiki_entity*} tables. + * {@code null} or {@code false} keeps the legacy behaviour (page graph + * only). Off by default because extraction adds LLM calls per chunk. + */ + private Boolean entityExtractionEnabled; + + /** + * Optional whitelist of entity types to extract, e.g. + * {@code ["person","organization","location"]}. {@code null} or empty + * lets the extractor use its built-in default type set. + */ + private List entityTypes; } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityEntity.java new file mode 100644 index 00000000..51a148a5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityEntity.java @@ -0,0 +1,72 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +/** + * Canonical named-entity node extracted and de-duplicated from source chunks. + * + *

Mention-granularity counterpart to {@link WikiPageEntity} (which models + * document/topic granularity). Entities link to their source occurrences via + * {@link WikiEntityMentionEntity} and to one another via + * {@link WikiEntityRelationEntity}, forming an entity-level knowledge graph + * beneath the page graph cached in {@link WikiRelationEntity}. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_wiki_entity") +public class WikiEntityEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long kbId; + + /** Display name chosen for the merged entity. */ + private String canonicalName; + + /** Case/whitespace-folded key used for exact-match de-duplication. */ + private String normalizedKey; + + /** Entity taxonomy: person | organization | location | event | product | concept | other. */ + private String type; + + /** JSON array of surface forms merged into this entity. */ + private String aliasesJson; + + /** One-line summary synthesized from the mentions. */ + private String description; + + /** 0..1 importance score derived from mention frequency / distribution. */ + private BigDecimal salience; + + /** Number of mentions resolved to this entity. */ + private Integer mentionCount; + + /** Float32 little-endian name/description vector used for near-duplicate merge. */ + private byte[] embedding; + + /** Model name that produced {@link #embedding}; used for re-embed detection. */ + private String embeddingModel; + + /** Fingerprint of the inputs that produced this row; used for cache invalidation. */ + private String computedHash; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityMentionEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityMentionEntity.java new file mode 100644 index 00000000..e710d26f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityMentionEntity.java @@ -0,0 +1,64 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +/** + * Links a canonical {@link WikiEntityEntity} to a single source occurrence. + * + *

One row per (entity, chunk) occurrence. {@link #pageId} is back-filled + * from the chunk's citing pages so the entity layer connects to the page + * layer: entity → mention → chunk → citing page. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_wiki_entity_mention") +public class WikiEntityMentionEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long kbId; + + /** The resolved canonical entity. */ + private Long entityId; + + /** Source chunk the mention was found in. */ + private Long chunkId; + + /** A wiki page that cites {@link #chunkId}, when known; null otherwise. */ + private Long pageId; + + /** The exact text as it appeared in the source. */ + private String surfaceForm; + + /** Character offset of the mention within the chunk, when known. */ + private Integer charOffset; + + /** 0..1 extraction confidence. */ + private BigDecimal confidence; + + /** Short surrounding quote (≤ 500 chars enforced in Java layer). */ + private String evidence; + + /** Provenance tag: llm-extracted | manual. */ + private String source; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityRelationEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityRelationEntity.java new file mode 100644 index 00000000..84666ea3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiEntityRelationEntity.java @@ -0,0 +1,64 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +/** + * Directed subject → predicate → object triple between two canonical entities; + * the edges of the entity-level knowledge graph. + * + *

Distinct from {@link WikiRelationEntity}, which scores page-to-page edges. + * A row here is one fact triple connecting two {@link WikiEntityEntity} nodes. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_wiki_entity_relation") +public class WikiEntityRelationEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long kbId; + + /** Head entity. */ + private Long subjectEntityId; + + /** Free-text relation label, e.g. {@code works_for}, {@code located_in}. */ + private String predicate; + + /** Tail entity. */ + private Long objectEntityId; + + /** Short justification quote (≤ 500 chars enforced in Java layer). */ + private String evidence; + + /** 0..1 extraction confidence. */ + private BigDecimal confidence; + + /** Provenance tag: llm-extracted | inferred | manual. */ + private String source; + + /** Source chunk the triple was extracted from, when known. */ + private Long evidenceChunkId; + + /** Fingerprint of the inputs that produced this row; used for cache invalidation. */ + private String computedHash; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMapper.java new file mode 100644 index 00000000..2eb0392d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMapper.java @@ -0,0 +1,17 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiEntityEntity; + +/** + * Mapper for canonical named-entity nodes. + * + *

Write paths upsert keyed by ({@code kb_id}, {@code normalized_key}, + * {@code type}); read paths list by {@code kb_id} ordered by {@code salience}. + * + * @author MateClaw Team + */ +@Mapper +public interface WikiEntityMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMentionMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMentionMapper.java new file mode 100644 index 00000000..97ad180f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityMentionMapper.java @@ -0,0 +1,17 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiEntityMentionEntity; + +/** + * Mapper for entity-to-source occurrence links. + * + *

Read paths fetch mentions by {@code entity_id} or by {@code page_id}; + * cache invalidation soft-deletes by {@code chunk_id} or {@code kb_id}. + * + * @author MateClaw Team + */ +@Mapper +public interface WikiEntityMentionMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityRelationMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityRelationMapper.java new file mode 100644 index 00000000..137ce63a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiEntityRelationMapper.java @@ -0,0 +1,19 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiEntityRelationEntity; + +/** + * Mapper for entity-to-entity fact triples. + * + *

Read paths traverse the ego-graph by {@code subject_entity_id} / + * {@code object_entity_id}; write paths upsert keyed by the triple + * ({@code kb_id}, {@code subject_entity_id}, {@code predicate}, + * {@code object_entity_id}). + * + * @author MateClaw Team + */ +@Mapper +public interface WikiEntityRelationMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityExtractionService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityExtractionService.java new file mode 100644 index 00000000..9ead0e72 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityExtractionService.java @@ -0,0 +1,457 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.converter.BeanOutputConverter; +import org.springframework.stereotype.Service; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.dto.EntityExtractionResult; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.WikiKbConfig; +import vip.mate.wiki.job.WikiKbConfigParser; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.model.WikiChunkEntity; +import vip.mate.wiki.model.WikiEntityEntity; +import vip.mate.wiki.model.WikiEntityMentionEntity; +import vip.mate.wiki.model.WikiEntityRelationEntity; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.repository.WikiEntityMapper; +import vip.mate.wiki.repository.WikiEntityMentionMapper; +import vip.mate.wiki.repository.WikiEntityRelationMapper; +import vip.mate.wiki.repository.WikiPageCitationMapper; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Extracts a named-entity knowledge graph from source chunks: it pulls + * entities (person, organization, location, ...) and subject→predicate→object + * relations out of each chunk via a structured LLM call, resolves entities to + * canonical nodes (exact-key dedup plus optional embedding near-merge), and + * persists nodes, mentions, and edges into the {@code mate_wiki_entity*} + * tables. + * + *

This is an opt-in pass gated by {@link WikiKbConfig#getEntityExtractionEnabled()}; + * it runs after ingest/embedding and never blocks the page-generation pipeline. + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiEntityExtractionService { + + private static final List DEFAULT_ENTITY_TYPES = + List.of("person", "organization", "location", "event", "product", "concept"); + + /** Cosine threshold above which a new entity is merged into an existing same-type node. */ + private static final float MERGE_THRESHOLD = 0.92f; + + /** Max chunk characters sent to the model per call, to bound token spend. */ + private static final int MAX_CHUNK_CHARS = 6000; + + /** Evidence column is capped at 500 chars in the schema. */ + private static final int MAX_EVIDENCE = 500; + + private final WikiKnowledgeBaseService kbService; + private final WikiChunkService chunkService; + private final WikiEmbeddingService embeddingService; + private final WikiModelRoutingService routingService; + private final ModelConfigService modelConfigService; + private final ObjectMapper objectMapper; + + private final WikiEntityMapper entityMapper; + private final WikiEntityMentionMapper mentionMapper; + private final WikiEntityRelationMapper relationMapper; + private final WikiPageCitationMapper citationMapper; + + /** Extract entities from every not-yet-processed chunk of one raw material. */ + public int extractForRaw(Long kbId, Long rawId) { + return extract(kbId, chunkService.listByRawId(rawId), false); + } + + /** + * Extract entities across the whole KB. + * + * @param force when {@code true}, re-extract chunks that already have + * mentions (used for a manual full rebuild); otherwise skip + * chunks already processed + */ + public int extractForKb(Long kbId, boolean force) { + return extract(kbId, chunkService.listByKbId(kbId), force); + } + + private int extract(Long kbId, List chunks, boolean force) { + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + if (kb == null || chunks == null || chunks.isEmpty()) { + return 0; + } + ChatModel chatModel = resolveChatModel(kbId); + if (chatModel == null) { + log.warn("[WikiEntity] No chat model available for kbId={}, skipping extraction", kbId); + return 0; + } + + List types = resolveEntityTypes(kb); + BeanOutputConverter converter = + new BeanOutputConverter<>(EntityExtractionResult.class); + String systemPrompt = buildSystemPrompt(types); + + // Per-run resolution cache: type+normalizedKey → entityId. Seeded lazily + // from the DB so entities resolve consistently within and across chunks. + Map resolved = new HashMap<>(); + // Same-type existing entities with embeddings, for near-duplicate merge. + EntityIndex index = new EntityIndex(kbId); + + for (WikiChunkEntity chunk : chunks) { + if (chunk.getContent() == null || chunk.getContent().isBlank()) { + continue; + } + if (!force && hasMentions(chunk.getId())) { + continue; + } + try { + EntityExtractionResult result = callExtract(chatModel, converter, systemPrompt, chunk); + if (result == null) { + continue; + } + persistChunk(kbId, chunk, result, resolved, index); + } catch (Exception e) { + log.warn("[WikiEntity] Extraction failed for chunkId={} kbId={}: {}", + chunk.getId(), kbId, e.getMessage()); + } + } + return resolved.size(); + } + + // ---- LLM call --------------------------------------------------------- + + private EntityExtractionResult callExtract(ChatModel chatModel, + BeanOutputConverter converter, + String systemPrompt, + WikiChunkEntity chunk) { + String content = chunk.getContent(); + if (content.length() > MAX_CHUNK_CHARS) { + content = content.substring(0, MAX_CHUNK_CHARS); + } + String userPrompt = "Source text:\n\"\"\"\n" + content + "\n\"\"\"\n\n" + converter.getFormat(); + Prompt prompt = new Prompt(List.of( + new SystemMessage(systemPrompt), + new UserMessage(userPrompt))); + ChatResponse response = chatModel.call(prompt); + String text = response.getResult().getOutput().getText(); + if (text == null || text.isBlank()) { + return null; + } + try { + return converter.convert(text); + } catch (Exception e) { + log.debug("[WikiEntity] Structured parse failed, skipping chunk: {}", e.getMessage()); + return null; + } + } + + private String buildSystemPrompt(List types) { + return "You are a knowledge-graph entity extractor. From the given source text, " + + "extract named entities and the factual relations between them.\n" + + "Entity types to use: " + String.join(", ", types) + ".\n" + + "Rules:\n" + + "- Only extract entities explicitly named in the text; do not invent any.\n" + + "- Use the most complete surface form as the name; list shorter forms as aliases.\n" + + "- For each relation, subject and object must both appear in the entities list.\n" + + "- Keep predicates short and snake_case (e.g. works_for, located_in, founded).\n" + + "- Provide a short verbatim evidence quote for each entity and relation.\n" + + "- If nothing relevant is present, return empty lists."; + } + + // ---- persistence ------------------------------------------------------ + + private void persistChunk(Long kbId, WikiChunkEntity chunk, EntityExtractionResult result, + Map resolved, EntityIndex index) { + Long pageId = firstCitingPage(chunk.getId()); + + // Resolve each entity to a canonical id, persist its mention for this chunk. + Map localByName = new HashMap<>(); + if (result.getEntities() != null) { + for (EntityExtractionResult.ExtractedEntity e : result.getEntities()) { + if (e == null || e.getName() == null || e.getName().isBlank()) { + continue; + } + String type = normalizeType(e.getType()); + Long entityId = resolveEntity(kbId, e, type, resolved, index); + if (entityId == null) { + continue; + } + localByName.put(normalize(e.getName()), entityId); + if (e.getAliases() != null) { + for (String alias : e.getAliases()) { + if (alias != null && !alias.isBlank()) { + localByName.put(normalize(alias), entityId); + } + } + } + insertMention(kbId, entityId, chunk.getId(), pageId, e.getName(), e.getEvidence()); + bumpMentionCount(entityId); + } + } + + // Persist relations whose endpoints both resolved. + if (result.getRelations() != null) { + for (EntityExtractionResult.ExtractedRelation r : result.getRelations()) { + if (r == null || r.getSubject() == null || r.getObject() == null + || r.getPredicate() == null || r.getPredicate().isBlank()) { + continue; + } + Long subjectId = localByName.get(normalize(r.getSubject())); + Long objectId = localByName.get(normalize(r.getObject())); + if (subjectId == null || objectId == null || subjectId.equals(objectId)) { + continue; + } + upsertRelation(kbId, subjectId, objectId, normalizePredicate(r.getPredicate()), + r.getEvidence(), chunk.getId()); + } + } + } + + /** + * Resolve an extracted entity to a canonical node id: run cache → exact + * key match in DB → embedding near-match → create new. + */ + private Long resolveEntity(Long kbId, EntityExtractionResult.ExtractedEntity e, String type, + Map resolved, EntityIndex index) { + String key = normalize(e.getName()); + String cacheKey = type + "" + key; + Long cached = resolved.get(cacheKey); + if (cached != null) { + return cached; + } + + WikiEntityEntity existing = entityMapper.selectOne(new LambdaQueryWrapper() + .eq(WikiEntityEntity::getKbId, kbId) + .eq(WikiEntityEntity::getNormalizedKey, key) + .eq(WikiEntityEntity::getType, type) + .last("LIMIT 1")); + if (existing != null) { + resolved.put(cacheKey, existing.getId()); + return existing.getId(); + } + + // Embedding near-duplicate merge across spellings/languages. + float[] vec = embedName(kbId, e); + if (vec != null) { + Long near = index.findNearest(type, vec); + if (near != null) { + resolved.put(cacheKey, near); + return near; + } + } + + WikiEntityEntity created = new WikiEntityEntity(); + created.setKbId(kbId); + created.setCanonicalName(e.getName().trim()); + created.setNormalizedKey(key); + created.setType(type); + created.setAliasesJson(writeJson(e.getAliases())); + created.setDescription(truncate(e.getDescription(), MAX_EVIDENCE)); + created.setMentionCount(0); + created.setSalience(BigDecimal.ZERO); + if (vec != null) { + created.setEmbedding(WikiEmbeddingService.floatsToBytes(vec)); + } + entityMapper.insert(created); + resolved.put(cacheKey, created.getId()); + index.add(type, created.getId(), vec); + return created.getId(); + } + + private void insertMention(Long kbId, Long entityId, Long chunkId, Long pageId, + String surfaceForm, String evidence) { + WikiEntityMentionEntity m = new WikiEntityMentionEntity(); + m.setKbId(kbId); + m.setEntityId(entityId); + m.setChunkId(chunkId); + m.setPageId(pageId); + m.setSurfaceForm(truncate(surfaceForm, 256)); + m.setConfidence(BigDecimal.valueOf(0.9)); + m.setEvidence(truncate(evidence, MAX_EVIDENCE)); + m.setSource("llm-extracted"); + mentionMapper.insert(m); + } + + private void bumpMentionCount(Long entityId) { + WikiEntityEntity e = entityMapper.selectById(entityId); + if (e == null) { + return; + } + int count = (e.getMentionCount() == null ? 0 : e.getMentionCount()) + 1; + e.setMentionCount(count); + // Saturating popularity score in [0,1): count / (count + 5). + e.setSalience(BigDecimal.valueOf((double) count / (count + 5.0)) + .setScale(4, RoundingMode.HALF_UP)); + entityMapper.updateById(e); + } + + private void upsertRelation(Long kbId, Long subjectId, Long objectId, String predicate, + String evidence, Long chunkId) { + WikiEntityRelationEntity existing = relationMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiEntityRelationEntity::getKbId, kbId) + .eq(WikiEntityRelationEntity::getSubjectEntityId, subjectId) + .eq(WikiEntityRelationEntity::getPredicate, predicate) + .eq(WikiEntityRelationEntity::getObjectEntityId, objectId) + .last("LIMIT 1")); + if (existing != null) { + return; + } + WikiEntityRelationEntity rel = new WikiEntityRelationEntity(); + rel.setKbId(kbId); + rel.setSubjectEntityId(subjectId); + rel.setPredicate(predicate); + rel.setObjectEntityId(objectId); + rel.setEvidence(truncate(evidence, MAX_EVIDENCE)); + rel.setConfidence(BigDecimal.valueOf(0.8)); + rel.setSource("llm-extracted"); + rel.setEvidenceChunkId(chunkId); + relationMapper.insert(rel); + } + + // ---- helpers ---------------------------------------------------------- + + private boolean hasMentions(Long chunkId) { + Long count = mentionMapper.selectCount(new LambdaQueryWrapper() + .eq(WikiEntityMentionEntity::getChunkId, chunkId)); + return count != null && count > 0; + } + + private Long firstCitingPage(Long chunkId) { + List pages = citationMapper.listPageIdsByChunkId(chunkId); + return (pages == null || pages.isEmpty()) ? null : pages.get(0); + } + + private float[] embedName(Long kbId, EntityExtractionResult.ExtractedEntity e) { + try { + String text = e.getName() + (e.getDescription() == null ? "" : ". " + e.getDescription()); + return embeddingService.embedQuery(kbId, text); + } catch (Exception ex) { + return null; + } + } + + private ChatModel resolveChatModel(Long kbId) { + try { + Long modelId = routingService.selectModelId(kbId, "heavy_ingest", WikiJobStep.ENTITY_EXTRACTION); + if (modelId != null) { + return routingService.buildChatModel(modelId); + } + } catch (Exception e) { + log.warn("[WikiEntity] Model routing failed for kbId={}, using default: {}", kbId, e.getMessage()); + } + var def = modelConfigService.getDefaultModel(); + if (def == null) { + return null; + } + return routingService.buildChatModel(def.getId()); + } + + private List resolveEntityTypes(WikiKnowledgeBaseEntity kb) { + if (kb.getConfigContent() != null) { + WikiKbConfig config = WikiKbConfigParser.parse(objectMapper, kb.getConfigContent()); + if (config != null && config.getEntityTypes() != null && !config.getEntityTypes().isEmpty()) { + return config.getEntityTypes(); + } + } + return DEFAULT_ENTITY_TYPES; + } + + private String normalize(String s) { + return s == null ? "" : s.trim().toLowerCase(Locale.ROOT).replaceAll("\\s+", " "); + } + + private String normalizeType(String type) { + String t = type == null ? "" : type.trim().toLowerCase(Locale.ROOT); + return t.isEmpty() ? "other" : t; + } + + private String normalizePredicate(String predicate) { + String p = predicate.trim().toLowerCase(Locale.ROOT).replaceAll("\\s+", "_"); + return p.length() > 64 ? p.substring(0, 64) : p; + } + + private String truncate(String s, int max) { + if (s == null) { + return null; + } + String t = s.trim(); + return t.length() > max ? t.substring(0, max) : t; + } + + private String writeJson(Object value) { + if (value == null) { + return null; + } + try { + return objectMapper.writeValueAsString(value); + } catch (Exception e) { + return null; + } + } + + /** + * In-memory index of same-type entity embeddings for near-duplicate merge + * within a single extraction run. Bounded by the KB's existing entity count. + */ + private final class EntityIndex { + private final Map> vectorsByType = new HashMap<>(); + private final Map> idsByType = new HashMap<>(); + + EntityIndex(Long kbId) { + List existing = entityMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiEntityEntity::getKbId, kbId) + .isNotNull(WikiEntityEntity::getEmbedding)); + for (WikiEntityEntity e : existing) { + if (e.getEmbedding() != null) { + add(e.getType(), e.getId(), WikiEmbeddingService.bytesToFloats(e.getEmbedding())); + } + } + } + + void add(String type, Long id, float[] vec) { + if (vec == null) { + return; + } + vectorsByType.computeIfAbsent(type, k -> new ArrayList<>()).add(vec); + idsByType.computeIfAbsent(type, k -> new ArrayList<>()).add(id); + } + + Long findNearest(String type, float[] vec) { + List vectors = vectorsByType.get(type); + List ids = idsByType.get(type); + if (vectors == null || vectors.isEmpty()) { + return null; + } + float best = MERGE_THRESHOLD; + Long bestId = null; + for (int i = 0; i < vectors.size(); i++) { + float sim = WikiEmbeddingService.cosine(vec, vectors.get(i)); + if (sim >= best) { + best = sim; + bestId = ids.get(i); + } + } + return bestId; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityGraphService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityGraphService.java new file mode 100644 index 00000000..ec3e92ac --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEntityGraphService.java @@ -0,0 +1,177 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.dto.WikiEntityGraphView; +import vip.mate.wiki.dto.WikiEntityView; +import vip.mate.wiki.model.WikiEntityEntity; +import vip.mate.wiki.model.WikiEntityMentionEntity; +import vip.mate.wiki.model.WikiEntityRelationEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiEntityMapper; +import vip.mate.wiki.repository.WikiEntityMentionMapper; +import vip.mate.wiki.repository.WikiEntityRelationMapper; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Read-side queries over the entity-level knowledge graph: entity listing and + * single-entity ego-graph assembly for the wiki graph view. + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiEntityGraphService { + + private final WikiEntityMapper entityMapper; + private final WikiEntityMentionMapper mentionMapper; + private final WikiEntityRelationMapper relationMapper; + private final WikiPageService pageService; + private final ObjectMapper objectMapper; + + /** List entities in a KB, optionally filtered by type, ranked by salience. */ + public List listEntities(Long kbId, String type, int limit) { + LambdaQueryWrapper q = new LambdaQueryWrapper() + .eq(WikiEntityEntity::getKbId, kbId) + .orderByDesc(WikiEntityEntity::getSalience) + .last("LIMIT " + Math.max(1, Math.min(limit, 500))); + if (type != null && !type.isBlank()) { + q.eq(WikiEntityEntity::getType, type.trim().toLowerCase()); + } + List out = new ArrayList<>(); + for (WikiEntityEntity e : entityMapper.selectList(q)) { + out.add(toView(e)); + } + return out; + } + + /** + * Assemble the whole-KB entity graph: the top entities by salience plus the + * relation edges that connect any two of them. + */ + public WikiEntityGraphView graph(Long kbId, int limit) { + WikiEntityGraphView view = new WikiEntityGraphView(); + List nodes = listEntities(kbId, null, limit); + view.setNodes(nodes); + if (nodes.isEmpty()) { + return view; + } + Set ids = new LinkedHashSet<>(); + for (WikiEntityView n : nodes) { + ids.add(n.getId()); + } + List rels = relationMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiEntityRelationEntity::getKbId, kbId) + .last("LIMIT " + Math.max(1, Math.min(limit * 5, 2000)))); + for (WikiEntityRelationEntity r : rels) { + if (!ids.contains(r.getSubjectEntityId()) || !ids.contains(r.getObjectEntityId())) { + continue; + } + WikiEntityGraphView.Edge edge = new WikiEntityGraphView.Edge(); + edge.setId(r.getId()); + edge.setSubjectEntityId(r.getSubjectEntityId()); + edge.setPredicate(r.getPredicate()); + edge.setObjectEntityId(r.getObjectEntityId()); + edge.setEvidence(r.getEvidence()); + edge.setConfidence(r.getConfidence()); + view.getEdges().add(edge); + } + return view; + } + + /** Assemble the ego-graph around one entity. */ + public WikiEntityGraphView ego(Long kbId, Long entityId, int limit) { + WikiEntityGraphView view = new WikiEntityGraphView(); + WikiEntityEntity center = entityMapper.selectById(entityId); + if (center == null || !center.getKbId().equals(kbId)) { + return view; + } + view.setCenter(toView(center)); + + int edgeLimit = Math.max(1, Math.min(limit, 200)); + List edges = relationMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiEntityRelationEntity::getKbId, kbId) + .and(w -> w.eq(WikiEntityRelationEntity::getSubjectEntityId, entityId) + .or().eq(WikiEntityRelationEntity::getObjectEntityId, entityId)) + .last("LIMIT " + edgeLimit)); + + Set neighborIds = new LinkedHashSet<>(); + for (WikiEntityRelationEntity r : edges) { + WikiEntityGraphView.Edge edge = new WikiEntityGraphView.Edge(); + edge.setId(r.getId()); + edge.setSubjectEntityId(r.getSubjectEntityId()); + edge.setPredicate(r.getPredicate()); + edge.setObjectEntityId(r.getObjectEntityId()); + edge.setEvidence(r.getEvidence()); + edge.setConfidence(r.getConfidence()); + view.getEdges().add(edge); + neighborIds.add(r.getSubjectEntityId()); + neighborIds.add(r.getObjectEntityId()); + } + neighborIds.remove(entityId); + if (!neighborIds.isEmpty()) { + for (WikiEntityEntity n : entityMapper.selectBatchIds(neighborIds)) { + view.getNodes().add(toView(n)); + } + } + + // Pages that mention the center entity — the bridge to the page layer. + List mentions = mentionMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiEntityMentionEntity::getEntityId, entityId) + .isNotNull(WikiEntityMentionEntity::getPageId) + .last("LIMIT 200")); + Set pageIds = new LinkedHashSet<>(); + for (WikiEntityMentionEntity m : mentions) { + pageIds.add(m.getPageId()); + } + for (Long pageId : pageIds) { + WikiPageEntity page = pageService.getById(pageId); + if (page == null) { + continue; + } + WikiEntityGraphView.PageRef ref = new WikiEntityGraphView.PageRef(); + ref.setPageId(page.getId()); + ref.setSlug(page.getSlug()); + ref.setTitle(page.getTitle()); + view.getPages().add(ref); + } + return view; + } + + private WikiEntityView toView(WikiEntityEntity e) { + WikiEntityView v = new WikiEntityView(); + v.setId(e.getId()); + v.setKbId(e.getKbId()); + v.setCanonicalName(e.getCanonicalName()); + v.setType(e.getType()); + v.setDescription(e.getDescription()); + v.setSalience(e.getSalience()); + v.setMentionCount(e.getMentionCount()); + v.setAliases(parseAliases(e.getAliasesJson())); + return v; + } + + private List parseAliases(String json) { + if (json == null || json.isBlank()) { + return Collections.emptyList(); + } + try { + return objectMapper.readValue(json, new TypeReference>() {}); + } catch (Exception e) { + return Collections.emptyList(); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index e02f9282..dafc22a9 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -1,7 +1,9 @@ package vip.mate.wiki.service; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -10,24 +12,49 @@ import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.converter.BeanOutputConverter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Lazy; +import org.springframework.dao.DuplicateKeyException; import org.springframework.retry.support.RetryTemplate; import org.springframework.stereotype.Service; import vip.mate.agent.AgentGraphBuilder; import vip.mate.agent.prompt.PromptLoader; +import vip.mate.llm.failover.ProviderHealthTracker; import vip.mate.llm.model.ModelConfigEntity; import vip.mate.llm.service.ModelConfigService; +import vip.mate.llm.service.ModelProviderService; import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.RouteResult; +import vip.mate.wiki.dto.RoutedPageMeta; import vip.mate.wiki.dto.WikiChunkDraft; +import vip.mate.wiki.event.WikiFactPageUpdatedEvent; +import vip.mate.wiki.event.WikiKbDirtyEvent; +import vip.mate.wiki.event.WikiPageCreatedEvent; import vip.mate.wiki.event.WikiProcessingEvent; +import vip.mate.wiki.job.WikiEmbeddingProviderFailingException; +import vip.mate.wiki.job.WikiJobStage; +import vip.mate.wiki.job.WikiJobStep; import vip.mate.wiki.job.WikiKbConfig; import vip.mate.wiki.job.WikiKbConfigParser; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.job.WikiProcessingJobService; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.profile.WikiMetadataValidator; +import vip.mate.wiki.profile.WikiPageTypeDef; +import vip.mate.wiki.profile.WikiPageTypeProfile; +import vip.mate.wiki.profile.WikiPageTypeProfileService; import vip.mate.wiki.sse.WikiProgressBus; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -60,23 +87,24 @@ public class WikiProcessingService { private final ObjectMapper objectMapper; private final WikiProgressBus progressBus; private final WikiCitationService citationService; - private final org.springframework.context.ApplicationEventPublisher eventPublisher; + private final ApplicationEventPublisher eventPublisher; + private final WikiEntityExtractionService entityExtractionService; /** * Optional KB pageType profile. Field-injected (not a constructor arg) so * existing instantiations are unaffected; when absent the batch-create * prompt falls back to the legacy hardcoded pageType enum. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private vip.mate.wiki.profile.WikiPageTypeProfileService pageTypeProfileService; + @Autowired(required = false) + private WikiPageTypeProfileService pageTypeProfileService; /** Optional metadata validator, paired with {@link #pageTypeProfileService}. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private vip.mate.wiki.profile.WikiMetadataValidator metadataValidator; + @Autowired(required = false) + private WikiMetadataValidator metadataValidator; /** Optional dependency/stale engine for layered-knowledge wiring. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private vip.mate.wiki.service.WikiDependencyService dependencyService; + @Autowired(required = false) + private WikiDependencyService dependencyService; /** * Read-the-failover-chain handle. Optional so the existing constructors and @@ -84,8 +112,8 @@ public class WikiProcessingService { * fallback hop iterates {@code listEnabledModels} in DB order — same * behavior as before this PR. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private vip.mate.llm.service.ModelProviderService modelProviderService; + @Autowired(required = false) + private ModelProviderService modelProviderService; /** * Per-provider failure counter / cooldown bookkeeping. Optional for the @@ -94,12 +122,12 @@ public class WikiProcessingService { * successful call we clear the failure counter for the provider that * actually responded. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker; + @Autowired(required = false) + private ProviderHealthTracker providerHealthTracker; - @org.springframework.beans.factory.annotation.Autowired(required = false) - @org.springframework.context.annotation.Lazy - private vip.mate.wiki.job.WikiProcessingJobService wikiJobService; + @Autowired(required = false) + @Lazy + private WikiProcessingJobService wikiJobService; /** * RFC-051 PR-1c: optional preprocessor that fills chunk metadata @@ -107,7 +135,7 @@ public class WikiProcessingService { * Marked optional so unit tests that construct this service directly * (without Spring) can opt out without exploding. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) + @Autowired(required = false) private DocumentPreprocessService preprocessService; /** @@ -115,7 +143,7 @@ public class WikiProcessingService { * the KB before each ingest. Optional so the older lazy-only unit tests * don't need to wire it. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) + @Autowired(required = false) private WikiScaffoldService scaffoldService; /** @@ -124,7 +152,7 @@ public class WikiProcessingService { * without Spring don't need to supply it — when absent, the post-ingestion * auto-scan is simply skipped. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) + @Autowired(required = false) private WikiLintJobService lintJobService; /** @@ -133,14 +161,14 @@ public class WikiProcessingService { * routing chain (stepModels[step] -> wikiDefaultModelId -> system * default) for a model rather than always pulling the system default. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - private vip.mate.wiki.job.WikiModelRoutingService modelRoutingService; + @Autowired(required = false) + private WikiModelRoutingService modelRoutingService; /** RFC-051 PR-2b/2c: optional overview rebuilder + log appender. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) + @Autowired(required = false) private WikiOverviewService overviewService; - @org.springframework.beans.factory.annotation.Autowired(required = false) + @Autowired(required = false) private WikiLogService logService; /** @@ -148,8 +176,8 @@ public class WikiProcessingService { * of the KB's apply-default transformation templates. Missing in the * legacy unit tests that wire this service directly. */ - @org.springframework.beans.factory.annotation.Autowired(required = false) - @org.springframework.context.annotation.Lazy + @Autowired(required = false) + @Lazy private WikiTransformationExecutor transformationExecutor; /** Parallel chunk / material processing executor (JDK 21 virtual threads) */ @@ -203,7 +231,7 @@ public class WikiProcessingService { /** KBs with a reclassify pass currently running, used to reject concurrent * re-triggers (which would double LLM spend and race page-type writes). */ - private final java.util.Set reclassifyInFlight = ConcurrentHashMap.newKeySet(); + private final Set reclassifyInFlight = ConcurrentHashMap.newKeySet(); /** * Process one raw material. @@ -272,7 +300,7 @@ public class WikiProcessingService { try { var job = wikiJobService.createHeavyIngest(kb.getId(), rawId); jobId = job.getId(); - wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.ROUTING); + wikiJobService.transition(jobId, WikiJobStage.ROUTING); } catch (Exception e) { log.warn("[Wiki] Failed to create heavy ingest job record for raw={}: {}", rawId, e.getMessage()); } @@ -284,7 +312,7 @@ public class WikiProcessingService { // RFC-012 M3：广播 raw.started（前端切到 indeterminate 进度条） progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_STARTED, - java.util.Map.of("rawId", rawId, "phase", "route")); + Map.of("rawId", rawId, "phase", "route")); try { // Phase 1: 获取文本内容 @@ -323,7 +351,7 @@ public class WikiProcessingService { // Transition job to phase_a (chunk processing begins) if (wikiJobService != null && jobId != null) { - try { wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.PHASE_A_RUNNING); } catch (Exception ignored) {} + try { wikiJobService.transition(jobId, WikiJobStage.PHASE_A_RUNNING); } catch (Exception ignored) {} } // Phase 3: LLM 消化 @@ -416,10 +444,10 @@ public class WikiProcessingService { // RFC-012 M3：广播终态 if ("failed".equals(finalStatus)) { progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED, - java.util.Map.of("rawId", rawId, "error", finalDetail == null ? "" : finalDetail)); + Map.of("rawId", rawId, "error", finalDetail == null ? "" : finalDetail)); } else { progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED, - java.util.Map.of( + Map.of( "rawId", rawId, "status", finalStatus, "totalPages", totalPages, @@ -430,10 +458,10 @@ public class WikiProcessingService { if (wikiJobService != null && jobId != null) { try { var terminalStage = switch (finalStatus) { - case "failed" -> vip.mate.wiki.job.WikiJobStage.FAILED; - case "partial" -> vip.mate.wiki.job.WikiJobStage.PARTIAL; - case "cancelled" -> vip.mate.wiki.job.WikiJobStage.CANCELLED; - default -> vip.mate.wiki.job.WikiJobStage.COMPLETED; + case "failed" -> WikiJobStage.FAILED; + case "partial" -> WikiJobStage.PARTIAL; + case "cancelled" -> WikiJobStage.CANCELLED; + default -> WikiJobStage.COMPLETED; }; wikiJobService.transition(jobId, terminalStage); } catch (Exception ignored) {} @@ -460,7 +488,7 @@ public class WikiProcessingService { // schedule (debounced) an LLM-generated overview narrative refresh. // Stats rebuild above is sync; narrative regen runs after-commit. if (nonTerminalSideEffects) { - eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kb.getId())); + eventPublisher.publishEvent(new WikiKbDirtyEvent(this, kb.getId())); } // Run apply-default transformation templates against the newly @@ -495,7 +523,7 @@ public class WikiProcessingService { if (embedded > 0) { log.info("[Wiki] Async embedding completed: kbId={}, embedded={}", fKbId, embedded); } - } catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) { + } catch (WikiEmbeddingProviderFailingException ex) { // Circuit-breaker tripped — the provider has consistently failed. // The exception's own log line in WikiEmbeddingService is enough; // emit a calmer notice here instead of a generic failure log. @@ -507,6 +535,26 @@ public class WikiProcessingService { }); } + // Entity-level knowledge graph extraction — opt-in per KB. Runs as a + // separate async pass so it never blocks (or fails) the ingest pipeline; + // its inputs (chunks, citations) are already committed at this point. + if (totalChunks > 0 && !"cancelled".equals(finalStatus) + && isEntityExtractionEnabled(kb)) { + final Long fKbId = kb.getId(); + final Long fRawId = rawId; + WIKI_EXECUTOR.submit(() -> { + try { + int count = entityExtractionService.extractForRaw(fKbId, fRawId); + if (count > 0) { + log.info("[Wiki] Async entity extraction completed: kbId={}, rawId={}, entities={}", + fKbId, fRawId, count); + } + } catch (Exception ex) { + log.warn("[Wiki] Async entity extraction failed for kbId={}: {}", fKbId, ex.getMessage()); + } + }); + } + } catch (Exception e) { // If the user requested cancellation while this run was in flight, // surface the abort as 'cancelled' rather than 'failed' even when @@ -527,8 +575,8 @@ public class WikiProcessingService { if (wikiJobService != null && jobId != null) { try { wikiJobService.transition(jobId, cancelled - ? vip.mate.wiki.job.WikiJobStage.CANCELLED - : vip.mate.wiki.job.WikiJobStage.FAILED); + ? WikiJobStage.CANCELLED + : WikiJobStage.FAILED); } catch (Exception ignored) {} } // Broadcast: cancelled rows reuse the COMPLETED event with status="cancelled" @@ -536,10 +584,10 @@ public class WikiProcessingService { // go through RAW_FAILED (which the UI surfaces as a red banner). if (cancelled) { progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED, - java.util.Map.of("rawId", rawId, "status", "cancelled")); + Map.of("rawId", rawId, "status", "cancelled")); } else { progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED, - java.util.Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage())); + Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage())); } } finally { // RFC-012 M2 v2 UI v2：写入最终进度并清理共享计数器 @@ -665,7 +713,7 @@ public class WikiProcessingService { if (modelId != null && modelRoutingService != null) { chatModel = modelRoutingService.buildChatModel(modelId); } else { - chatModel = resolveChatModel(kbId, vip.mate.wiki.job.WikiJobStep.CREATE_PAGE).chatModel; + chatModel = resolveChatModel(kbId, WikiJobStep.CREATE_PAGE).chatModel; } systemPrompt = PromptLoader.loadPrompt("wiki/classify-page-system") .replace("{allowed_page_types}", pageTypeProfileService.describeForPrompt(kbId)); @@ -714,13 +762,13 @@ public class WikiProcessingService { page.getId(), kbId, e.getMessage()); } finally { progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE, - java.util.Map.of("kind", "reclassify", "done", done, "total", total)); + Map.of("kind", "reclassify", "done", done, "total", total)); } } log.info("[Wiki] reclassifyKB done kbId={} pages={} changed={} failed={}", kbId, total, changed, failed); progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED, - java.util.Map.of("kind", "reclassify", "done", total, "total", total, + Map.of("kind", "reclassify", "done", total, "total", total, "changed", changed, "failed", failed)); } finally { reclassifyInFlight.remove(kbId); @@ -950,7 +998,7 @@ public class WikiProcessingService { )); if (isAborted(raw.getId(), "single-chunk legacy")) return 0; String llmResponse = callLlmWithResilientRetry(prompt, "chunk of raw=" + raw.getId(), - kb.getId(), vip.mate.wiki.job.WikiJobStep.CREATE_PAGE); + kb.getId(), WikiJobStep.CREATE_PAGE); return applyLlmResponse(kb.getId(), raw.getId(), llmResponse); } @@ -1004,9 +1052,9 @@ public class WikiProcessingService { // LLM produces strict RouteResult JSON. KB config wins; falls back to global // mate.wiki.use-structured-route default when the KB hasn't expressed a preference. boolean useStructured = resolveStructuredRouteFlag(kb); - org.springframework.ai.converter.BeanOutputConverter routeConverter = + BeanOutputConverter routeConverter = useStructured - ? new org.springframework.ai.converter.BeanOutputConverter<>(vip.mate.wiki.dto.RouteResult.class) + ? new BeanOutputConverter<>(RouteResult.class) : null; if (routeConverter != null) { routeUser = routeUser + "\n\n" + routeConverter.getFormat(); @@ -1018,7 +1066,7 @@ public class WikiProcessingService { )); if (isAborted(rawId, "route phase")) return 0; String routeResponse = callLlmWithResilientRetry(routePrompt, "route chunk of raw=" + rawId, - kbId, vip.mate.wiki.job.WikiJobStep.ROUTE); + kbId, WikiJobStep.ROUTE); // RFC-012 follow-up #3：phase B 现在并行执行，计数必须是 atomic AtomicInteger created = new AtomicInteger(0); @@ -1031,12 +1079,12 @@ public class WikiProcessingService { boolean structuredOk = false; if (routeConverter != null) { try { - vip.mate.wiki.dto.RouteResult bound = routeConverter.convert(routeResponse); + RouteResult bound = routeConverter.convert(routeResponse); if (bound != null) { - for (vip.mate.wiki.dto.RoutedPageMeta meta : bound.create()) { + for (RoutedPageMeta meta : bound.create()) { if (meta == null || meta.slug() == null || meta.slug().isBlank() || meta.title() == null || meta.title().isBlank()) continue; - com.fasterxml.jackson.databind.node.ObjectNode node = objectMapper.createObjectNode(); + ObjectNode node = objectMapper.createObjectNode(); node.put("slug", meta.slug()); node.put("title", meta.title()); if (meta.summary() != null) node.put("summary", meta.summary()); @@ -1079,7 +1127,7 @@ public class WikiProcessingService { )); String retryResponse = callLlmWithResilientRetry(retryPrompt, "route chunk RETRY of raw=" + rawId, - kbId, vip.mate.wiki.job.WikiJobStep.ROUTE); + kbId, WikiJobStep.ROUTE); routeJson = parseJsonResponse(retryResponse); if (routeJson == null) { log.warn("[Wiki] Route phase: failed to parse JSON for kbId={}, rawId={}, responseLen={}, first200={}", @@ -1112,7 +1160,7 @@ public class WikiProcessingService { // so no content is silently dropped (mirrors llm_wiki source-summary guarantee). if (totalPlanned == 0 && textContent.length() >= properties.getChunkFallbackMinChars()) { String overviewSlug = WikiPageService.toSlug(rawTitle) + "-overview"; - com.fasterxml.jackson.databind.node.ObjectNode fallbackMeta = + ObjectNode fallbackMeta = objectMapper.createObjectNode(); fallbackMeta.put("slug", overviewSlug); fallbackMeta.put("title", rawTitle + " 概述"); @@ -1133,7 +1181,7 @@ public class WikiProcessingService { log.info("[Wiki] Progress: switching to phase-b for raw={}", rawId); // RFC-012 M3：route 完成、phase-b 启动 → 通知前端确定进度（可显示 0/N） progressBus.broadcast(kbId, WikiProgressBus.EVENT_ROUTE_DONE, - java.util.Map.of( + Map.of( "rawId", rawId, "phase", "phase-b", "done", pc.done.get(), @@ -1195,7 +1243,7 @@ public class WikiProcessingService { if (!ok) pc.failed.incrementAndGet(); rawService.updateProgress(rawId, "phase-b", d, pc.total.get()); progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE, - java.util.Map.of( + Map.of( "rawId", rawId, "kind", "merge", "ok", ok, @@ -1288,7 +1336,7 @@ public class WikiProcessingService { String batchResponse = callLlmWithResilientRetry(batchPrompt, "batch-create " + subBatch.size() + " pages of raw=" + rawId + " subBatch=" + (bStart / batchSize + 1), - kbId, vip.mate.wiki.job.WikiJobStep.CREATE_PAGE); + kbId, WikiJobStep.CREATE_PAGE); List parsedPages = batchParser.parse(batchResponse); @@ -1328,7 +1376,7 @@ public class WikiProcessingService { pc.failed.incrementAndGet(); rawService.updateProgress(rawId, "phase-b", d, pc.total.get()); progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE, - java.util.Map.of("rawId", rawId, "kind", "create", + Map.of("rawId", rawId, "kind", "create", "ok", false, "done", d, "total", pc.total.get())); } continue; @@ -1377,7 +1425,7 @@ public class WikiProcessingService { pc.failed.incrementAndGet(); rawService.updateProgress(rawId, "phase-b", d, pc.total.get()); progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE, - java.util.Map.of("rawId", rawId, "kind", "create-retry", + Map.of("rawId", rawId, "kind", "create-retry", "ok", false, "done", d, "total", pc.total.get())); } continue; @@ -1422,7 +1470,7 @@ public class WikiProcessingService { if (!ok) pc.failed.incrementAndGet(); rawService.updateProgress(rawId, "phase-b", d, pc.total.get()); progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE, - java.util.Map.of( + Map.of( "rawId", rawId, "kind", "create", "ok", ok, @@ -1433,7 +1481,7 @@ public class WikiProcessingService { // Retry any pages that LLM omitted from the batch response int subBatchNum = bStart / batchSize + 1; - java.util.Set returnedSlugs = new java.util.HashSet<>(); + Set returnedSlugs = new HashSet<>(); for (WikiBatchCreateParser.ParsedPage pp : parsedPages) { returnedSlugs.add(pp.slug()); } @@ -1481,7 +1529,7 @@ public class WikiProcessingService { )); if (isAborted(raw.getId(), "retry-create slug=" + slug)) return null; return callLlmWithResilientRetry(prompt, "retry-create slug=" + slug + " of raw=" + raw.getId(), - kb.getId(), vip.mate.wiki.job.WikiJobStep.CREATE_PAGE); + kb.getId(), WikiJobStep.CREATE_PAGE); } /** @@ -1572,7 +1620,7 @@ public class WikiProcessingService { if (delta < 0) pc.failed.incrementAndGet(); rawService.updateProgress(rawId, "phase-b", d, pc.total.get()); progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE, - java.util.Map.of("rawId", rawId, "kind", "create-retry", + Map.of("rawId", rawId, "kind", "create-retry", "ok", delta >= 0, "done", d, "total", pc.total.get())); } return delta; @@ -1723,7 +1771,7 @@ public class WikiProcessingService { log.info("[Wiki] Phase B create page slug='{}' done (created)", slug); citationService.buildCitationsAsync(created.getId(), kbId); return true; - } catch (org.springframework.dao.DuplicateKeyException e) { + } catch (DuplicateKeyException e) { // Fallback 2: concurrent INSERT race — degrade to update pageService.updatePageByAi(kbId, slug, content, pageSummary, rawId); WikiPageEntity raced = pageService.getBySlug(kbId, slug); @@ -1756,13 +1804,13 @@ public class WikiProcessingService { // transactions), so the count is accurate. Idempotent + dedup-guarded // downstream, so firing on update paths is safe. if (eventPublisher != null && pageType != null && !pageType.isBlank()) { - eventPublisher.publishEvent(new vip.mate.wiki.event.WikiPageCreatedEvent(kbId, pageType, pageId)); + eventPublisher.publishEvent(new WikiPageCreatedEvent(kbId, pageType, pageId)); } // When an existing fact page is updated, propagate staleness to the // experience pages depending on it (async, off the ingest thread). if (isUpdate && eventPublisher != null && dependencyService != null && pageTypeProfileService != null && !pageTypeProfileService.isExperience(kbId, pageType)) { - eventPublisher.publishEvent(new vip.mate.wiki.event.WikiFactPageUpdatedEvent( + eventPublisher.publishEvent(new WikiFactPageUpdatedEvent( kbId, pageId, "fact page updated during ingest")); } } @@ -1792,7 +1840,7 @@ public class WikiProcessingService { if (!pageTypeProfileService.isExperience(kbId, pageType)) { return; // only experience pages declare fact dependencies } - java.util.List depIds = new java.util.ArrayList<>(); + List depIds = new ArrayList<>(); for (JsonNode n : dependsOnNode) { String slug = n.asText(""); if (slug.isBlank()) continue; @@ -1802,7 +1850,7 @@ public class WikiProcessingService { } } try { - java.util.List rejected = dependencyService.setDependencies(kbId, pageId, depIds); + List rejected = dependencyService.setDependencies(kbId, pageId, depIds); if (!rejected.isEmpty()) { log.warn("[Wiki] page {} dependency warnings: {}", pageId, rejected); } @@ -1853,11 +1901,11 @@ public class WikiProcessingService { return; } try { - vip.mate.wiki.profile.WikiPageTypeProfile profile = pageTypeProfileService.resolveProfile(kbId); - vip.mate.wiki.profile.WikiPageTypeDef def = profile.get(pageType); + WikiPageTypeProfile profile = pageTypeProfileService.resolveProfile(kbId); + WikiPageTypeDef def = profile.get(pageType); @SuppressWarnings("unchecked") - java.util.Map raw = objectMapper.convertValue(metadataNode, java.util.Map.class); - vip.mate.wiki.profile.WikiMetadataValidator.ValidationResult result = + Map raw = objectMapper.convertValue(metadataNode, Map.class); + WikiMetadataValidator.ValidationResult result = metadataValidator.validate(def, raw, profile.isAllowAdditionalFields(), "create"); String metadataJson = objectMapper.writeValueAsString(result.getCleaned()); String validationJson = result.getWarnings().isEmpty() @@ -1925,7 +1973,7 @@ public class WikiProcessingService { if (isAborted(rawId, "merge slug=" + slug)) return false; String response = callLlmWithResilientRetry(prompt, "merge page slug=" + slug + " of raw=" + rawId, - kbId, vip.mate.wiki.job.WikiJobStep.MERGE_PAGE); + kbId, WikiJobStep.MERGE_PAGE); JsonNode mergeJson = parseJsonResponse(response); if (mergeJson == null) { log.warn("[Wiki] Phase B merge page slug='{}' returned unparseable JSON, skipping", slug); @@ -2053,7 +2101,7 @@ public class WikiProcessingService { try { if (isAborted(raw.getId(), "doc analysis")) return ""; String response = callLlmWithResilientRetry(prompt, "analyze doc raw=" + raw.getId(), - kb.getId(), vip.mate.wiki.job.WikiJobStep.ROUTE); + kb.getId(), WikiJobStep.ROUTE); JsonNode json = parseJsonResponse(response); if (json != null) { // Validate related_pages against the active KB slug set BEFORE @@ -2120,7 +2168,7 @@ public class WikiProcessingService { JsonNode relatedNode = analysisJson.path("related_pages"); if (!relatedNode.isArray() || relatedNode.size() == 0) return analysisJson; - java.util.Set activeSlugs; + Set activeSlugs; try { activeSlugs = linkService.lowercaseSlugSet(pageService.listSummaries(kbId)); } catch (RuntimeException e) { @@ -2134,12 +2182,12 @@ public class WikiProcessingService { return result; } - com.fasterxml.jackson.databind.node.ArrayNode keptArray = objectMapper.createArrayNode(); - java.util.List dropped = new java.util.ArrayList<>(); + ArrayNode keptArray = objectMapper.createArrayNode(); + List dropped = new ArrayList<>(); for (JsonNode el : relatedNode) { String slug = el.asText("").trim(); if (slug.isEmpty()) continue; - if (activeSlugs.contains(slug.toLowerCase(java.util.Locale.ROOT))) { + if (activeSlugs.contains(slug.toLowerCase(Locale.ROOT))) { keptArray.add(slug); } else { dropped.add(slug); @@ -2257,7 +2305,7 @@ public class WikiProcessingService { * is available. Falls back to the system default on any lookup failure * so a misconfigured KB never blocks ingest. */ - private ChatModel buildChatModelFor(Long kbId, vip.mate.wiki.job.WikiJobStep step) { + private ChatModel buildChatModelFor(Long kbId, WikiJobStep step) { if (modelRoutingService != null && kbId != null && step != null) { try { Long modelId = modelRoutingService.selectModelId(kbId, "heavy_ingest", step); @@ -2295,7 +2343,7 @@ public class WikiProcessingService { * pick the routed chat model; passing {@code null} for either reproduces * the legacy behavior (system default model). */ - private String callLlmWithResilientRetry(Prompt prompt, String ctx, Long kbId, vip.mate.wiki.job.WikiJobStep step) { + private String callLlmWithResilientRetry(Prompt prompt, String ctx, Long kbId, WikiJobStep step) { long backoffMs = 1000; final long maxBackoffMs = 60_000; final int maxAttempts = Math.max(1, properties.getLlmMaxAttempts()); @@ -2434,7 +2482,7 @@ public class WikiProcessingService { /** Pair of modelId + built ChatModel — null modelId means we used the system default. */ private record ResolvedChatModel(Long modelId, ChatModel chatModel) {} - private ResolvedChatModel resolveChatModel(Long kbId, vip.mate.wiki.job.WikiJobStep step) { + private ResolvedChatModel resolveChatModel(Long kbId, WikiJobStep step) { if (modelRoutingService != null && kbId != null && step != null) { try { Long modelId = modelRoutingService.selectModelId(kbId, "heavy_ingest", step); @@ -2466,7 +2514,7 @@ public class WikiProcessingService { * stable, never random. * *

Skips providers currently in cooldown ({@link - * vip.mate.llm.failover.ProviderHealthTracker}) so a flapping provider + * ProviderHealthTracker}) so a flapping provider * doesn't keep getting tried while we wait for it to recover. */ private ResolvedChatModel pickFallbackChatModel(Long failedModelId) { @@ -2758,8 +2806,8 @@ public class WikiProcessingService { )); if (isAborted(raw.getId(), "repair page=" + page.getSlug())) return; String response = callLlmWithResilientRetry(prompt, "repair page=" + page.getSlug(), - kb.getId(), vip.mate.wiki.job.WikiJobStep.MERGE_PAGE); - com.fasterxml.jackson.databind.JsonNode pageJson = parseJsonResponse(response); + kb.getId(), WikiJobStep.MERGE_PAGE); + JsonNode pageJson = parseJsonResponse(response); if (pageJson == null) return; String content = pageJson.path("content").asText(""); @@ -2773,7 +2821,7 @@ public class WikiProcessingService { private List parseSourceRawIds(String json) { if (json == null || json.isBlank()) return List.of(); try { - return objectMapper.readValue(json, new com.fasterxml.jackson.core.type.TypeReference>() {}); + return objectMapper.readValue(json, new TypeReference>() {}); } catch (Exception e) { return List.of(); } @@ -2843,6 +2891,17 @@ public class WikiProcessingService { return config != null ? config.getIngestMode() : null; } + /** + * Read the {@code entityExtractionEnabled} opt-in from KB config. Defaults + * to {@code false} on any parse error or missing field — extraction is an + * opt-in cost and must never be turned on implicitly. + */ + private boolean isEntityExtractionEnabled(WikiKnowledgeBaseEntity kb) { + if (kb == null || kb.getConfigContent() == null) return false; + WikiKbConfig config = WikiKbConfigParser.parse(objectMapper, kb.getConfigContent()); + return config != null && Boolean.TRUE.equals(config.getEntityExtractionEnabled()); + } + /** * Returns {@code true} when the caller should bail out of an in-flight * processing path because the raw material has been deleted. @@ -2911,7 +2970,7 @@ public class WikiProcessingService { rawService.updateProgress(rawId, "lazy", 0, 0); progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_STARTED, - java.util.Map.of("rawId", rawId, "phase", "lazy")); + Map.of("rawId", rawId, "phase", "lazy")); try { String textContent = rawService.getTextContent(raw); @@ -2919,7 +2978,7 @@ public class WikiProcessingService { rawService.updateProcessingStatus(rawId, "failed", "No text content available"); kbService.updateStatus(kbId, "active"); progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED, - java.util.Map.of("rawId", rawId, "error", "No text content available")); + Map.of("rawId", rawId, "error", "No text content available")); return; } @@ -2957,7 +3016,7 @@ public class WikiProcessingService { if (embedded > 0) { log.info("[Wiki] Lazy async embedding completed: kbId={}, embedded={}", fKbId, embedded); } - } catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) { + } catch (WikiEmbeddingProviderFailingException ex) { log.warn("[Wiki] Lazy async embedding aborted by circuit-breaker for kbId={}: {}", fKbId, ex.getMessage()); } catch (Exception ex) { @@ -2975,7 +3034,7 @@ public class WikiProcessingService { kbService.updateStatus(kbId, "active"); progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED, - java.util.Map.of( + Map.of( "rawId", rawId, "status", "completed", "totalPages", 0, @@ -2993,7 +3052,7 @@ public class WikiProcessingService { // RFC-051 PR-2b: refresh overview stats. if (overviewService != null) overviewService.rebuild(kbId); // Tier 2: dirty event drives the LLM-narrated overview section. - eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kbId)); + eventPublisher.publishEvent(new WikiKbDirtyEvent(this, kbId)); log.info("[Wiki] Lazy processing completed for raw={}, kbId={}, chunks={}", rawId, kbId, totalChunks); @@ -3002,7 +3061,7 @@ public class WikiProcessingService { rawService.updateProcessingStatus(rawId, "failed", e.getMessage()); kbService.updateStatus(kbId, "active"); progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED, - java.util.Map.of("rawId", rawId, + Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage())); } } diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V148__wiki_entity.sql b/mateclaw-server/src/main/resources/db/migration/h2/V148__wiki_entity.sql new file mode 100644 index 00000000..fddba8cc --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V148__wiki_entity.sql @@ -0,0 +1,56 @@ +-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks. +-- +-- Distinct from mate_wiki_page (document/topic granularity) — a row here is a +-- mention-granularity named entity (person, organization, location, event, ...) +-- resolved and de-duplicated across the knowledge base. The entity layer sits +-- beneath the page layer: entities are linked to their source chunks (and, via +-- citing pages, to wiki pages) through mate_wiki_entity_mention, and to each +-- other through mate_wiki_entity_relation. +-- +-- canonical_name display name chosen for the merged entity +-- normalized_key case/whitespace-folded key used for exact-match dedup +-- type entity taxonomy: person | organization | location | +-- event | product | concept | other +-- aliases_json JSON array of surface forms merged into this entity +-- description one-line summary synthesized from the mentions +-- salience 0..1 importance score (mention frequency / distribution) +-- mention_count number of mentions resolved to this entity +-- embedding float32 little-endian name/description vector used for +-- near-duplicate merge across spellings/languages +-- computed_hash fingerprint of the inputs that produced this row + +CREATE TABLE IF NOT EXISTS mate_wiki_entity ( + id BIGINT PRIMARY KEY, + kb_id BIGINT NOT NULL, + + canonical_name VARCHAR(256) NOT NULL, + normalized_key VARCHAR(256) NOT NULL, + type VARCHAR(32) NOT NULL, + + aliases_json CLOB, + description CLOB, + salience DECIMAL(5, 4), + mention_count INT NOT NULL DEFAULT 0, + + embedding BLOB, + embedding_model VARCHAR(64), + + computed_hash VARCHAR(64), + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +-- Exact-match dedup identity within a KB (deleted included so soft-deleted +-- rows can coexist with re-inserted ones during re-extract cycles). +CREATE UNIQUE INDEX IF NOT EXISTS uk_we_key + ON mate_wiki_entity (kb_id, normalized_key, type, deleted); + +-- KB-wide listing and "top entities by salience". +CREATE INDEX IF NOT EXISTS idx_we_kb + ON mate_wiki_entity (kb_id, deleted); +CREATE INDEX IF NOT EXISTS idx_we_salience + ON mate_wiki_entity (kb_id, salience DESC); +CREATE INDEX IF NOT EXISTS idx_we_type + ON mate_wiki_entity (kb_id, type, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V149__wiki_entity_mention.sql b/mateclaw-server/src/main/resources/db/migration/h2/V149__wiki_entity_mention.sql new file mode 100644 index 00000000..a8ac79a2 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V149__wiki_entity_mention.sql @@ -0,0 +1,38 @@ +-- mate_wiki_entity_mention: links a canonical entity to a source occurrence. +-- +-- One row per (entity, chunk) occurrence. page_id is back-filled from the +-- chunk's citing pages so the entity layer connects to the page layer: +-- entity -> mention -> chunk -> citing page. A NULL page_id means the source +-- chunk is not yet cited by any generated page. +-- +-- entity_id the resolved canonical entity (mate_wiki_entity.id) +-- chunk_id source chunk the mention was found in +-- page_id a wiki page that cites that chunk, when known +-- surface_form the exact text as it appeared in the source +-- char_offset character offset of the mention within the chunk, when known +-- confidence 0..1 extraction confidence +-- evidence short surrounding quote (<= 500 chars enforced in Java) +-- source provenance tag: llm-extracted | manual + +CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention ( + id BIGINT PRIMARY KEY, + kb_id BIGINT NOT NULL, + entity_id BIGINT NOT NULL, + chunk_id BIGINT, + page_id BIGINT, + + surface_form VARCHAR(256), + char_offset INT, + confidence DECIMAL(4, 3), + evidence CLOB, + source VARCHAR(32), + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_wem_entity ON mate_wiki_entity_mention (entity_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wem_chunk ON mate_wiki_entity_mention (chunk_id); +CREATE INDEX IF NOT EXISTS idx_wem_page ON mate_wiki_entity_mention (page_id); +CREATE INDEX IF NOT EXISTS idx_wem_kb ON mate_wiki_entity_mention (kb_id, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V150__wiki_entity_relation.sql b/mateclaw-server/src/main/resources/db/migration/h2/V150__wiki_entity_relation.sql new file mode 100644 index 00000000..1502d4fc --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V150__wiki_entity_relation.sql @@ -0,0 +1,42 @@ +-- mate_wiki_entity_relation: directed subject -> predicate -> object triples +-- between canonical entities (the entity-level knowledge graph edges). +-- +-- Distinct from mate_wiki_relation, which scores page-to-page edges. A row +-- here is one fact triple connecting two mate_wiki_entity nodes. +-- +-- subject_entity_id head entity +-- predicate free-text relation label (e.g. "works_for", "located_in") +-- object_entity_id tail entity +-- evidence short justification quote (<= 500 chars enforced in Java) +-- confidence 0..1 extraction confidence +-- source provenance tag: llm-extracted | inferred | manual +-- evidence_chunk_id source chunk the triple was extracted from, when known +-- computed_hash fingerprint of the inputs that produced this row + +CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation ( + id BIGINT PRIMARY KEY, + kb_id BIGINT NOT NULL, + + subject_entity_id BIGINT NOT NULL, + predicate VARCHAR(64) NOT NULL, + object_entity_id BIGINT NOT NULL, + + evidence CLOB, + confidence DECIMAL(4, 3), + source VARCHAR(32), + evidence_chunk_id BIGINT, + computed_hash VARCHAR(64), + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +-- Unique triple identity within a KB. +CREATE UNIQUE INDEX IF NOT EXISTS uk_wer_triple + ON mate_wiki_entity_relation (kb_id, subject_entity_id, predicate, object_entity_id, deleted); + +-- Ego-graph traversal in both directions. +CREATE INDEX IF NOT EXISTS idx_wer_subject ON mate_wiki_entity_relation (kb_id, subject_entity_id); +CREATE INDEX IF NOT EXISTS idx_wer_object ON mate_wiki_entity_relation (kb_id, object_entity_id); +CREATE INDEX IF NOT EXISTS idx_wer_kb ON mate_wiki_entity_relation (kb_id, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/kingbase/V148__wiki_entity.sql b/mateclaw-server/src/main/resources/db/migration/kingbase/V148__wiki_entity.sql new file mode 100644 index 00000000..2f88fff7 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/kingbase/V148__wiki_entity.sql @@ -0,0 +1,31 @@ +-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks. +-- See the H2 file for the design rationale. + +CREATE TABLE IF NOT EXISTS mate_wiki_entity ( + id BIGINT PRIMARY KEY, + kb_id BIGINT NOT NULL, + + canonical_name VARCHAR(256) NOT NULL, + normalized_key VARCHAR(256) NOT NULL, + type VARCHAR(32) NOT NULL, + + aliases_json TEXT, + description TEXT, + salience DECIMAL(5, 4), + mention_count INT NOT NULL DEFAULT 0, + + embedding BYTEA, + embedding_model VARCHAR(64), + + computed_hash VARCHAR(64), + + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted SMALLINT NOT NULL DEFAULT 0 +); + +CREATE UNIQUE INDEX IF NOT EXISTS uk_we_key + ON mate_wiki_entity (kb_id, normalized_key, type, deleted); +CREATE INDEX IF NOT EXISTS idx_we_kb ON mate_wiki_entity (kb_id, deleted); +CREATE INDEX IF NOT EXISTS idx_we_salience ON mate_wiki_entity (kb_id, salience DESC); +CREATE INDEX IF NOT EXISTS idx_we_type ON mate_wiki_entity (kb_id, type, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/kingbase/V149__wiki_entity_mention.sql b/mateclaw-server/src/main/resources/db/migration/kingbase/V149__wiki_entity_mention.sql new file mode 100644 index 00000000..d989b281 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/kingbase/V149__wiki_entity_mention.sql @@ -0,0 +1,25 @@ +-- mate_wiki_entity_mention: links a canonical entity to a source occurrence. +-- See the H2 file for the design rationale. + +CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention ( + id BIGINT PRIMARY KEY, + kb_id BIGINT NOT NULL, + entity_id BIGINT NOT NULL, + chunk_id BIGINT, + page_id BIGINT, + + surface_form VARCHAR(256), + char_offset INT, + confidence DECIMAL(4, 3), + evidence TEXT, + source VARCHAR(32), + + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted SMALLINT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_wem_entity ON mate_wiki_entity_mention (entity_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wem_chunk ON mate_wiki_entity_mention (chunk_id); +CREATE INDEX IF NOT EXISTS idx_wem_page ON mate_wiki_entity_mention (page_id); +CREATE INDEX IF NOT EXISTS idx_wem_kb ON mate_wiki_entity_mention (kb_id, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/kingbase/V150__wiki_entity_relation.sql b/mateclaw-server/src/main/resources/db/migration/kingbase/V150__wiki_entity_relation.sql new file mode 100644 index 00000000..df1fbfe0 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/kingbase/V150__wiki_entity_relation.sql @@ -0,0 +1,27 @@ +-- mate_wiki_entity_relation: directed subject -> predicate -> object triples +-- between canonical entities. See the H2 file for the design rationale. + +CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation ( + id BIGINT PRIMARY KEY, + kb_id BIGINT NOT NULL, + + subject_entity_id BIGINT NOT NULL, + predicate VARCHAR(64) NOT NULL, + object_entity_id BIGINT NOT NULL, + + evidence TEXT, + confidence DECIMAL(4, 3), + source VARCHAR(32), + evidence_chunk_id BIGINT, + computed_hash VARCHAR(64), + + create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted SMALLINT NOT NULL DEFAULT 0 +); + +CREATE UNIQUE INDEX IF NOT EXISTS uk_wer_triple + ON mate_wiki_entity_relation (kb_id, subject_entity_id, predicate, object_entity_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wer_subject ON mate_wiki_entity_relation (kb_id, subject_entity_id); +CREATE INDEX IF NOT EXISTS idx_wer_object ON mate_wiki_entity_relation (kb_id, object_entity_id); +CREATE INDEX IF NOT EXISTS idx_wer_kb ON mate_wiki_entity_relation (kb_id, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V148__wiki_entity.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V148__wiki_entity.sql new file mode 100644 index 00000000..90a1040c --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V148__wiki_entity.sql @@ -0,0 +1,32 @@ +-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks. +-- See the H2 file for the design rationale. + +CREATE TABLE IF NOT EXISTS mate_wiki_entity ( + id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + + canonical_name VARCHAR(256) NOT NULL, + normalized_key VARCHAR(256) NOT NULL, + type VARCHAR(32) NOT NULL, + + aliases_json LONGTEXT, + description LONGTEXT, + salience DECIMAL(5, 4), + mention_count INT NOT NULL DEFAULT 0, + + embedding BLOB, + embedding_model VARCHAR(64), + + computed_hash VARCHAR(64), + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + PRIMARY KEY (id), + UNIQUE KEY uk_we_key (kb_id, normalized_key, type, deleted), + KEY idx_we_kb (kb_id, deleted), + KEY idx_we_salience (kb_id, salience DESC), + KEY idx_we_type (kb_id, type, deleted) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Canonical named-entity nodes extracted and de-duplicated from source chunks.'; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V149__wiki_entity_mention.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V149__wiki_entity_mention.sql new file mode 100644 index 00000000..557491a3 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V149__wiki_entity_mention.sql @@ -0,0 +1,27 @@ +-- mate_wiki_entity_mention: links a canonical entity to a source occurrence. +-- See the H2 file for the design rationale. + +CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention ( + id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + entity_id BIGINT NOT NULL, + chunk_id BIGINT, + page_id BIGINT, + + surface_form VARCHAR(256), + char_offset INT, + confidence DECIMAL(4, 3), + evidence TEXT, + source VARCHAR(32), + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + PRIMARY KEY (id), + KEY idx_wem_entity (entity_id, deleted), + KEY idx_wem_chunk (chunk_id), + KEY idx_wem_page (page_id), + KEY idx_wem_kb (kb_id, deleted) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Entity-to-source occurrence links connecting the entity layer to chunks and pages.'; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V150__wiki_entity_relation.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V150__wiki_entity_relation.sql new file mode 100644 index 00000000..02d91a92 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V150__wiki_entity_relation.sql @@ -0,0 +1,28 @@ +-- mate_wiki_entity_relation: directed subject -> predicate -> object triples +-- between canonical entities. See the H2 file for the design rationale. + +CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation ( + id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + + subject_entity_id BIGINT NOT NULL, + predicate VARCHAR(64) NOT NULL, + object_entity_id BIGINT NOT NULL, + + evidence TEXT, + confidence DECIMAL(4, 3), + source VARCHAR(32), + evidence_chunk_id BIGINT, + computed_hash VARCHAR(64), + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + PRIMARY KEY (id), + UNIQUE KEY uk_wer_triple (kb_id, subject_entity_id, predicate, object_entity_id, deleted), + KEY idx_wer_subject (kb_id, subject_entity_id), + KEY idx_wer_object (kb_id, object_entity_id), + KEY idx_wer_kb (kb_id, deleted) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Entity-to-entity fact triples forming the entity-level knowledge graph edges.'; diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiEntityExtractionServiceTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiEntityExtractionServiceTest.java new file mode 100644 index 00000000..9bfffde2 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiEntityExtractionServiceTest.java @@ -0,0 +1,179 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.model.WikiChunkEntity; +import vip.mate.wiki.model.WikiEntityEntity; +import vip.mate.wiki.model.WikiEntityMentionEntity; +import vip.mate.wiki.model.WikiEntityRelationEntity; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.repository.WikiEntityMapper; +import vip.mate.wiki.repository.WikiEntityMentionMapper; +import vip.mate.wiki.repository.WikiEntityRelationMapper; +import vip.mate.wiki.repository.WikiPageCitationMapper; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Verifies the entity-extraction runtime logic against a mocked model + DB: + * the structured LLM output is parsed, entities are de-duplicated across + * chunks via the run cache, mentions are written per occurrence, and a + * resolved relation triple is persisted. This exercises the real + * {@link WikiEntityExtractionService} control flow, not just a stubbed call. + */ +class WikiEntityExtractionServiceTest { + + private static final Long KB_ID = 1L; + private static final Long RAW_ID = 100L; + + private static final String LLM_JSON = """ + { + "entities": [ + {"name": "Alice", "type": "person", "aliases": [], "description": "An engineer", "evidence": "Alice works at Acme"}, + {"name": "Acme", "type": "organization", "aliases": [], "description": "A company", "evidence": "Acme Corp"} + ], + "relations": [ + {"subject": "Alice", "predicate": "works for", "object": "Acme", "evidence": "Alice works at Acme"} + ] + } + """; + + private WikiKnowledgeBaseService kbService; + private WikiChunkService chunkService; + private WikiEmbeddingService embeddingService; + private WikiModelRoutingService routingService; + private ModelConfigService modelConfigService; + private WikiEntityMapper entityMapper; + private WikiEntityMentionMapper mentionMapper; + private WikiEntityRelationMapper relationMapper; + private WikiPageCitationMapper citationMapper; + + private WikiEntityExtractionService service; + + /** Simulated entity store so selectById sees what insert assigned. */ + private final Map store = new HashMap<>(); + + @BeforeEach + void setUp() { + kbService = mock(WikiKnowledgeBaseService.class); + chunkService = mock(WikiChunkService.class); + embeddingService = mock(WikiEmbeddingService.class); + routingService = mock(WikiModelRoutingService.class); + modelConfigService = mock(ModelConfigService.class); + entityMapper = mock(WikiEntityMapper.class); + mentionMapper = mock(WikiEntityMentionMapper.class); + relationMapper = mock(WikiEntityRelationMapper.class); + citationMapper = mock(WikiPageCitationMapper.class); + + service = new WikiEntityExtractionService( + kbService, chunkService, embeddingService, routingService, + modelConfigService, new ObjectMapper(), + entityMapper, mentionMapper, relationMapper, citationMapper); + + WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity(); + kb.setId(KB_ID); + when(kbService.getById(KB_ID)).thenReturn(kb); + + // Model routing → a mock ChatModel that always returns the canned JSON. + ChatModel canned = cannedModel(LLM_JSON); + when(routingService.selectModelId(eq(KB_ID), any(), eq(WikiJobStep.ENTITY_EXTRACTION))) + .thenReturn(7L); + when(routingService.buildChatModel(7L)).thenReturn(canned); + + // No prior entities (with embeddings), no embedding vectors → exercise + // the exact-key dedup path (no embedding merge). + when(entityMapper.selectList(any())).thenReturn(Collections.emptyList()); + when(entityMapper.selectOne(any())).thenReturn(null); + when(embeddingService.embedQuery(anyLong(), any())).thenReturn(null); + + // No existing mentions → every chunk gets processed. + when(mentionMapper.selectCount(any(Wrapper.class))).thenReturn(0L); + when(citationMapper.listPageIdsByChunkId(anyLong())).thenReturn(Collections.emptyList()); + when(relationMapper.selectOne(any())).thenReturn(null); + + // insert assigns a snowflake-like id and records the row so selectById works. + AtomicLong seq = new AtomicLong(1000L); + when(entityMapper.insert(any(WikiEntityEntity.class))).thenAnswer(inv -> { + WikiEntityEntity e = inv.getArgument(0); + e.setId(seq.getAndIncrement()); + store.put(e.getId(), e); + return 1; + }); + when(entityMapper.selectById(anyLong())).thenAnswer(inv -> store.get(inv.getArgument(0))); + } + + @Test + @DisplayName("extractForRaw: dedups entities across chunks, writes mentions and a relation") + void extractForRaw_buildsGraph() { + when(chunkService.listByRawId(RAW_ID)).thenReturn(List.of( + chunk(1L, "Alice works at Acme."), + chunk(2L, "Acme promoted Alice."))); + + int touched = service.extractForRaw(KB_ID, RAW_ID); + + // Two distinct canonical entities despite two chunks naming them. + assertEquals(2, touched, "should resolve exactly two canonical entities"); + verify(entityMapper, times(2)).insert(any(WikiEntityEntity.class)); + + // One mention per (entity, chunk) occurrence → 2 entities * 2 chunks. + verify(mentionMapper, times(4)).insert(any(WikiEntityMentionEntity.class)); + + // The works_for triple is persisted once per chunk it appears in. + verify(relationMapper, times(2)).insert(any(WikiEntityRelationEntity.class)); + } + + @Test + @DisplayName("extractForRaw: skips chunks that already have mentions") + void extractForRaw_skipsProcessedChunks() { + when(chunkService.listByRawId(RAW_ID)).thenReturn(List.of(chunk(1L, "Alice works at Acme."))); + when(mentionMapper.selectCount(any(Wrapper.class))).thenReturn(3L); + + int touched = service.extractForRaw(KB_ID, RAW_ID); + + assertEquals(0, touched); + verify(entityMapper, times(0)).insert(any(WikiEntityEntity.class)); + } + + private WikiChunkEntity chunk(Long id, String content) { + WikiChunkEntity c = new WikiChunkEntity(); + c.setId(id); + c.setKbId(KB_ID); + c.setRawId(RAW_ID); + c.setContent(content); + return c; + } + + private ChatModel cannedModel(String body) { + ChatModel model = mock(ChatModel.class); + when(model.call(any(Prompt.class))).thenReturn(new ChatResponse(List.of( + new Generation(new AssistantMessage(body), + ChatGenerationMetadata.builder().finishReason("STOP").build())))); + return model; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingFallbackTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingFallbackTest.java index 0dce5681..16518cbb 100644 --- a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingFallbackTest.java +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingFallbackTest.java @@ -68,7 +68,8 @@ class WikiProcessingFallbackTest { om, mock(WikiProgressBus.class), mock(WikiCitationService.class), - mock(ApplicationEventPublisher.class)); + mock(ApplicationEventPublisher.class), + mock(WikiEntityExtractionService.class)); // Inject the optional fields via reflection — Spring would do this // post-construction in production, but the test instantiates directly. diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingServiceLazyTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingServiceLazyTest.java index f49591a0..075407fc 100644 --- a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingServiceLazyTest.java +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiProcessingServiceLazyTest.java @@ -67,7 +67,8 @@ class WikiProcessingServiceLazyTest { new WikiLinkService(om), properties, modelConfigService, agentGraphBuilder, om, progressBus, citationService, - mock(org.springframework.context.ApplicationEventPublisher.class)); + mock(org.springframework.context.ApplicationEventPublisher.class), + mock(WikiEntityExtractionService.class)); } private WikiRawMaterialEntity raw() { diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 49a5c439..00fb40d0 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -846,6 +846,16 @@ export const wikiApi = { getPageCitations: (kbId: number | string, pageId: number | string) => http.get(`/wiki/kb/${kbId}/pages/${pageId}/citations`), + // Entity-level knowledge graph + listEntities: (kbId: number | string, params?: { type?: string; limit?: number }) => + http.get(`/wiki/kb/${kbId}/entities`, { params }), + getEntityGraph: (kbId: number | string, limit = 150) => + http.get(`/wiki/kb/${kbId}/entity-graph`, { params: { limit } }), + getEntityEgo: (kbId: number | string, entityId: number | string, limit = 50) => + http.get(`/wiki/kb/${kbId}/entities/${entityId}/graph`, { params: { limit } }), + extractEntities: (kbId: number | string, force = false) => + http.post(`/wiki/kb/${kbId}/entities/extract`, null, { params: { force } }), + // RFC-030: Jobs getWikiJobs: (kbId: number, rawId: number) => http.get(`/wiki/kb/${kbId}/jobs`, { params: { rawId } }), diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 0e6ace05..4fc3d992 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -2442,6 +2442,11 @@ export default { ingestModeLazy: 'Lazy', ingestModeEagerHint: 'Upload runs the full LLM pipeline to generate wiki pages immediately.', ingestModeLazyHint: 'Upload only extracts, chunks, and embeds. No page generation; search works right away.', + entityExtraction: 'Entity extraction', + entityExtractionHint: 'Extract named entities (people, organizations, locations, ...) and their relations from raw materials to build an entity-level knowledge graph. Adds processing time and token cost; off by default.', + entityExtractionEnable: 'Enable entity extraction', + entityExtractionRun: 'Extract now', + entityExtractionRunning: 'Extracting…', modelStrategy: 'Model Strategy', globalDefault: 'Global default', selectModel: 'Select a model…', @@ -2517,6 +2522,15 @@ export default { linksTo: 'Links to', openPage: 'Open page', empty: 'No graph data — process some raw materials first', + modePages: 'Pages', + modeEntities: 'Entities', + entityEmpty: 'No entities yet — enable entity extraction in the KB config and reprocess', + entityType: 'Type', + mentions: 'mentions', + mentionedIn: 'Mentioned in', + relations: 'Relations', + description: 'Description', + aliases: 'Aliases', }, }, cronField: { diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index fa2064eb..ca304ac1 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -2454,6 +2454,11 @@ export default { ingestModeLazy: '先入索引', ingestModeEagerHint: '上传后立即调用 LLM 生成完整 Wiki 页面。', ingestModeLazyHint: '上传仅抽取、切片、向量化，跳过页面生成；上传即可搜索，页面按需编译。', + entityExtraction: '实体抽取', + entityExtractionHint: '从原始材料中抽取命名实体（人物、组织、地点等）及其关系，构建实体级知识图谱。会增加处理时间与 Token 消耗，默认关闭。', + entityExtractionEnable: '开启实体抽取', + entityExtractionRun: '立即抽取', + entityExtractionRunning: '抽取中…', modelStrategy: '模型策略', globalDefault: '跟随全局默认', selectModel: '选择可用模型…', @@ -2529,6 +2534,15 @@ export default { linksTo: '链接到', openPage: '打开页面', empty: '暂无图谱数据，请先处理原始材料', + modePages: '页面', + modeEntities: '实体', + entityEmpty: '暂无实体数据，请在知识库配置中开启实体抽取后重新处理', + entityType: '类型', + mentions: '提及', + mentionedIn: '出现于页面', + relations: '关系', + description: '描述', + aliases: '别名', }, }, cronField: { diff --git a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue index 329e1564..b2babbfb 100644 --- a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue +++ b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue @@ -46,6 +46,33 @@ + +

+
+
+
{{ t('wiki.configPanel.entityExtraction') }}
+
{{ t('wiki.configPanel.entityExtractionHint') }}
+
+ +
+
+ + +
+
+
@@ -230,6 +257,42 @@ async function saveIngestMode() { } } +// ── Entity extraction ── +// Opt-in named-entity knowledge graph extraction. Off by default because it +// adds an LLM call per chunk on top of the page pipeline. +const entityExtractionEnabled = ref(false) +const savingEntityExtraction = ref(false) +const extracting = ref(false) + +async function saveEntityExtraction() { + if (!store.currentKB) return + savingEntityExtraction.value = true + try { + let existingConfig: any = {} + try { + if (store.currentKB.configContent) existingConfig = JSON.parse(store.currentKB.configContent) + } catch { /* config may be plain text rules */ } + existingConfig.entityExtractionEnabled = entityExtractionEnabled.value ? true : undefined + await wikiApi.updateConfig(store.currentKB.id, JSON.stringify(existingConfig, null, 2)) + } catch (e) { + console.error('[WikiConfig] Failed to save entity extraction toggle', e) + } finally { + savingEntityExtraction.value = false + } +} + +async function runExtraction() { + if (!store.currentKB) return + extracting.value = true + try { + await wikiApi.extractEntities(store.currentKB.id) + } catch (e) { + console.error('[WikiConfig] Failed to start entity extraction', e) + } finally { + extracting.value = false + } +} + // ── Model strategy ── const stepKeys = ['route', 'create_page', 'merge_page', 'enrich', 'summary'] const stepModels = reactive>({}) @@ -246,6 +309,7 @@ function loadStepModels() { fallbackModelIds.value = [] wikiGlobalModelId.value = '' ingestMode.value = 'eager' + entityExtractionEnabled.value = false if (!store.currentKB) return try { const cfg = store.currentKB.configContent ? JSON.parse(store.currentKB.configContent) : null @@ -258,6 +322,7 @@ function loadStepModels() { if (cfg?.fallbackModelIds) fallbackModelIds.value = cfg.fallbackModelIds.map(String) if (cfg?.wikiDefaultModelId) wikiGlobalModelId.value = String(cfg.wikiDefaultModelId) if (cfg?.ingestMode === 'lazy') ingestMode.value = 'lazy' + if (cfg?.entityExtractionEnabled) entityExtractionEnabled.value = true } catch { /* not JSON */ } } @@ -475,6 +540,12 @@ loadProviderNames().then(() => { } .btn-save:hover { opacity: 0.88; } .btn-save:disabled { background: var(--mc-border); cursor: not-allowed; } +.btn-save--ghost { background: transparent; color: var(--mc-primary); border: 1px solid var(--mc-primary); } +.btn-save--ghost:disabled { background: transparent; color: var(--mc-text-tertiary); border-color: var(--mc-border); } + +/* Entity extraction toggle */ +.entity-toggle { display: flex; align-items: center; gap: 6px; font-size: 13px; color: var(--mc-text-primary); cursor: pointer; } +.entity-toggle__label { user-select: none; } /* Ingest mode radio group */ .ingest-mode-row { display: flex; gap: 8px; flex-wrap: wrap; } diff --git a/mateclaw-ui/src/views/Wiki/components/WikiEntityGraphView.vue b/mateclaw-ui/src/views/Wiki/components/WikiEntityGraphView.vue new file mode 100644 index 00000000..58e639ea --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiEntityGraphView.vue @@ -0,0 +1,264 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiGraphToolbar.vue b/mateclaw-ui/src/views/Wiki/components/WikiGraphToolbar.vue index 66bda5c1..c1958eda 100644 --- a/mateclaw-ui/src/views/Wiki/components/WikiGraphToolbar.vue +++ b/mateclaw-ui/src/views/Wiki/components/WikiGraphToolbar.vue @@ -13,7 +13,7 @@ {{ edgeCount }} {{ t('wiki.graph.edges') }} - + @@ -22,11 +22,23 @@
-