feat(wiki): entity-level knowledge graph extraction (#336)

Add an opt-in named-entity extraction pass so the wiki knowledge graph
captures fine-grained entities (people, organizations, locations, ...)
and their relations, not just page-level link relations.

- new tables mate_wiki_entity / _mention / _relation (h2/mysql/kingbase)
- structured LLM extraction per chunk with entity resolution
  (normalized-key dedup + embedding near-merge), mention/relation
  persistence and page linking via chunk citations
- per-KB opt-in toggle (off by default); async dispatch after embedding
- read API: entity list, KB graph, entity ego-graph, manual extract
- UI: entity-layer toggle in the graph view + KB config toggle
- replace inline fully-qualified class names with imports in WikiProcessingService

Closes #336
This commit is contained in:
matevip 2026-06-17 14:17:54 +08:00
parent fe68f22aa8
commit 6e7c137154
34 changed files with 2164 additions and 102 deletions

View File

@ -0,0 +1,76 @@
package vip.mate.wiki.controller;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import vip.mate.wiki.dto.WikiEntityGraphView;
import vip.mate.wiki.dto.WikiEntityView;
import vip.mate.wiki.service.WikiEntityExtractionService;
import vip.mate.wiki.service.WikiEntityGraphService;
import vip.mate.wiki.service.WikiProcessingService;
import java.util.List;
import java.util.Map;
/**
* Read and manual-trigger endpoints for the entity-level knowledge graph.
*
* @author MateClaw Team
*/
@Slf4j
@RestController
@RequestMapping("/api/v1/wiki")
@RequiredArgsConstructor
public class WikiEntityController {
private final WikiEntityGraphService graphService;
private final WikiEntityExtractionService extractionService;
/** List entities in a KB, optionally filtered by type, ranked by salience. */
@GetMapping("/kb/{kbId}/entities")
public List<WikiEntityView> listEntities(@PathVariable Long kbId,
@RequestParam(required = false) String type,
@RequestParam(defaultValue = "100") int limit) {
return graphService.listEntities(kbId, type, limit);
}
/** Whole-KB entity graph: top entities by salience plus the edges among them. */
@GetMapping("/kb/{kbId}/entity-graph")
public WikiEntityGraphView kbEntityGraph(@PathVariable Long kbId,
@RequestParam(defaultValue = "150") int limit) {
return graphService.graph(kbId, limit);
}
/** Ego-graph around a single entity: neighbors, edges, and mentioning pages. */
@GetMapping("/kb/{kbId}/entities/{entityId}/graph")
public WikiEntityGraphView entityGraph(@PathVariable Long kbId,
@PathVariable Long entityId,
@RequestParam(defaultValue = "50") int limit) {
return graphService.ego(kbId, entityId, limit);
}
/**
* Manually trigger an entity-extraction pass for a KB. Runs on the wiki
* executor so the request returns immediately.
*
* @param force when true, re-extract chunks that already have mentions
*/
@PostMapping("/kb/{kbId}/entities/extract")
public Map<String, Object> extract(@PathVariable Long kbId,
@RequestParam(defaultValue = "false") boolean force) {
WikiProcessingService.WIKI_EXECUTOR.submit(() -> {
try {
int count = extractionService.extractForKb(kbId, force);
log.info("[WikiEntity] Manual extraction completed: kbId={}, entities={}", kbId, count);
} catch (Exception e) {
log.warn("[WikiEntity] Manual extraction failed for kbId={}: {}", kbId, e.getMessage());
}
});
return Map.of("status", "started", "kbId", kbId);
}
}

View File

@ -0,0 +1,49 @@
package vip.mate.wiki.dto;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* Structured output schema for a single entity-extraction LLM call over one
* source chunk. Bound via {@code BeanOutputConverter} and tolerant of a model
* returning either field empty.
*
* @author MateClaw Team
*/
@Data
public class EntityExtractionResult {
/** Named entities found in the chunk. */
private List<ExtractedEntity> entities = new ArrayList<>();
/** Subject → predicate → object triples between the extracted entities. */
private List<ExtractedRelation> relations = new ArrayList<>();
@Data
public static class ExtractedEntity {
/** Canonical surface form of the entity as it appears in the text. */
private String name;
/** One of the requested types: person | organization | location | event | product | concept | other. */
private String type;
/** Alternate names / spellings for the same entity, if any. */
private List<String> aliases = new ArrayList<>();
/** One-line description grounded in the chunk. */
private String description;
/** Short verbatim quote evidencing the entity. */
private String evidence;
}
@Data
public static class ExtractedRelation {
/** Subject entity name (should match an entry in {@link #entities}). */
private String subject;
/** Relation label, e.g. "works_for", "located_in", "founded". */
private String predicate;
/** Object entity name (should match an entry in {@link #entities}). */
private String object;
/** Short verbatim quote evidencing the relation. */
private String evidence;
}
}

View File

@ -0,0 +1,40 @@
package vip.mate.wiki.dto;
import lombok.Data;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
/**
* Ego-graph around one entity: the center node, its neighbor entity nodes,
* the relation edges connecting them, and the wiki pages that mention the
* center entity (the bridge from the entity layer to the page layer).
*
* @author MateClaw Team
*/
@Data
public class WikiEntityGraphView {
private WikiEntityView center;
private List<WikiEntityView> nodes = new ArrayList<>();
private List<Edge> edges = new ArrayList<>();
private List<PageRef> pages = new ArrayList<>();
@Data
public static class Edge {
private Long id;
private Long subjectEntityId;
private String predicate;
private Long objectEntityId;
private String evidence;
private BigDecimal confidence;
}
@Data
public static class PageRef {
private Long pageId;
private String slug;
private String title;
}
}

View File

@ -0,0 +1,24 @@
package vip.mate.wiki.dto;
import lombok.Data;
import java.math.BigDecimal;
import java.util.List;
/**
* API-facing projection of a canonical entity node. Excludes the raw
* embedding vector and other internal columns.
*
* @author MateClaw Team
*/
@Data
public class WikiEntityView {
private Long id;
private Long kbId;
private String canonicalName;
private String type;
private List<String> aliases;
private String description;
private BigDecimal salience;
private Integer mentionCount;
}

View File

@ -1,9 +1,8 @@
package vip.mate.wiki.job;
/**
* RFC-030: Logical steps within a wiki processing job,
* used for per-step model routing.
* Logical steps within a wiki processing job, used for per-step model routing.
*/
public enum WikiJobStep {
ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY
ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY, ENTITY_EXTRACTION
}

View File

@ -57,4 +57,21 @@ public class WikiKbConfig {
* pageType to be granted explicitly per agent.
*/
private String defaultReadPolicy;
/**
* Opt-in for entity-level knowledge graph extraction on this KB. When
* {@code true}, an extraction pass runs after ingest/embedding to pull
* named entities (person, organization, location, ...) and their
* relations from source chunks into the {@code mate_wiki_entity*} tables.
* {@code null} or {@code false} keeps the legacy behaviour (page graph
* only). Off by default because extraction adds LLM calls per chunk.
*/
private Boolean entityExtractionEnabled;
/**
* Optional whitelist of entity types to extract, e.g.
* {@code ["person","organization","location"]}. {@code null} or empty
* lets the extractor use its built-in default type set.
*/
private List<String> entityTypes;
}

View File

@ -0,0 +1,72 @@
package vip.mate.wiki.model;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableLogic;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
/**
* Canonical named-entity node extracted and de-duplicated from source chunks.
*
* <p>Mention-granularity counterpart to {@link WikiPageEntity} (which models
* document/topic granularity). Entities link to their source occurrences via
* {@link WikiEntityMentionEntity} and to one another via
* {@link WikiEntityRelationEntity}, forming an entity-level knowledge graph
* beneath the page graph cached in {@link WikiRelationEntity}.
*
* @author MateClaw Team
*/
@Data
@TableName("mate_wiki_entity")
public class WikiEntityEntity {
@TableId(type = IdType.ASSIGN_ID)
private Long id;
private Long kbId;
/** Display name chosen for the merged entity. */
private String canonicalName;
/** Case/whitespace-folded key used for exact-match de-duplication. */
private String normalizedKey;
/** Entity taxonomy: person | organization | location | event | product | concept | other. */
private String type;
/** JSON array of surface forms merged into this entity. */
private String aliasesJson;
/** One-line summary synthesized from the mentions. */
private String description;
/** 0..1 importance score derived from mention frequency / distribution. */
private BigDecimal salience;
/** Number of mentions resolved to this entity. */
private Integer mentionCount;
/** Float32 little-endian name/description vector used for near-duplicate merge. */
private byte[] embedding;
/** Model name that produced {@link #embedding}; used for re-embed detection. */
private String embeddingModel;
/** Fingerprint of the inputs that produced this row; used for cache invalidation. */
private String computedHash;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableLogic
private Integer deleted;
}

View File

@ -0,0 +1,64 @@
package vip.mate.wiki.model;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableLogic;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
/**
* Links a canonical {@link WikiEntityEntity} to a single source occurrence.
*
* <p>One row per (entity, chunk) occurrence. {@link #pageId} is back-filled
* from the chunk's citing pages so the entity layer connects to the page
* layer: entity mention chunk citing page.
*
* @author MateClaw Team
*/
@Data
@TableName("mate_wiki_entity_mention")
public class WikiEntityMentionEntity {
@TableId(type = IdType.ASSIGN_ID)
private Long id;
private Long kbId;
/** The resolved canonical entity. */
private Long entityId;
/** Source chunk the mention was found in. */
private Long chunkId;
/** A wiki page that cites {@link #chunkId}, when known; null otherwise. */
private Long pageId;
/** The exact text as it appeared in the source. */
private String surfaceForm;
/** Character offset of the mention within the chunk, when known. */
private Integer charOffset;
/** 0..1 extraction confidence. */
private BigDecimal confidence;
/** Short surrounding quote (≤ 500 chars enforced in Java layer). */
private String evidence;
/** Provenance tag: llm-extracted | manual. */
private String source;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableLogic
private Integer deleted;
}

View File

@ -0,0 +1,64 @@
package vip.mate.wiki.model;
import com.baomidou.mybatisplus.annotation.FieldFill;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableLogic;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.math.BigDecimal;
import java.time.LocalDateTime;
/**
* Directed subject predicate object triple between two canonical entities;
* the edges of the entity-level knowledge graph.
*
* <p>Distinct from {@link WikiRelationEntity}, which scores page-to-page edges.
* A row here is one fact triple connecting two {@link WikiEntityEntity} nodes.
*
* @author MateClaw Team
*/
@Data
@TableName("mate_wiki_entity_relation")
public class WikiEntityRelationEntity {
@TableId(type = IdType.ASSIGN_ID)
private Long id;
private Long kbId;
/** Head entity. */
private Long subjectEntityId;
/** Free-text relation label, e.g. {@code works_for}, {@code located_in}. */
private String predicate;
/** Tail entity. */
private Long objectEntityId;
/** Short justification quote (≤ 500 chars enforced in Java layer). */
private String evidence;
/** 0..1 extraction confidence. */
private BigDecimal confidence;
/** Provenance tag: llm-extracted | inferred | manual. */
private String source;
/** Source chunk the triple was extracted from, when known. */
private Long evidenceChunkId;
/** Fingerprint of the inputs that produced this row; used for cache invalidation. */
private String computedHash;
@TableField(fill = FieldFill.INSERT)
private LocalDateTime createTime;
@TableField(fill = FieldFill.INSERT_UPDATE)
private LocalDateTime updateTime;
@TableLogic
private Integer deleted;
}

View File

@ -0,0 +1,17 @@
package vip.mate.wiki.repository;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import vip.mate.wiki.model.WikiEntityEntity;
/**
* Mapper for canonical named-entity nodes.
*
* <p>Write paths upsert keyed by ({@code kb_id}, {@code normalized_key},
* {@code type}); read paths list by {@code kb_id} ordered by {@code salience}.
*
* @author MateClaw Team
*/
@Mapper
public interface WikiEntityMapper extends BaseMapper<WikiEntityEntity> {
}

View File

@ -0,0 +1,17 @@
package vip.mate.wiki.repository;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import vip.mate.wiki.model.WikiEntityMentionEntity;
/**
* Mapper for entity-to-source occurrence links.
*
* <p>Read paths fetch mentions by {@code entity_id} or by {@code page_id};
* cache invalidation soft-deletes by {@code chunk_id} or {@code kb_id}.
*
* @author MateClaw Team
*/
@Mapper
public interface WikiEntityMentionMapper extends BaseMapper<WikiEntityMentionEntity> {
}

View File

@ -0,0 +1,19 @@
package vip.mate.wiki.repository;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import vip.mate.wiki.model.WikiEntityRelationEntity;
/**
* Mapper for entity-to-entity fact triples.
*
* <p>Read paths traverse the ego-graph by {@code subject_entity_id} /
* {@code object_entity_id}; write paths upsert keyed by the triple
* ({@code kb_id}, {@code subject_entity_id}, {@code predicate},
* {@code object_entity_id}).
*
* @author MateClaw Team
*/
@Mapper
public interface WikiEntityRelationMapper extends BaseMapper<WikiEntityRelationEntity> {
}

View File

@ -0,0 +1,457 @@
package vip.mate.wiki.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.stereotype.Service;
import vip.mate.llm.service.ModelConfigService;
import vip.mate.wiki.dto.EntityExtractionResult;
import vip.mate.wiki.job.WikiJobStep;
import vip.mate.wiki.job.WikiKbConfig;
import vip.mate.wiki.job.WikiKbConfigParser;
import vip.mate.wiki.job.WikiModelRoutingService;
import vip.mate.wiki.model.WikiChunkEntity;
import vip.mate.wiki.model.WikiEntityEntity;
import vip.mate.wiki.model.WikiEntityMentionEntity;
import vip.mate.wiki.model.WikiEntityRelationEntity;
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
import vip.mate.wiki.repository.WikiEntityMapper;
import vip.mate.wiki.repository.WikiEntityMentionMapper;
import vip.mate.wiki.repository.WikiEntityRelationMapper;
import vip.mate.wiki.repository.WikiPageCitationMapper;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* Extracts a named-entity knowledge graph from source chunks: it pulls
* entities (person, organization, location, ...) and subjectpredicateobject
* relations out of each chunk via a structured LLM call, resolves entities to
* canonical nodes (exact-key dedup plus optional embedding near-merge), and
* persists nodes, mentions, and edges into the {@code mate_wiki_entity*}
* tables.
*
* <p>This is an opt-in pass gated by {@link WikiKbConfig#getEntityExtractionEnabled()};
* it runs after ingest/embedding and never blocks the page-generation pipeline.
*
* @author MateClaw Team
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class WikiEntityExtractionService {
private static final List<String> DEFAULT_ENTITY_TYPES =
List.of("person", "organization", "location", "event", "product", "concept");
/** Cosine threshold above which a new entity is merged into an existing same-type node. */
private static final float MERGE_THRESHOLD = 0.92f;
/** Max chunk characters sent to the model per call, to bound token spend. */
private static final int MAX_CHUNK_CHARS = 6000;
/** Evidence column is capped at 500 chars in the schema. */
private static final int MAX_EVIDENCE = 500;
private final WikiKnowledgeBaseService kbService;
private final WikiChunkService chunkService;
private final WikiEmbeddingService embeddingService;
private final WikiModelRoutingService routingService;
private final ModelConfigService modelConfigService;
private final ObjectMapper objectMapper;
private final WikiEntityMapper entityMapper;
private final WikiEntityMentionMapper mentionMapper;
private final WikiEntityRelationMapper relationMapper;
private final WikiPageCitationMapper citationMapper;
/** Extract entities from every not-yet-processed chunk of one raw material. */
public int extractForRaw(Long kbId, Long rawId) {
return extract(kbId, chunkService.listByRawId(rawId), false);
}
/**
* Extract entities across the whole KB.
*
* @param force when {@code true}, re-extract chunks that already have
* mentions (used for a manual full rebuild); otherwise skip
* chunks already processed
*/
public int extractForKb(Long kbId, boolean force) {
return extract(kbId, chunkService.listByKbId(kbId), force);
}
private int extract(Long kbId, List<WikiChunkEntity> chunks, boolean force) {
WikiKnowledgeBaseEntity kb = kbService.getById(kbId);
if (kb == null || chunks == null || chunks.isEmpty()) {
return 0;
}
ChatModel chatModel = resolveChatModel(kbId);
if (chatModel == null) {
log.warn("[WikiEntity] No chat model available for kbId={}, skipping extraction", kbId);
return 0;
}
List<String> types = resolveEntityTypes(kb);
BeanOutputConverter<EntityExtractionResult> converter =
new BeanOutputConverter<>(EntityExtractionResult.class);
String systemPrompt = buildSystemPrompt(types);
// Per-run resolution cache: type+normalizedKey entityId. Seeded lazily
// from the DB so entities resolve consistently within and across chunks.
Map<String, Long> resolved = new HashMap<>();
// Same-type existing entities with embeddings, for near-duplicate merge.
EntityIndex index = new EntityIndex(kbId);
for (WikiChunkEntity chunk : chunks) {
if (chunk.getContent() == null || chunk.getContent().isBlank()) {
continue;
}
if (!force && hasMentions(chunk.getId())) {
continue;
}
try {
EntityExtractionResult result = callExtract(chatModel, converter, systemPrompt, chunk);
if (result == null) {
continue;
}
persistChunk(kbId, chunk, result, resolved, index);
} catch (Exception e) {
log.warn("[WikiEntity] Extraction failed for chunkId={} kbId={}: {}",
chunk.getId(), kbId, e.getMessage());
}
}
return resolved.size();
}
// ---- LLM call ---------------------------------------------------------
private EntityExtractionResult callExtract(ChatModel chatModel,
BeanOutputConverter<EntityExtractionResult> converter,
String systemPrompt,
WikiChunkEntity chunk) {
String content = chunk.getContent();
if (content.length() > MAX_CHUNK_CHARS) {
content = content.substring(0, MAX_CHUNK_CHARS);
}
String userPrompt = "Source text:\n\"\"\"\n" + content + "\n\"\"\"\n\n" + converter.getFormat();
Prompt prompt = new Prompt(List.of(
new SystemMessage(systemPrompt),
new UserMessage(userPrompt)));
ChatResponse response = chatModel.call(prompt);
String text = response.getResult().getOutput().getText();
if (text == null || text.isBlank()) {
return null;
}
try {
return converter.convert(text);
} catch (Exception e) {
log.debug("[WikiEntity] Structured parse failed, skipping chunk: {}", e.getMessage());
return null;
}
}
private String buildSystemPrompt(List<String> types) {
return "You are a knowledge-graph entity extractor. From the given source text, "
+ "extract named entities and the factual relations between them.\n"
+ "Entity types to use: " + String.join(", ", types) + ".\n"
+ "Rules:\n"
+ "- Only extract entities explicitly named in the text; do not invent any.\n"
+ "- Use the most complete surface form as the name; list shorter forms as aliases.\n"
+ "- For each relation, subject and object must both appear in the entities list.\n"
+ "- Keep predicates short and snake_case (e.g. works_for, located_in, founded).\n"
+ "- Provide a short verbatim evidence quote for each entity and relation.\n"
+ "- If nothing relevant is present, return empty lists.";
}
// ---- persistence ------------------------------------------------------
private void persistChunk(Long kbId, WikiChunkEntity chunk, EntityExtractionResult result,
Map<String, Long> resolved, EntityIndex index) {
Long pageId = firstCitingPage(chunk.getId());
// Resolve each entity to a canonical id, persist its mention for this chunk.
Map<String, Long> localByName = new HashMap<>();
if (result.getEntities() != null) {
for (EntityExtractionResult.ExtractedEntity e : result.getEntities()) {
if (e == null || e.getName() == null || e.getName().isBlank()) {
continue;
}
String type = normalizeType(e.getType());
Long entityId = resolveEntity(kbId, e, type, resolved, index);
if (entityId == null) {
continue;
}
localByName.put(normalize(e.getName()), entityId);
if (e.getAliases() != null) {
for (String alias : e.getAliases()) {
if (alias != null && !alias.isBlank()) {
localByName.put(normalize(alias), entityId);
}
}
}
insertMention(kbId, entityId, chunk.getId(), pageId, e.getName(), e.getEvidence());
bumpMentionCount(entityId);
}
}
// Persist relations whose endpoints both resolved.
if (result.getRelations() != null) {
for (EntityExtractionResult.ExtractedRelation r : result.getRelations()) {
if (r == null || r.getSubject() == null || r.getObject() == null
|| r.getPredicate() == null || r.getPredicate().isBlank()) {
continue;
}
Long subjectId = localByName.get(normalize(r.getSubject()));
Long objectId = localByName.get(normalize(r.getObject()));
if (subjectId == null || objectId == null || subjectId.equals(objectId)) {
continue;
}
upsertRelation(kbId, subjectId, objectId, normalizePredicate(r.getPredicate()),
r.getEvidence(), chunk.getId());
}
}
}
/**
* Resolve an extracted entity to a canonical node id: run cache exact
* key match in DB embedding near-match create new.
*/
private Long resolveEntity(Long kbId, EntityExtractionResult.ExtractedEntity e, String type,
Map<String, Long> resolved, EntityIndex index) {
String key = normalize(e.getName());
String cacheKey = type + "" + key;
Long cached = resolved.get(cacheKey);
if (cached != null) {
return cached;
}
WikiEntityEntity existing = entityMapper.selectOne(new LambdaQueryWrapper<WikiEntityEntity>()
.eq(WikiEntityEntity::getKbId, kbId)
.eq(WikiEntityEntity::getNormalizedKey, key)
.eq(WikiEntityEntity::getType, type)
.last("LIMIT 1"));
if (existing != null) {
resolved.put(cacheKey, existing.getId());
return existing.getId();
}
// Embedding near-duplicate merge across spellings/languages.
float[] vec = embedName(kbId, e);
if (vec != null) {
Long near = index.findNearest(type, vec);
if (near != null) {
resolved.put(cacheKey, near);
return near;
}
}
WikiEntityEntity created = new WikiEntityEntity();
created.setKbId(kbId);
created.setCanonicalName(e.getName().trim());
created.setNormalizedKey(key);
created.setType(type);
created.setAliasesJson(writeJson(e.getAliases()));
created.setDescription(truncate(e.getDescription(), MAX_EVIDENCE));
created.setMentionCount(0);
created.setSalience(BigDecimal.ZERO);
if (vec != null) {
created.setEmbedding(WikiEmbeddingService.floatsToBytes(vec));
}
entityMapper.insert(created);
resolved.put(cacheKey, created.getId());
index.add(type, created.getId(), vec);
return created.getId();
}
private void insertMention(Long kbId, Long entityId, Long chunkId, Long pageId,
String surfaceForm, String evidence) {
WikiEntityMentionEntity m = new WikiEntityMentionEntity();
m.setKbId(kbId);
m.setEntityId(entityId);
m.setChunkId(chunkId);
m.setPageId(pageId);
m.setSurfaceForm(truncate(surfaceForm, 256));
m.setConfidence(BigDecimal.valueOf(0.9));
m.setEvidence(truncate(evidence, MAX_EVIDENCE));
m.setSource("llm-extracted");
mentionMapper.insert(m);
}
private void bumpMentionCount(Long entityId) {
WikiEntityEntity e = entityMapper.selectById(entityId);
if (e == null) {
return;
}
int count = (e.getMentionCount() == null ? 0 : e.getMentionCount()) + 1;
e.setMentionCount(count);
// Saturating popularity score in [0,1): count / (count + 5).
e.setSalience(BigDecimal.valueOf((double) count / (count + 5.0))
.setScale(4, RoundingMode.HALF_UP));
entityMapper.updateById(e);
}
private void upsertRelation(Long kbId, Long subjectId, Long objectId, String predicate,
String evidence, Long chunkId) {
WikiEntityRelationEntity existing = relationMapper.selectOne(
new LambdaQueryWrapper<WikiEntityRelationEntity>()
.eq(WikiEntityRelationEntity::getKbId, kbId)
.eq(WikiEntityRelationEntity::getSubjectEntityId, subjectId)
.eq(WikiEntityRelationEntity::getPredicate, predicate)
.eq(WikiEntityRelationEntity::getObjectEntityId, objectId)
.last("LIMIT 1"));
if (existing != null) {
return;
}
WikiEntityRelationEntity rel = new WikiEntityRelationEntity();
rel.setKbId(kbId);
rel.setSubjectEntityId(subjectId);
rel.setPredicate(predicate);
rel.setObjectEntityId(objectId);
rel.setEvidence(truncate(evidence, MAX_EVIDENCE));
rel.setConfidence(BigDecimal.valueOf(0.8));
rel.setSource("llm-extracted");
rel.setEvidenceChunkId(chunkId);
relationMapper.insert(rel);
}
// ---- helpers ----------------------------------------------------------
private boolean hasMentions(Long chunkId) {
Long count = mentionMapper.selectCount(new LambdaQueryWrapper<WikiEntityMentionEntity>()
.eq(WikiEntityMentionEntity::getChunkId, chunkId));
return count != null && count > 0;
}
private Long firstCitingPage(Long chunkId) {
List<Long> pages = citationMapper.listPageIdsByChunkId(chunkId);
return (pages == null || pages.isEmpty()) ? null : pages.get(0);
}
private float[] embedName(Long kbId, EntityExtractionResult.ExtractedEntity e) {
try {
String text = e.getName() + (e.getDescription() == null ? "" : ". " + e.getDescription());
return embeddingService.embedQuery(kbId, text);
} catch (Exception ex) {
return null;
}
}
private ChatModel resolveChatModel(Long kbId) {
try {
Long modelId = routingService.selectModelId(kbId, "heavy_ingest", WikiJobStep.ENTITY_EXTRACTION);
if (modelId != null) {
return routingService.buildChatModel(modelId);
}
} catch (Exception e) {
log.warn("[WikiEntity] Model routing failed for kbId={}, using default: {}", kbId, e.getMessage());
}
var def = modelConfigService.getDefaultModel();
if (def == null) {
return null;
}
return routingService.buildChatModel(def.getId());
}
private List<String> resolveEntityTypes(WikiKnowledgeBaseEntity kb) {
if (kb.getConfigContent() != null) {
WikiKbConfig config = WikiKbConfigParser.parse(objectMapper, kb.getConfigContent());
if (config != null && config.getEntityTypes() != null && !config.getEntityTypes().isEmpty()) {
return config.getEntityTypes();
}
}
return DEFAULT_ENTITY_TYPES;
}
private String normalize(String s) {
return s == null ? "" : s.trim().toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
}
private String normalizeType(String type) {
String t = type == null ? "" : type.trim().toLowerCase(Locale.ROOT);
return t.isEmpty() ? "other" : t;
}
private String normalizePredicate(String predicate) {
String p = predicate.trim().toLowerCase(Locale.ROOT).replaceAll("\\s+", "_");
return p.length() > 64 ? p.substring(0, 64) : p;
}
private String truncate(String s, int max) {
if (s == null) {
return null;
}
String t = s.trim();
return t.length() > max ? t.substring(0, max) : t;
}
private String writeJson(Object value) {
if (value == null) {
return null;
}
try {
return objectMapper.writeValueAsString(value);
} catch (Exception e) {
return null;
}
}
/**
* In-memory index of same-type entity embeddings for near-duplicate merge
* within a single extraction run. Bounded by the KB's existing entity count.
*/
private final class EntityIndex {
private final Map<String, List<float[]>> vectorsByType = new HashMap<>();
private final Map<String, List<Long>> idsByType = new HashMap<>();
EntityIndex(Long kbId) {
List<WikiEntityEntity> existing = entityMapper.selectList(
new LambdaQueryWrapper<WikiEntityEntity>()
.eq(WikiEntityEntity::getKbId, kbId)
.isNotNull(WikiEntityEntity::getEmbedding));
for (WikiEntityEntity e : existing) {
if (e.getEmbedding() != null) {
add(e.getType(), e.getId(), WikiEmbeddingService.bytesToFloats(e.getEmbedding()));
}
}
}
void add(String type, Long id, float[] vec) {
if (vec == null) {
return;
}
vectorsByType.computeIfAbsent(type, k -> new ArrayList<>()).add(vec);
idsByType.computeIfAbsent(type, k -> new ArrayList<>()).add(id);
}
Long findNearest(String type, float[] vec) {
List<float[]> vectors = vectorsByType.get(type);
List<Long> ids = idsByType.get(type);
if (vectors == null || vectors.isEmpty()) {
return null;
}
float best = MERGE_THRESHOLD;
Long bestId = null;
for (int i = 0; i < vectors.size(); i++) {
float sim = WikiEmbeddingService.cosine(vec, vectors.get(i));
if (sim >= best) {
best = sim;
bestId = ids.get(i);
}
}
return bestId;
}
}
}

View File

@ -0,0 +1,177 @@
package vip.mate.wiki.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import vip.mate.wiki.dto.WikiEntityGraphView;
import vip.mate.wiki.dto.WikiEntityView;
import vip.mate.wiki.model.WikiEntityEntity;
import vip.mate.wiki.model.WikiEntityMentionEntity;
import vip.mate.wiki.model.WikiEntityRelationEntity;
import vip.mate.wiki.model.WikiPageEntity;
import vip.mate.wiki.repository.WikiEntityMapper;
import vip.mate.wiki.repository.WikiEntityMentionMapper;
import vip.mate.wiki.repository.WikiEntityRelationMapper;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
/**
* Read-side queries over the entity-level knowledge graph: entity listing and
* single-entity ego-graph assembly for the wiki graph view.
*
* @author MateClaw Team
*/
@Slf4j
@Service
@RequiredArgsConstructor
public class WikiEntityGraphService {
private final WikiEntityMapper entityMapper;
private final WikiEntityMentionMapper mentionMapper;
private final WikiEntityRelationMapper relationMapper;
private final WikiPageService pageService;
private final ObjectMapper objectMapper;
/** List entities in a KB, optionally filtered by type, ranked by salience. */
public List<WikiEntityView> listEntities(Long kbId, String type, int limit) {
LambdaQueryWrapper<WikiEntityEntity> q = new LambdaQueryWrapper<WikiEntityEntity>()
.eq(WikiEntityEntity::getKbId, kbId)
.orderByDesc(WikiEntityEntity::getSalience)
.last("LIMIT " + Math.max(1, Math.min(limit, 500)));
if (type != null && !type.isBlank()) {
q.eq(WikiEntityEntity::getType, type.trim().toLowerCase());
}
List<WikiEntityView> out = new ArrayList<>();
for (WikiEntityEntity e : entityMapper.selectList(q)) {
out.add(toView(e));
}
return out;
}
/**
* Assemble the whole-KB entity graph: the top entities by salience plus the
* relation edges that connect any two of them.
*/
public WikiEntityGraphView graph(Long kbId, int limit) {
WikiEntityGraphView view = new WikiEntityGraphView();
List<WikiEntityView> nodes = listEntities(kbId, null, limit);
view.setNodes(nodes);
if (nodes.isEmpty()) {
return view;
}
Set<Long> ids = new LinkedHashSet<>();
for (WikiEntityView n : nodes) {
ids.add(n.getId());
}
List<WikiEntityRelationEntity> rels = relationMapper.selectList(
new LambdaQueryWrapper<WikiEntityRelationEntity>()
.eq(WikiEntityRelationEntity::getKbId, kbId)
.last("LIMIT " + Math.max(1, Math.min(limit * 5, 2000))));
for (WikiEntityRelationEntity r : rels) {
if (!ids.contains(r.getSubjectEntityId()) || !ids.contains(r.getObjectEntityId())) {
continue;
}
WikiEntityGraphView.Edge edge = new WikiEntityGraphView.Edge();
edge.setId(r.getId());
edge.setSubjectEntityId(r.getSubjectEntityId());
edge.setPredicate(r.getPredicate());
edge.setObjectEntityId(r.getObjectEntityId());
edge.setEvidence(r.getEvidence());
edge.setConfidence(r.getConfidence());
view.getEdges().add(edge);
}
return view;
}
/** Assemble the ego-graph around one entity. */
public WikiEntityGraphView ego(Long kbId, Long entityId, int limit) {
WikiEntityGraphView view = new WikiEntityGraphView();
WikiEntityEntity center = entityMapper.selectById(entityId);
if (center == null || !center.getKbId().equals(kbId)) {
return view;
}
view.setCenter(toView(center));
int edgeLimit = Math.max(1, Math.min(limit, 200));
List<WikiEntityRelationEntity> edges = relationMapper.selectList(
new LambdaQueryWrapper<WikiEntityRelationEntity>()
.eq(WikiEntityRelationEntity::getKbId, kbId)
.and(w -> w.eq(WikiEntityRelationEntity::getSubjectEntityId, entityId)
.or().eq(WikiEntityRelationEntity::getObjectEntityId, entityId))
.last("LIMIT " + edgeLimit));
Set<Long> neighborIds = new LinkedHashSet<>();
for (WikiEntityRelationEntity r : edges) {
WikiEntityGraphView.Edge edge = new WikiEntityGraphView.Edge();
edge.setId(r.getId());
edge.setSubjectEntityId(r.getSubjectEntityId());
edge.setPredicate(r.getPredicate());
edge.setObjectEntityId(r.getObjectEntityId());
edge.setEvidence(r.getEvidence());
edge.setConfidence(r.getConfidence());
view.getEdges().add(edge);
neighborIds.add(r.getSubjectEntityId());
neighborIds.add(r.getObjectEntityId());
}
neighborIds.remove(entityId);
if (!neighborIds.isEmpty()) {
for (WikiEntityEntity n : entityMapper.selectBatchIds(neighborIds)) {
view.getNodes().add(toView(n));
}
}
// Pages that mention the center entity the bridge to the page layer.
List<WikiEntityMentionEntity> mentions = mentionMapper.selectList(
new LambdaQueryWrapper<WikiEntityMentionEntity>()
.eq(WikiEntityMentionEntity::getEntityId, entityId)
.isNotNull(WikiEntityMentionEntity::getPageId)
.last("LIMIT 200"));
Set<Long> pageIds = new LinkedHashSet<>();
for (WikiEntityMentionEntity m : mentions) {
pageIds.add(m.getPageId());
}
for (Long pageId : pageIds) {
WikiPageEntity page = pageService.getById(pageId);
if (page == null) {
continue;
}
WikiEntityGraphView.PageRef ref = new WikiEntityGraphView.PageRef();
ref.setPageId(page.getId());
ref.setSlug(page.getSlug());
ref.setTitle(page.getTitle());
view.getPages().add(ref);
}
return view;
}
private WikiEntityView toView(WikiEntityEntity e) {
WikiEntityView v = new WikiEntityView();
v.setId(e.getId());
v.setKbId(e.getKbId());
v.setCanonicalName(e.getCanonicalName());
v.setType(e.getType());
v.setDescription(e.getDescription());
v.setSalience(e.getSalience());
v.setMentionCount(e.getMentionCount());
v.setAliases(parseAliases(e.getAliasesJson()));
return v;
}
private List<String> parseAliases(String json) {
if (json == null || json.isBlank()) {
return Collections.emptyList();
}
try {
return objectMapper.readValue(json, new TypeReference<List<String>>() {});
} catch (Exception e) {
return Collections.emptyList();
}
}
}

View File

@ -1,7 +1,9 @@
package vip.mate.wiki.service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -10,24 +12,49 @@ import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.annotation.Lazy;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.stereotype.Service;
import vip.mate.agent.AgentGraphBuilder;
import vip.mate.agent.prompt.PromptLoader;
import vip.mate.llm.failover.ProviderHealthTracker;
import vip.mate.llm.model.ModelConfigEntity;
import vip.mate.llm.service.ModelConfigService;
import vip.mate.llm.service.ModelProviderService;
import vip.mate.wiki.WikiProperties;
import vip.mate.wiki.dto.RouteResult;
import vip.mate.wiki.dto.RoutedPageMeta;
import vip.mate.wiki.dto.WikiChunkDraft;
import vip.mate.wiki.event.WikiFactPageUpdatedEvent;
import vip.mate.wiki.event.WikiKbDirtyEvent;
import vip.mate.wiki.event.WikiPageCreatedEvent;
import vip.mate.wiki.event.WikiProcessingEvent;
import vip.mate.wiki.job.WikiEmbeddingProviderFailingException;
import vip.mate.wiki.job.WikiJobStage;
import vip.mate.wiki.job.WikiJobStep;
import vip.mate.wiki.job.WikiKbConfig;
import vip.mate.wiki.job.WikiKbConfigParser;
import vip.mate.wiki.job.WikiModelRoutingService;
import vip.mate.wiki.job.WikiProcessingJobService;
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
import vip.mate.wiki.model.WikiPageEntity;
import vip.mate.wiki.model.WikiRawMaterialEntity;
import vip.mate.wiki.profile.WikiMetadataValidator;
import vip.mate.wiki.profile.WikiPageTypeDef;
import vip.mate.wiki.profile.WikiPageTypeProfile;
import vip.mate.wiki.profile.WikiPageTypeProfileService;
import vip.mate.wiki.sse.WikiProgressBus;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
@ -60,23 +87,24 @@ public class WikiProcessingService {
private final ObjectMapper objectMapper;
private final WikiProgressBus progressBus;
private final WikiCitationService citationService;
private final org.springframework.context.ApplicationEventPublisher eventPublisher;
private final ApplicationEventPublisher eventPublisher;
private final WikiEntityExtractionService entityExtractionService;
/**
* Optional KB pageType profile. Field-injected (not a constructor arg) so
* existing instantiations are unaffected; when absent the batch-create
* prompt falls back to the legacy hardcoded pageType enum.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
private vip.mate.wiki.profile.WikiPageTypeProfileService pageTypeProfileService;
@Autowired(required = false)
private WikiPageTypeProfileService pageTypeProfileService;
/** Optional metadata validator, paired with {@link #pageTypeProfileService}. */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private vip.mate.wiki.profile.WikiMetadataValidator metadataValidator;
@Autowired(required = false)
private WikiMetadataValidator metadataValidator;
/** Optional dependency/stale engine for layered-knowledge wiring. */
@org.springframework.beans.factory.annotation.Autowired(required = false)
private vip.mate.wiki.service.WikiDependencyService dependencyService;
@Autowired(required = false)
private WikiDependencyService dependencyService;
/**
* Read-the-failover-chain handle. Optional so the existing constructors and
@ -84,8 +112,8 @@ public class WikiProcessingService {
* fallback hop iterates {@code listEnabledModels} in DB order same
* behavior as before this PR.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
private vip.mate.llm.service.ModelProviderService modelProviderService;
@Autowired(required = false)
private ModelProviderService modelProviderService;
/**
* Per-provider failure counter / cooldown bookkeeping. Optional for the
@ -94,12 +122,12 @@ public class WikiProcessingService {
* successful call we clear the failure counter for the provider that
* actually responded.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
private vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker;
@Autowired(required = false)
private ProviderHealthTracker providerHealthTracker;
@org.springframework.beans.factory.annotation.Autowired(required = false)
@org.springframework.context.annotation.Lazy
private vip.mate.wiki.job.WikiProcessingJobService wikiJobService;
@Autowired(required = false)
@Lazy
private WikiProcessingJobService wikiJobService;
/**
* RFC-051 PR-1c: optional preprocessor that fills chunk metadata
@ -107,7 +135,7 @@ public class WikiProcessingService {
* Marked optional so unit tests that construct this service directly
* (without Spring) can opt out without exploding.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
@Autowired(required = false)
private DocumentPreprocessService preprocessService;
/**
@ -115,7 +143,7 @@ public class WikiProcessingService {
* the KB before each ingest. Optional so the older lazy-only unit tests
* don't need to wire it.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
@Autowired(required = false)
private WikiScaffoldService scaffoldService;
/**
@ -124,7 +152,7 @@ public class WikiProcessingService {
* without Spring don't need to supply it when absent, the post-ingestion
* auto-scan is simply skipped.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
@Autowired(required = false)
private WikiLintJobService lintJobService;
/**
@ -133,14 +161,14 @@ public class WikiProcessingService {
* routing chain (stepModels[step] -&gt; wikiDefaultModelId -&gt; system
* default) for a model rather than always pulling the system default.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
private vip.mate.wiki.job.WikiModelRoutingService modelRoutingService;
@Autowired(required = false)
private WikiModelRoutingService modelRoutingService;
/** RFC-051 PR-2b/2c: optional overview rebuilder + log appender. */
@org.springframework.beans.factory.annotation.Autowired(required = false)
@Autowired(required = false)
private WikiOverviewService overviewService;
@org.springframework.beans.factory.annotation.Autowired(required = false)
@Autowired(required = false)
private WikiLogService logService;
/**
@ -148,8 +176,8 @@ public class WikiProcessingService {
* of the KB's apply-default transformation templates. Missing in the
* legacy unit tests that wire this service directly.
*/
@org.springframework.beans.factory.annotation.Autowired(required = false)
@org.springframework.context.annotation.Lazy
@Autowired(required = false)
@Lazy
private WikiTransformationExecutor transformationExecutor;
/** Parallel chunk / material processing executor (JDK 21 virtual threads) */
@ -203,7 +231,7 @@ public class WikiProcessingService {
/** KBs with a reclassify pass currently running, used to reject concurrent
* re-triggers (which would double LLM spend and race page-type writes). */
private final java.util.Set<Long> reclassifyInFlight = ConcurrentHashMap.newKeySet();
private final Set<Long> reclassifyInFlight = ConcurrentHashMap.newKeySet();
/**
* Process one raw material.
@ -272,7 +300,7 @@ public class WikiProcessingService {
try {
var job = wikiJobService.createHeavyIngest(kb.getId(), rawId);
jobId = job.getId();
wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.ROUTING);
wikiJobService.transition(jobId, WikiJobStage.ROUTING);
} catch (Exception e) {
log.warn("[Wiki] Failed to create heavy ingest job record for raw={}: {}", rawId, e.getMessage());
}
@ -284,7 +312,7 @@ public class WikiProcessingService {
// RFC-012 M3广播 raw.started前端切到 indeterminate 进度条
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_STARTED,
java.util.Map.of("rawId", rawId, "phase", "route"));
Map.of("rawId", rawId, "phase", "route"));
try {
// Phase 1: 获取文本内容
@ -323,7 +351,7 @@ public class WikiProcessingService {
// Transition job to phase_a (chunk processing begins)
if (wikiJobService != null && jobId != null) {
try { wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.PHASE_A_RUNNING); } catch (Exception ignored) {}
try { wikiJobService.transition(jobId, WikiJobStage.PHASE_A_RUNNING); } catch (Exception ignored) {}
}
// Phase 3: LLM 消化
@ -416,10 +444,10 @@ public class WikiProcessingService {
// RFC-012 M3广播终态
if ("failed".equals(finalStatus)) {
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED,
java.util.Map.of("rawId", rawId, "error", finalDetail == null ? "" : finalDetail));
Map.of("rawId", rawId, "error", finalDetail == null ? "" : finalDetail));
} else {
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED,
java.util.Map.of(
Map.of(
"rawId", rawId,
"status", finalStatus,
"totalPages", totalPages,
@ -430,10 +458,10 @@ public class WikiProcessingService {
if (wikiJobService != null && jobId != null) {
try {
var terminalStage = switch (finalStatus) {
case "failed" -> vip.mate.wiki.job.WikiJobStage.FAILED;
case "partial" -> vip.mate.wiki.job.WikiJobStage.PARTIAL;
case "cancelled" -> vip.mate.wiki.job.WikiJobStage.CANCELLED;
default -> vip.mate.wiki.job.WikiJobStage.COMPLETED;
case "failed" -> WikiJobStage.FAILED;
case "partial" -> WikiJobStage.PARTIAL;
case "cancelled" -> WikiJobStage.CANCELLED;
default -> WikiJobStage.COMPLETED;
};
wikiJobService.transition(jobId, terminalStage);
} catch (Exception ignored) {}
@ -460,7 +488,7 @@ public class WikiProcessingService {
// schedule (debounced) an LLM-generated overview narrative refresh.
// Stats rebuild above is sync; narrative regen runs after-commit.
if (nonTerminalSideEffects) {
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kb.getId()));
eventPublisher.publishEvent(new WikiKbDirtyEvent(this, kb.getId()));
}
// Run apply-default transformation templates against the newly
@ -495,7 +523,7 @@ public class WikiProcessingService {
if (embedded > 0) {
log.info("[Wiki] Async embedding completed: kbId={}, embedded={}", fKbId, embedded);
}
} catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) {
} catch (WikiEmbeddingProviderFailingException ex) {
// Circuit-breaker tripped the provider has consistently failed.
// The exception's own log line in WikiEmbeddingService is enough;
// emit a calmer notice here instead of a generic failure log.
@ -507,6 +535,26 @@ public class WikiProcessingService {
});
}
// Entity-level knowledge graph extraction opt-in per KB. Runs as a
// separate async pass so it never blocks (or fails) the ingest pipeline;
// its inputs (chunks, citations) are already committed at this point.
if (totalChunks > 0 && !"cancelled".equals(finalStatus)
&& isEntityExtractionEnabled(kb)) {
final Long fKbId = kb.getId();
final Long fRawId = rawId;
WIKI_EXECUTOR.submit(() -> {
try {
int count = entityExtractionService.extractForRaw(fKbId, fRawId);
if (count > 0) {
log.info("[Wiki] Async entity extraction completed: kbId={}, rawId={}, entities={}",
fKbId, fRawId, count);
}
} catch (Exception ex) {
log.warn("[Wiki] Async entity extraction failed for kbId={}: {}", fKbId, ex.getMessage());
}
});
}
} catch (Exception e) {
// If the user requested cancellation while this run was in flight,
// surface the abort as 'cancelled' rather than 'failed' even when
@ -527,8 +575,8 @@ public class WikiProcessingService {
if (wikiJobService != null && jobId != null) {
try {
wikiJobService.transition(jobId, cancelled
? vip.mate.wiki.job.WikiJobStage.CANCELLED
: vip.mate.wiki.job.WikiJobStage.FAILED);
? WikiJobStage.CANCELLED
: WikiJobStage.FAILED);
} catch (Exception ignored) {}
}
// Broadcast: cancelled rows reuse the COMPLETED event with status="cancelled"
@ -536,10 +584,10 @@ public class WikiProcessingService {
// go through RAW_FAILED (which the UI surfaces as a red banner).
if (cancelled) {
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED,
java.util.Map.of("rawId", rawId, "status", "cancelled"));
Map.of("rawId", rawId, "status", "cancelled"));
} else {
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED,
java.util.Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage()));
Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage()));
}
} finally {
// RFC-012 M2 v2 UI v2写入最终进度并清理共享计数器
@ -665,7 +713,7 @@ public class WikiProcessingService {
if (modelId != null && modelRoutingService != null) {
chatModel = modelRoutingService.buildChatModel(modelId);
} else {
chatModel = resolveChatModel(kbId, vip.mate.wiki.job.WikiJobStep.CREATE_PAGE).chatModel;
chatModel = resolveChatModel(kbId, WikiJobStep.CREATE_PAGE).chatModel;
}
systemPrompt = PromptLoader.loadPrompt("wiki/classify-page-system")
.replace("{allowed_page_types}", pageTypeProfileService.describeForPrompt(kbId));
@ -714,13 +762,13 @@ public class WikiProcessingService {
page.getId(), kbId, e.getMessage());
} finally {
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
java.util.Map.of("kind", "reclassify", "done", done, "total", total));
Map.of("kind", "reclassify", "done", done, "total", total));
}
}
log.info("[Wiki] reclassifyKB done kbId={} pages={} changed={} failed={}",
kbId, total, changed, failed);
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED,
java.util.Map.of("kind", "reclassify", "done", total, "total", total,
Map.of("kind", "reclassify", "done", total, "total", total,
"changed", changed, "failed", failed));
} finally {
reclassifyInFlight.remove(kbId);
@ -950,7 +998,7 @@ public class WikiProcessingService {
));
if (isAborted(raw.getId(), "single-chunk legacy")) return 0;
String llmResponse = callLlmWithResilientRetry(prompt, "chunk of raw=" + raw.getId(),
kb.getId(), vip.mate.wiki.job.WikiJobStep.CREATE_PAGE);
kb.getId(), WikiJobStep.CREATE_PAGE);
return applyLlmResponse(kb.getId(), raw.getId(), llmResponse);
}
@ -1004,9 +1052,9 @@ public class WikiProcessingService {
// LLM produces strict RouteResult JSON. KB config wins; falls back to global
// mate.wiki.use-structured-route default when the KB hasn't expressed a preference.
boolean useStructured = resolveStructuredRouteFlag(kb);
org.springframework.ai.converter.BeanOutputConverter<vip.mate.wiki.dto.RouteResult> routeConverter =
BeanOutputConverter<RouteResult> routeConverter =
useStructured
? new org.springframework.ai.converter.BeanOutputConverter<>(vip.mate.wiki.dto.RouteResult.class)
? new BeanOutputConverter<>(RouteResult.class)
: null;
if (routeConverter != null) {
routeUser = routeUser + "\n\n" + routeConverter.getFormat();
@ -1018,7 +1066,7 @@ public class WikiProcessingService {
));
if (isAborted(rawId, "route phase")) return 0;
String routeResponse = callLlmWithResilientRetry(routePrompt, "route chunk of raw=" + rawId,
kbId, vip.mate.wiki.job.WikiJobStep.ROUTE);
kbId, WikiJobStep.ROUTE);
// RFC-012 follow-up #3phase B 现在并行执行计数必须是 atomic
AtomicInteger created = new AtomicInteger(0);
@ -1031,12 +1079,12 @@ public class WikiProcessingService {
boolean structuredOk = false;
if (routeConverter != null) {
try {
vip.mate.wiki.dto.RouteResult bound = routeConverter.convert(routeResponse);
RouteResult bound = routeConverter.convert(routeResponse);
if (bound != null) {
for (vip.mate.wiki.dto.RoutedPageMeta meta : bound.create()) {
for (RoutedPageMeta meta : bound.create()) {
if (meta == null || meta.slug() == null || meta.slug().isBlank()
|| meta.title() == null || meta.title().isBlank()) continue;
com.fasterxml.jackson.databind.node.ObjectNode node = objectMapper.createObjectNode();
ObjectNode node = objectMapper.createObjectNode();
node.put("slug", meta.slug());
node.put("title", meta.title());
if (meta.summary() != null) node.put("summary", meta.summary());
@ -1079,7 +1127,7 @@ public class WikiProcessingService {
));
String retryResponse = callLlmWithResilientRetry(retryPrompt,
"route chunk RETRY of raw=" + rawId,
kbId, vip.mate.wiki.job.WikiJobStep.ROUTE);
kbId, WikiJobStep.ROUTE);
routeJson = parseJsonResponse(retryResponse);
if (routeJson == null) {
log.warn("[Wiki] Route phase: failed to parse JSON for kbId={}, rawId={}, responseLen={}, first200={}",
@ -1112,7 +1160,7 @@ public class WikiProcessingService {
// so no content is silently dropped (mirrors llm_wiki source-summary guarantee).
if (totalPlanned == 0 && textContent.length() >= properties.getChunkFallbackMinChars()) {
String overviewSlug = WikiPageService.toSlug(rawTitle) + "-overview";
com.fasterxml.jackson.databind.node.ObjectNode fallbackMeta =
ObjectNode fallbackMeta =
objectMapper.createObjectNode();
fallbackMeta.put("slug", overviewSlug);
fallbackMeta.put("title", rawTitle + " 概述");
@ -1133,7 +1181,7 @@ public class WikiProcessingService {
log.info("[Wiki] Progress: switching to phase-b for raw={}", rawId);
// RFC-012 M3route 完成phase-b 启动 通知前端确定进度可显示 0/N
progressBus.broadcast(kbId, WikiProgressBus.EVENT_ROUTE_DONE,
java.util.Map.of(
Map.of(
"rawId", rawId,
"phase", "phase-b",
"done", pc.done.get(),
@ -1195,7 +1243,7 @@ public class WikiProcessingService {
if (!ok) pc.failed.incrementAndGet();
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
java.util.Map.of(
Map.of(
"rawId", rawId,
"kind", "merge",
"ok", ok,
@ -1288,7 +1336,7 @@ public class WikiProcessingService {
String batchResponse = callLlmWithResilientRetry(batchPrompt,
"batch-create " + subBatch.size() + " pages of raw=" + rawId
+ " subBatch=" + (bStart / batchSize + 1),
kbId, vip.mate.wiki.job.WikiJobStep.CREATE_PAGE);
kbId, WikiJobStep.CREATE_PAGE);
List<WikiBatchCreateParser.ParsedPage> parsedPages = batchParser.parse(batchResponse);
@ -1328,7 +1376,7 @@ public class WikiProcessingService {
pc.failed.incrementAndGet();
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
java.util.Map.of("rawId", rawId, "kind", "create",
Map.of("rawId", rawId, "kind", "create",
"ok", false, "done", d, "total", pc.total.get()));
}
continue;
@ -1377,7 +1425,7 @@ public class WikiProcessingService {
pc.failed.incrementAndGet();
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
java.util.Map.of("rawId", rawId, "kind", "create-retry",
Map.of("rawId", rawId, "kind", "create-retry",
"ok", false, "done", d, "total", pc.total.get()));
}
continue;
@ -1422,7 +1470,7 @@ public class WikiProcessingService {
if (!ok) pc.failed.incrementAndGet();
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
java.util.Map.of(
Map.of(
"rawId", rawId,
"kind", "create",
"ok", ok,
@ -1433,7 +1481,7 @@ public class WikiProcessingService {
// Retry any pages that LLM omitted from the batch response
int subBatchNum = bStart / batchSize + 1;
java.util.Set<String> returnedSlugs = new java.util.HashSet<>();
Set<String> returnedSlugs = new HashSet<>();
for (WikiBatchCreateParser.ParsedPage pp : parsedPages) {
returnedSlugs.add(pp.slug());
}
@ -1481,7 +1529,7 @@ public class WikiProcessingService {
));
if (isAborted(raw.getId(), "retry-create slug=" + slug)) return null;
return callLlmWithResilientRetry(prompt, "retry-create slug=" + slug + " of raw=" + raw.getId(),
kb.getId(), vip.mate.wiki.job.WikiJobStep.CREATE_PAGE);
kb.getId(), WikiJobStep.CREATE_PAGE);
}
/**
@ -1572,7 +1620,7 @@ public class WikiProcessingService {
if (delta < 0) pc.failed.incrementAndGet();
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
java.util.Map.of("rawId", rawId, "kind", "create-retry",
Map.of("rawId", rawId, "kind", "create-retry",
"ok", delta >= 0, "done", d, "total", pc.total.get()));
}
return delta;
@ -1723,7 +1771,7 @@ public class WikiProcessingService {
log.info("[Wiki] Phase B create page slug='{}' done (created)", slug);
citationService.buildCitationsAsync(created.getId(), kbId);
return true;
} catch (org.springframework.dao.DuplicateKeyException e) {
} catch (DuplicateKeyException e) {
// Fallback 2: concurrent INSERT race degrade to update
pageService.updatePageByAi(kbId, slug, content, pageSummary, rawId);
WikiPageEntity raced = pageService.getBySlug(kbId, slug);
@ -1756,13 +1804,13 @@ public class WikiProcessingService {
// transactions), so the count is accurate. Idempotent + dedup-guarded
// downstream, so firing on update paths is safe.
if (eventPublisher != null && pageType != null && !pageType.isBlank()) {
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiPageCreatedEvent(kbId, pageType, pageId));
eventPublisher.publishEvent(new WikiPageCreatedEvent(kbId, pageType, pageId));
}
// When an existing fact page is updated, propagate staleness to the
// experience pages depending on it (async, off the ingest thread).
if (isUpdate && eventPublisher != null && dependencyService != null
&& pageTypeProfileService != null && !pageTypeProfileService.isExperience(kbId, pageType)) {
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiFactPageUpdatedEvent(
eventPublisher.publishEvent(new WikiFactPageUpdatedEvent(
kbId, pageId, "fact page updated during ingest"));
}
}
@ -1792,7 +1840,7 @@ public class WikiProcessingService {
if (!pageTypeProfileService.isExperience(kbId, pageType)) {
return; // only experience pages declare fact dependencies
}
java.util.List<Long> depIds = new java.util.ArrayList<>();
List<Long> depIds = new ArrayList<>();
for (JsonNode n : dependsOnNode) {
String slug = n.asText("");
if (slug.isBlank()) continue;
@ -1802,7 +1850,7 @@ public class WikiProcessingService {
}
}
try {
java.util.List<String> rejected = dependencyService.setDependencies(kbId, pageId, depIds);
List<String> rejected = dependencyService.setDependencies(kbId, pageId, depIds);
if (!rejected.isEmpty()) {
log.warn("[Wiki] page {} dependency warnings: {}", pageId, rejected);
}
@ -1853,11 +1901,11 @@ public class WikiProcessingService {
return;
}
try {
vip.mate.wiki.profile.WikiPageTypeProfile profile = pageTypeProfileService.resolveProfile(kbId);
vip.mate.wiki.profile.WikiPageTypeDef def = profile.get(pageType);
WikiPageTypeProfile profile = pageTypeProfileService.resolveProfile(kbId);
WikiPageTypeDef def = profile.get(pageType);
@SuppressWarnings("unchecked")
java.util.Map<String, Object> raw = objectMapper.convertValue(metadataNode, java.util.Map.class);
vip.mate.wiki.profile.WikiMetadataValidator.ValidationResult result =
Map<String, Object> raw = objectMapper.convertValue(metadataNode, Map.class);
WikiMetadataValidator.ValidationResult result =
metadataValidator.validate(def, raw, profile.isAllowAdditionalFields(), "create");
String metadataJson = objectMapper.writeValueAsString(result.getCleaned());
String validationJson = result.getWarnings().isEmpty()
@ -1925,7 +1973,7 @@ public class WikiProcessingService {
if (isAborted(rawId, "merge slug=" + slug)) return false;
String response = callLlmWithResilientRetry(prompt,
"merge page slug=" + slug + " of raw=" + rawId,
kbId, vip.mate.wiki.job.WikiJobStep.MERGE_PAGE);
kbId, WikiJobStep.MERGE_PAGE);
JsonNode mergeJson = parseJsonResponse(response);
if (mergeJson == null) {
log.warn("[Wiki] Phase B merge page slug='{}' returned unparseable JSON, skipping", slug);
@ -2053,7 +2101,7 @@ public class WikiProcessingService {
try {
if (isAborted(raw.getId(), "doc analysis")) return "";
String response = callLlmWithResilientRetry(prompt, "analyze doc raw=" + raw.getId(),
kb.getId(), vip.mate.wiki.job.WikiJobStep.ROUTE);
kb.getId(), WikiJobStep.ROUTE);
JsonNode json = parseJsonResponse(response);
if (json != null) {
// Validate related_pages against the active KB slug set BEFORE
@ -2120,7 +2168,7 @@ public class WikiProcessingService {
JsonNode relatedNode = analysisJson.path("related_pages");
if (!relatedNode.isArray() || relatedNode.size() == 0) return analysisJson;
java.util.Set<String> activeSlugs;
Set<String> activeSlugs;
try {
activeSlugs = linkService.lowercaseSlugSet(pageService.listSummaries(kbId));
} catch (RuntimeException e) {
@ -2134,12 +2182,12 @@ public class WikiProcessingService {
return result;
}
com.fasterxml.jackson.databind.node.ArrayNode keptArray = objectMapper.createArrayNode();
java.util.List<String> dropped = new java.util.ArrayList<>();
ArrayNode keptArray = objectMapper.createArrayNode();
List<String> dropped = new ArrayList<>();
for (JsonNode el : relatedNode) {
String slug = el.asText("").trim();
if (slug.isEmpty()) continue;
if (activeSlugs.contains(slug.toLowerCase(java.util.Locale.ROOT))) {
if (activeSlugs.contains(slug.toLowerCase(Locale.ROOT))) {
keptArray.add(slug);
} else {
dropped.add(slug);
@ -2257,7 +2305,7 @@ public class WikiProcessingService {
* is available. Falls back to the system default on any lookup failure
* so a misconfigured KB never blocks ingest.
*/
private ChatModel buildChatModelFor(Long kbId, vip.mate.wiki.job.WikiJobStep step) {
private ChatModel buildChatModelFor(Long kbId, WikiJobStep step) {
if (modelRoutingService != null && kbId != null && step != null) {
try {
Long modelId = modelRoutingService.selectModelId(kbId, "heavy_ingest", step);
@ -2295,7 +2343,7 @@ public class WikiProcessingService {
* pick the routed chat model; passing {@code null} for either reproduces
* the legacy behavior (system default model).
*/
private String callLlmWithResilientRetry(Prompt prompt, String ctx, Long kbId, vip.mate.wiki.job.WikiJobStep step) {
private String callLlmWithResilientRetry(Prompt prompt, String ctx, Long kbId, WikiJobStep step) {
long backoffMs = 1000;
final long maxBackoffMs = 60_000;
final int maxAttempts = Math.max(1, properties.getLlmMaxAttempts());
@ -2434,7 +2482,7 @@ public class WikiProcessingService {
/** Pair of modelId + built ChatModel — null modelId means we used the system default. */
private record ResolvedChatModel(Long modelId, ChatModel chatModel) {}
private ResolvedChatModel resolveChatModel(Long kbId, vip.mate.wiki.job.WikiJobStep step) {
private ResolvedChatModel resolveChatModel(Long kbId, WikiJobStep step) {
if (modelRoutingService != null && kbId != null && step != null) {
try {
Long modelId = modelRoutingService.selectModelId(kbId, "heavy_ingest", step);
@ -2466,7 +2514,7 @@ public class WikiProcessingService {
* stable, never random.
*
* <p>Skips providers currently in cooldown ({@link
* vip.mate.llm.failover.ProviderHealthTracker}) so a flapping provider
* ProviderHealthTracker}) so a flapping provider
* doesn't keep getting tried while we wait for it to recover.
*/
private ResolvedChatModel pickFallbackChatModel(Long failedModelId) {
@ -2758,8 +2806,8 @@ public class WikiProcessingService {
));
if (isAborted(raw.getId(), "repair page=" + page.getSlug())) return;
String response = callLlmWithResilientRetry(prompt, "repair page=" + page.getSlug(),
kb.getId(), vip.mate.wiki.job.WikiJobStep.MERGE_PAGE);
com.fasterxml.jackson.databind.JsonNode pageJson = parseJsonResponse(response);
kb.getId(), WikiJobStep.MERGE_PAGE);
JsonNode pageJson = parseJsonResponse(response);
if (pageJson == null) return;
String content = pageJson.path("content").asText("");
@ -2773,7 +2821,7 @@ public class WikiProcessingService {
private List<Long> parseSourceRawIds(String json) {
if (json == null || json.isBlank()) return List.of();
try {
return objectMapper.readValue(json, new com.fasterxml.jackson.core.type.TypeReference<List<Long>>() {});
return objectMapper.readValue(json, new TypeReference<List<Long>>() {});
} catch (Exception e) {
return List.of();
}
@ -2843,6 +2891,17 @@ public class WikiProcessingService {
return config != null ? config.getIngestMode() : null;
}
/**
* Read the {@code entityExtractionEnabled} opt-in from KB config. Defaults
* to {@code false} on any parse error or missing field extraction is an
* opt-in cost and must never be turned on implicitly.
*/
private boolean isEntityExtractionEnabled(WikiKnowledgeBaseEntity kb) {
if (kb == null || kb.getConfigContent() == null) return false;
WikiKbConfig config = WikiKbConfigParser.parse(objectMapper, kb.getConfigContent());
return config != null && Boolean.TRUE.equals(config.getEntityExtractionEnabled());
}
/**
* Returns {@code true} when the caller should bail out of an in-flight
* processing path because the raw material has been deleted.
@ -2911,7 +2970,7 @@ public class WikiProcessingService {
rawService.updateProgress(rawId, "lazy", 0, 0);
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_STARTED,
java.util.Map.of("rawId", rawId, "phase", "lazy"));
Map.of("rawId", rawId, "phase", "lazy"));
try {
String textContent = rawService.getTextContent(raw);
@ -2919,7 +2978,7 @@ public class WikiProcessingService {
rawService.updateProcessingStatus(rawId, "failed", "No text content available");
kbService.updateStatus(kbId, "active");
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED,
java.util.Map.of("rawId", rawId, "error", "No text content available"));
Map.of("rawId", rawId, "error", "No text content available"));
return;
}
@ -2957,7 +3016,7 @@ public class WikiProcessingService {
if (embedded > 0) {
log.info("[Wiki] Lazy async embedding completed: kbId={}, embedded={}", fKbId, embedded);
}
} catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) {
} catch (WikiEmbeddingProviderFailingException ex) {
log.warn("[Wiki] Lazy async embedding aborted by circuit-breaker for kbId={}: {}",
fKbId, ex.getMessage());
} catch (Exception ex) {
@ -2975,7 +3034,7 @@ public class WikiProcessingService {
kbService.updateStatus(kbId, "active");
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED,
java.util.Map.of(
Map.of(
"rawId", rawId,
"status", "completed",
"totalPages", 0,
@ -2993,7 +3052,7 @@ public class WikiProcessingService {
// RFC-051 PR-2b: refresh overview stats.
if (overviewService != null) overviewService.rebuild(kbId);
// Tier 2: dirty event drives the LLM-narrated overview section.
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kbId));
eventPublisher.publishEvent(new WikiKbDirtyEvent(this, kbId));
log.info("[Wiki] Lazy processing completed for raw={}, kbId={}, chunks={}",
rawId, kbId, totalChunks);
@ -3002,7 +3061,7 @@ public class WikiProcessingService {
rawService.updateProcessingStatus(rawId, "failed", e.getMessage());
kbService.updateStatus(kbId, "active");
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED,
java.util.Map.of("rawId", rawId,
Map.of("rawId", rawId,
"error", e.getMessage() == null ? "unknown" : e.getMessage()));
}
}

View File

@ -0,0 +1,56 @@
-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks.
--
-- Distinct from mate_wiki_page (document/topic granularity) — a row here is a
-- mention-granularity named entity (person, organization, location, event, ...)
-- resolved and de-duplicated across the knowledge base. The entity layer sits
-- beneath the page layer: entities are linked to their source chunks (and, via
-- citing pages, to wiki pages) through mate_wiki_entity_mention, and to each
-- other through mate_wiki_entity_relation.
--
-- canonical_name display name chosen for the merged entity
-- normalized_key case/whitespace-folded key used for exact-match dedup
-- type entity taxonomy: person | organization | location |
-- event | product | concept | other
-- aliases_json JSON array of surface forms merged into this entity
-- description one-line summary synthesized from the mentions
-- salience 0..1 importance score (mention frequency / distribution)
-- mention_count number of mentions resolved to this entity
-- embedding float32 little-endian name/description vector used for
-- near-duplicate merge across spellings/languages
-- computed_hash fingerprint of the inputs that produced this row
CREATE TABLE IF NOT EXISTS mate_wiki_entity (
id BIGINT PRIMARY KEY,
kb_id BIGINT NOT NULL,
canonical_name VARCHAR(256) NOT NULL,
normalized_key VARCHAR(256) NOT NULL,
type VARCHAR(32) NOT NULL,
aliases_json CLOB,
description CLOB,
salience DECIMAL(5, 4),
mention_count INT NOT NULL DEFAULT 0,
embedding BLOB,
embedding_model VARCHAR(64),
computed_hash VARCHAR(64),
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted INT NOT NULL DEFAULT 0
);
-- Exact-match dedup identity within a KB (deleted included so soft-deleted
-- rows can coexist with re-inserted ones during re-extract cycles).
CREATE UNIQUE INDEX IF NOT EXISTS uk_we_key
ON mate_wiki_entity (kb_id, normalized_key, type, deleted);
-- KB-wide listing and "top entities by salience".
CREATE INDEX IF NOT EXISTS idx_we_kb
ON mate_wiki_entity (kb_id, deleted);
CREATE INDEX IF NOT EXISTS idx_we_salience
ON mate_wiki_entity (kb_id, salience DESC);
CREATE INDEX IF NOT EXISTS idx_we_type
ON mate_wiki_entity (kb_id, type, deleted);

View File

@ -0,0 +1,38 @@
-- mate_wiki_entity_mention: links a canonical entity to a source occurrence.
--
-- One row per (entity, chunk) occurrence. page_id is back-filled from the
-- chunk's citing pages so the entity layer connects to the page layer:
-- entity -> mention -> chunk -> citing page. A NULL page_id means the source
-- chunk is not yet cited by any generated page.
--
-- entity_id the resolved canonical entity (mate_wiki_entity.id)
-- chunk_id source chunk the mention was found in
-- page_id a wiki page that cites that chunk, when known
-- surface_form the exact text as it appeared in the source
-- char_offset character offset of the mention within the chunk, when known
-- confidence 0..1 extraction confidence
-- evidence short surrounding quote (<= 500 chars enforced in Java)
-- source provenance tag: llm-extracted | manual
CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention (
id BIGINT PRIMARY KEY,
kb_id BIGINT NOT NULL,
entity_id BIGINT NOT NULL,
chunk_id BIGINT,
page_id BIGINT,
surface_form VARCHAR(256),
char_offset INT,
confidence DECIMAL(4, 3),
evidence CLOB,
source VARCHAR(32),
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted INT NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_wem_entity ON mate_wiki_entity_mention (entity_id, deleted);
CREATE INDEX IF NOT EXISTS idx_wem_chunk ON mate_wiki_entity_mention (chunk_id);
CREATE INDEX IF NOT EXISTS idx_wem_page ON mate_wiki_entity_mention (page_id);
CREATE INDEX IF NOT EXISTS idx_wem_kb ON mate_wiki_entity_mention (kb_id, deleted);

View File

@ -0,0 +1,42 @@
-- mate_wiki_entity_relation: directed subject -> predicate -> object triples
-- between canonical entities (the entity-level knowledge graph edges).
--
-- Distinct from mate_wiki_relation, which scores page-to-page edges. A row
-- here is one fact triple connecting two mate_wiki_entity nodes.
--
-- subject_entity_id head entity
-- predicate free-text relation label (e.g. "works_for", "located_in")
-- object_entity_id tail entity
-- evidence short justification quote (<= 500 chars enforced in Java)
-- confidence 0..1 extraction confidence
-- source provenance tag: llm-extracted | inferred | manual
-- evidence_chunk_id source chunk the triple was extracted from, when known
-- computed_hash fingerprint of the inputs that produced this row
CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation (
id BIGINT PRIMARY KEY,
kb_id BIGINT NOT NULL,
subject_entity_id BIGINT NOT NULL,
predicate VARCHAR(64) NOT NULL,
object_entity_id BIGINT NOT NULL,
evidence CLOB,
confidence DECIMAL(4, 3),
source VARCHAR(32),
evidence_chunk_id BIGINT,
computed_hash VARCHAR(64),
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted INT NOT NULL DEFAULT 0
);
-- Unique triple identity within a KB.
CREATE UNIQUE INDEX IF NOT EXISTS uk_wer_triple
ON mate_wiki_entity_relation (kb_id, subject_entity_id, predicate, object_entity_id, deleted);
-- Ego-graph traversal in both directions.
CREATE INDEX IF NOT EXISTS idx_wer_subject ON mate_wiki_entity_relation (kb_id, subject_entity_id);
CREATE INDEX IF NOT EXISTS idx_wer_object ON mate_wiki_entity_relation (kb_id, object_entity_id);
CREATE INDEX IF NOT EXISTS idx_wer_kb ON mate_wiki_entity_relation (kb_id, deleted);

View File

@ -0,0 +1,31 @@
-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks.
-- See the H2 file for the design rationale.
CREATE TABLE IF NOT EXISTS mate_wiki_entity (
id BIGINT PRIMARY KEY,
kb_id BIGINT NOT NULL,
canonical_name VARCHAR(256) NOT NULL,
normalized_key VARCHAR(256) NOT NULL,
type VARCHAR(32) NOT NULL,
aliases_json TEXT,
description TEXT,
salience DECIMAL(5, 4),
mention_count INT NOT NULL DEFAULT 0,
embedding BYTEA,
embedding_model VARCHAR(64),
computed_hash VARCHAR(64),
create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted SMALLINT NOT NULL DEFAULT 0
);
CREATE UNIQUE INDEX IF NOT EXISTS uk_we_key
ON mate_wiki_entity (kb_id, normalized_key, type, deleted);
CREATE INDEX IF NOT EXISTS idx_we_kb ON mate_wiki_entity (kb_id, deleted);
CREATE INDEX IF NOT EXISTS idx_we_salience ON mate_wiki_entity (kb_id, salience DESC);
CREATE INDEX IF NOT EXISTS idx_we_type ON mate_wiki_entity (kb_id, type, deleted);

View File

@ -0,0 +1,25 @@
-- mate_wiki_entity_mention: links a canonical entity to a source occurrence.
-- See the H2 file for the design rationale.
CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention (
id BIGINT PRIMARY KEY,
kb_id BIGINT NOT NULL,
entity_id BIGINT NOT NULL,
chunk_id BIGINT,
page_id BIGINT,
surface_form VARCHAR(256),
char_offset INT,
confidence DECIMAL(4, 3),
evidence TEXT,
source VARCHAR(32),
create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted SMALLINT NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_wem_entity ON mate_wiki_entity_mention (entity_id, deleted);
CREATE INDEX IF NOT EXISTS idx_wem_chunk ON mate_wiki_entity_mention (chunk_id);
CREATE INDEX IF NOT EXISTS idx_wem_page ON mate_wiki_entity_mention (page_id);
CREATE INDEX IF NOT EXISTS idx_wem_kb ON mate_wiki_entity_mention (kb_id, deleted);

View File

@ -0,0 +1,27 @@
-- mate_wiki_entity_relation: directed subject -> predicate -> object triples
-- between canonical entities. See the H2 file for the design rationale.
CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation (
id BIGINT PRIMARY KEY,
kb_id BIGINT NOT NULL,
subject_entity_id BIGINT NOT NULL,
predicate VARCHAR(64) NOT NULL,
object_entity_id BIGINT NOT NULL,
evidence TEXT,
confidence DECIMAL(4, 3),
source VARCHAR(32),
evidence_chunk_id BIGINT,
computed_hash VARCHAR(64),
create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
deleted SMALLINT NOT NULL DEFAULT 0
);
CREATE UNIQUE INDEX IF NOT EXISTS uk_wer_triple
ON mate_wiki_entity_relation (kb_id, subject_entity_id, predicate, object_entity_id, deleted);
CREATE INDEX IF NOT EXISTS idx_wer_subject ON mate_wiki_entity_relation (kb_id, subject_entity_id);
CREATE INDEX IF NOT EXISTS idx_wer_object ON mate_wiki_entity_relation (kb_id, object_entity_id);
CREATE INDEX IF NOT EXISTS idx_wer_kb ON mate_wiki_entity_relation (kb_id, deleted);

View File

@ -0,0 +1,32 @@
-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks.
-- See the H2 file for the design rationale.
CREATE TABLE IF NOT EXISTS mate_wiki_entity (
id BIGINT NOT NULL,
kb_id BIGINT NOT NULL,
canonical_name VARCHAR(256) NOT NULL,
normalized_key VARCHAR(256) NOT NULL,
type VARCHAR(32) NOT NULL,
aliases_json LONGTEXT,
description LONGTEXT,
salience DECIMAL(5, 4),
mention_count INT NOT NULL DEFAULT 0,
embedding BLOB,
embedding_model VARCHAR(64),
computed_hash VARCHAR(64),
create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
deleted TINYINT NOT NULL DEFAULT 0,
PRIMARY KEY (id),
UNIQUE KEY uk_we_key (kb_id, normalized_key, type, deleted),
KEY idx_we_kb (kb_id, deleted),
KEY idx_we_salience (kb_id, salience DESC),
KEY idx_we_type (kb_id, type, deleted)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
COMMENT = 'Canonical named-entity nodes extracted and de-duplicated from source chunks.';

View File

@ -0,0 +1,27 @@
-- mate_wiki_entity_mention: links a canonical entity to a source occurrence.
-- See the H2 file for the design rationale.
CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention (
id BIGINT NOT NULL,
kb_id BIGINT NOT NULL,
entity_id BIGINT NOT NULL,
chunk_id BIGINT,
page_id BIGINT,
surface_form VARCHAR(256),
char_offset INT,
confidence DECIMAL(4, 3),
evidence TEXT,
source VARCHAR(32),
create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
deleted TINYINT NOT NULL DEFAULT 0,
PRIMARY KEY (id),
KEY idx_wem_entity (entity_id, deleted),
KEY idx_wem_chunk (chunk_id),
KEY idx_wem_page (page_id),
KEY idx_wem_kb (kb_id, deleted)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
COMMENT = 'Entity-to-source occurrence links connecting the entity layer to chunks and pages.';

View File

@ -0,0 +1,28 @@
-- mate_wiki_entity_relation: directed subject -> predicate -> object triples
-- between canonical entities. See the H2 file for the design rationale.
CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation (
id BIGINT NOT NULL,
kb_id BIGINT NOT NULL,
subject_entity_id BIGINT NOT NULL,
predicate VARCHAR(64) NOT NULL,
object_entity_id BIGINT NOT NULL,
evidence TEXT,
confidence DECIMAL(4, 3),
source VARCHAR(32),
evidence_chunk_id BIGINT,
computed_hash VARCHAR(64),
create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
deleted TINYINT NOT NULL DEFAULT 0,
PRIMARY KEY (id),
UNIQUE KEY uk_wer_triple (kb_id, subject_entity_id, predicate, object_entity_id, deleted),
KEY idx_wer_subject (kb_id, subject_entity_id),
KEY idx_wer_object (kb_id, object_entity_id),
KEY idx_wer_kb (kb_id, deleted)
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
COMMENT = 'Entity-to-entity fact triples forming the entity-level knowledge graph edges.';

View File

@ -0,0 +1,179 @@
package vip.mate.wiki.service;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.ai.chat.messages.AssistantMessage;
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.model.Generation;
import org.springframework.ai.chat.prompt.Prompt;
import vip.mate.llm.service.ModelConfigService;
import vip.mate.wiki.job.WikiJobStep;
import vip.mate.wiki.job.WikiModelRoutingService;
import vip.mate.wiki.model.WikiChunkEntity;
import vip.mate.wiki.model.WikiEntityEntity;
import vip.mate.wiki.model.WikiEntityMentionEntity;
import vip.mate.wiki.model.WikiEntityRelationEntity;
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
import vip.mate.wiki.repository.WikiEntityMapper;
import vip.mate.wiki.repository.WikiEntityMentionMapper;
import vip.mate.wiki.repository.WikiEntityRelationMapper;
import vip.mate.wiki.repository.WikiPageCitationMapper;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Verifies the entity-extraction runtime logic against a mocked model + DB:
* the structured LLM output is parsed, entities are de-duplicated across
* chunks via the run cache, mentions are written per occurrence, and a
* resolved relation triple is persisted. This exercises the real
* {@link WikiEntityExtractionService} control flow, not just a stubbed call.
*/
class WikiEntityExtractionServiceTest {
private static final Long KB_ID = 1L;
private static final Long RAW_ID = 100L;
private static final String LLM_JSON = """
{
"entities": [
{"name": "Alice", "type": "person", "aliases": [], "description": "An engineer", "evidence": "Alice works at Acme"},
{"name": "Acme", "type": "organization", "aliases": [], "description": "A company", "evidence": "Acme Corp"}
],
"relations": [
{"subject": "Alice", "predicate": "works for", "object": "Acme", "evidence": "Alice works at Acme"}
]
}
""";
private WikiKnowledgeBaseService kbService;
private WikiChunkService chunkService;
private WikiEmbeddingService embeddingService;
private WikiModelRoutingService routingService;
private ModelConfigService modelConfigService;
private WikiEntityMapper entityMapper;
private WikiEntityMentionMapper mentionMapper;
private WikiEntityRelationMapper relationMapper;
private WikiPageCitationMapper citationMapper;
private WikiEntityExtractionService service;
/** Simulated entity store so selectById sees what insert assigned. */
private final Map<Long, WikiEntityEntity> store = new HashMap<>();
@BeforeEach
void setUp() {
kbService = mock(WikiKnowledgeBaseService.class);
chunkService = mock(WikiChunkService.class);
embeddingService = mock(WikiEmbeddingService.class);
routingService = mock(WikiModelRoutingService.class);
modelConfigService = mock(ModelConfigService.class);
entityMapper = mock(WikiEntityMapper.class);
mentionMapper = mock(WikiEntityMentionMapper.class);
relationMapper = mock(WikiEntityRelationMapper.class);
citationMapper = mock(WikiPageCitationMapper.class);
service = new WikiEntityExtractionService(
kbService, chunkService, embeddingService, routingService,
modelConfigService, new ObjectMapper(),
entityMapper, mentionMapper, relationMapper, citationMapper);
WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity();
kb.setId(KB_ID);
when(kbService.getById(KB_ID)).thenReturn(kb);
// Model routing a mock ChatModel that always returns the canned JSON.
ChatModel canned = cannedModel(LLM_JSON);
when(routingService.selectModelId(eq(KB_ID), any(), eq(WikiJobStep.ENTITY_EXTRACTION)))
.thenReturn(7L);
when(routingService.buildChatModel(7L)).thenReturn(canned);
// No prior entities (with embeddings), no embedding vectors exercise
// the exact-key dedup path (no embedding merge).
when(entityMapper.selectList(any())).thenReturn(Collections.emptyList());
when(entityMapper.selectOne(any())).thenReturn(null);
when(embeddingService.embedQuery(anyLong(), any())).thenReturn(null);
// No existing mentions every chunk gets processed.
when(mentionMapper.selectCount(any(Wrapper.class))).thenReturn(0L);
when(citationMapper.listPageIdsByChunkId(anyLong())).thenReturn(Collections.emptyList());
when(relationMapper.selectOne(any())).thenReturn(null);
// insert assigns a snowflake-like id and records the row so selectById works.
AtomicLong seq = new AtomicLong(1000L);
when(entityMapper.insert(any(WikiEntityEntity.class))).thenAnswer(inv -> {
WikiEntityEntity e = inv.getArgument(0);
e.setId(seq.getAndIncrement());
store.put(e.getId(), e);
return 1;
});
when(entityMapper.selectById(anyLong())).thenAnswer(inv -> store.get(inv.getArgument(0)));
}
@Test
@DisplayName("extractForRaw: dedups entities across chunks, writes mentions and a relation")
void extractForRaw_buildsGraph() {
when(chunkService.listByRawId(RAW_ID)).thenReturn(List.of(
chunk(1L, "Alice works at Acme."),
chunk(2L, "Acme promoted Alice.")));
int touched = service.extractForRaw(KB_ID, RAW_ID);
// Two distinct canonical entities despite two chunks naming them.
assertEquals(2, touched, "should resolve exactly two canonical entities");
verify(entityMapper, times(2)).insert(any(WikiEntityEntity.class));
// One mention per (entity, chunk) occurrence 2 entities * 2 chunks.
verify(mentionMapper, times(4)).insert(any(WikiEntityMentionEntity.class));
// The works_for triple is persisted once per chunk it appears in.
verify(relationMapper, times(2)).insert(any(WikiEntityRelationEntity.class));
}
@Test
@DisplayName("extractForRaw: skips chunks that already have mentions")
void extractForRaw_skipsProcessedChunks() {
when(chunkService.listByRawId(RAW_ID)).thenReturn(List.of(chunk(1L, "Alice works at Acme.")));
when(mentionMapper.selectCount(any(Wrapper.class))).thenReturn(3L);
int touched = service.extractForRaw(KB_ID, RAW_ID);
assertEquals(0, touched);
verify(entityMapper, times(0)).insert(any(WikiEntityEntity.class));
}
private WikiChunkEntity chunk(Long id, String content) {
WikiChunkEntity c = new WikiChunkEntity();
c.setId(id);
c.setKbId(KB_ID);
c.setRawId(RAW_ID);
c.setContent(content);
return c;
}
private ChatModel cannedModel(String body) {
ChatModel model = mock(ChatModel.class);
when(model.call(any(Prompt.class))).thenReturn(new ChatResponse(List.of(
new Generation(new AssistantMessage(body),
ChatGenerationMetadata.builder().finishReason("STOP").build()))));
return model;
}
}

View File

@ -68,7 +68,8 @@ class WikiProcessingFallbackTest {
om,
mock(WikiProgressBus.class),
mock(WikiCitationService.class),
mock(ApplicationEventPublisher.class));
mock(ApplicationEventPublisher.class),
mock(WikiEntityExtractionService.class));
// Inject the optional fields via reflection Spring would do this
// post-construction in production, but the test instantiates directly.

View File

@ -67,7 +67,8 @@ class WikiProcessingServiceLazyTest {
new WikiLinkService(om),
properties, modelConfigService, agentGraphBuilder, om,
progressBus, citationService,
mock(org.springframework.context.ApplicationEventPublisher.class));
mock(org.springframework.context.ApplicationEventPublisher.class),
mock(WikiEntityExtractionService.class));
}
private WikiRawMaterialEntity raw() {

View File

@ -846,6 +846,16 @@ export const wikiApi = {
getPageCitations: (kbId: number | string, pageId: number | string) =>
http.get(`/wiki/kb/${kbId}/pages/${pageId}/citations`),
// Entity-level knowledge graph
listEntities: (kbId: number | string, params?: { type?: string; limit?: number }) =>
http.get(`/wiki/kb/${kbId}/entities`, { params }),
getEntityGraph: (kbId: number | string, limit = 150) =>
http.get(`/wiki/kb/${kbId}/entity-graph`, { params: { limit } }),
getEntityEgo: (kbId: number | string, entityId: number | string, limit = 50) =>
http.get(`/wiki/kb/${kbId}/entities/${entityId}/graph`, { params: { limit } }),
extractEntities: (kbId: number | string, force = false) =>
http.post(`/wiki/kb/${kbId}/entities/extract`, null, { params: { force } }),
// RFC-030: Jobs
getWikiJobs: (kbId: number, rawId: number) =>
http.get(`/wiki/kb/${kbId}/jobs`, { params: { rawId } }),

View File

@ -2442,6 +2442,11 @@ export default {
ingestModeLazy: 'Lazy',
ingestModeEagerHint: 'Upload runs the full LLM pipeline to generate wiki pages immediately.',
ingestModeLazyHint: 'Upload only extracts, chunks, and embeds. No page generation; search works right away.',
entityExtraction: 'Entity extraction',
entityExtractionHint: 'Extract named entities (people, organizations, locations, ...) and their relations from raw materials to build an entity-level knowledge graph. Adds processing time and token cost; off by default.',
entityExtractionEnable: 'Enable entity extraction',
entityExtractionRun: 'Extract now',
entityExtractionRunning: 'Extracting…',
modelStrategy: 'Model Strategy',
globalDefault: 'Global default',
selectModel: 'Select a model…',
@ -2517,6 +2522,15 @@ export default {
linksTo: 'Links to',
openPage: 'Open page',
empty: 'No graph data — process some raw materials first',
modePages: 'Pages',
modeEntities: 'Entities',
entityEmpty: 'No entities yet — enable entity extraction in the KB config and reprocess',
entityType: 'Type',
mentions: 'mentions',
mentionedIn: 'Mentioned in',
relations: 'Relations',
description: 'Description',
aliases: 'Aliases',
},
},
cronField: {

View File

@ -2454,6 +2454,11 @@ export default {
ingestModeLazy: '先入索引',
ingestModeEagerHint: '上传后立即调用 LLM 生成完整 Wiki 页面。',
ingestModeLazyHint: '上传仅抽取、切片、向量化,跳过页面生成;上传即可搜索,页面按需编译。',
entityExtraction: '实体抽取',
entityExtractionHint: '从原始材料中抽取命名实体(人物、组织、地点等)及其关系,构建实体级知识图谱。会增加处理时间与 Token 消耗,默认关闭。',
entityExtractionEnable: '开启实体抽取',
entityExtractionRun: '立即抽取',
entityExtractionRunning: '抽取中…',
modelStrategy: '模型策略',
globalDefault: '跟随全局默认',
selectModel: '选择可用模型…',
@ -2529,6 +2534,15 @@ export default {
linksTo: '链接到',
openPage: '打开页面',
empty: '暂无图谱数据,请先处理原始材料',
modePages: '页面',
modeEntities: '实体',
entityEmpty: '暂无实体数据,请在知识库配置中开启实体抽取后重新处理',
entityType: '类型',
mentions: '提及',
mentionedIn: '出现于页面',
relations: '关系',
description: '描述',
aliases: '别名',
},
},
cronField: {

View File

@ -46,6 +46,33 @@
</div>
</div>
<!-- ①c Entity extraction -->
<div class="config-card">
<div class="config-card__head">
<div>
<div class="config-card__title">{{ t('wiki.configPanel.entityExtraction') }}</div>
<div class="config-card__hint">{{ t('wiki.configPanel.entityExtractionHint') }}</div>
</div>
<button class="btn-save" @click="saveEntityExtraction" :disabled="savingEntityExtraction">
{{ savingEntityExtraction ? t('wiki.saving') : t('common.save') }}
</button>
</div>
<div class="ingest-mode-row">
<label class="entity-toggle">
<input type="checkbox" v-model="entityExtractionEnabled" :disabled="savingEntityExtraction" />
<span class="entity-toggle__label">{{ t('wiki.configPanel.entityExtractionEnable') }}</span>
</label>
<button
v-if="entityExtractionEnabled"
class="btn-save btn-save--ghost"
@click="runExtraction"
:disabled="extracting"
>
{{ extracting ? t('wiki.configPanel.entityExtractionRunning') : t('wiki.configPanel.entityExtractionRun') }}
</button>
</div>
</div>
<!-- Model strategy -->
<div class="config-card config-card--clickable" @click="modelsOpen = true">
<div class="config-card__row">
@ -230,6 +257,42 @@ async function saveIngestMode() {
}
}
// Entity extraction
// Opt-in named-entity knowledge graph extraction. Off by default because it
// adds an LLM call per chunk on top of the page pipeline.
const entityExtractionEnabled = ref(false)
const savingEntityExtraction = ref(false)
const extracting = ref(false)
async function saveEntityExtraction() {
if (!store.currentKB) return
savingEntityExtraction.value = true
try {
let existingConfig: any = {}
try {
if (store.currentKB.configContent) existingConfig = JSON.parse(store.currentKB.configContent)
} catch { /* config may be plain text rules */ }
existingConfig.entityExtractionEnabled = entityExtractionEnabled.value ? true : undefined
await wikiApi.updateConfig(store.currentKB.id, JSON.stringify(existingConfig, null, 2))
} catch (e) {
console.error('[WikiConfig] Failed to save entity extraction toggle', e)
} finally {
savingEntityExtraction.value = false
}
}
async function runExtraction() {
if (!store.currentKB) return
extracting.value = true
try {
await wikiApi.extractEntities(store.currentKB.id)
} catch (e) {
console.error('[WikiConfig] Failed to start entity extraction', e)
} finally {
extracting.value = false
}
}
// Model strategy
const stepKeys = ['route', 'create_page', 'merge_page', 'enrich', 'summary']
const stepModels = reactive<Record<string, string>>({})
@ -246,6 +309,7 @@ function loadStepModels() {
fallbackModelIds.value = []
wikiGlobalModelId.value = ''
ingestMode.value = 'eager'
entityExtractionEnabled.value = false
if (!store.currentKB) return
try {
const cfg = store.currentKB.configContent ? JSON.parse(store.currentKB.configContent) : null
@ -258,6 +322,7 @@ function loadStepModels() {
if (cfg?.fallbackModelIds) fallbackModelIds.value = cfg.fallbackModelIds.map(String)
if (cfg?.wikiDefaultModelId) wikiGlobalModelId.value = String(cfg.wikiDefaultModelId)
if (cfg?.ingestMode === 'lazy') ingestMode.value = 'lazy'
if (cfg?.entityExtractionEnabled) entityExtractionEnabled.value = true
} catch { /* not JSON */ }
}
@ -475,6 +540,12 @@ loadProviderNames().then(() => {
}
.btn-save:hover { opacity: 0.88; }
.btn-save:disabled { background: var(--mc-border); cursor: not-allowed; }
.btn-save--ghost { background: transparent; color: var(--mc-primary); border: 1px solid var(--mc-primary); }
.btn-save--ghost:disabled { background: transparent; color: var(--mc-text-tertiary); border-color: var(--mc-border); }
/* Entity extraction toggle */
.entity-toggle { display: flex; align-items: center; gap: 6px; font-size: 13px; color: var(--mc-text-primary); cursor: pointer; }
.entity-toggle__label { user-select: none; }
/* Ingest mode radio group */
.ingest-mode-row { display: flex; gap: 8px; flex-wrap: wrap; }

View File

@ -0,0 +1,264 @@
<template>
<div class="entity-graph">
<div ref="chartEl" class="graph-canvas" />
<!-- Entity detail panel -->
<div v-if="selected" class="entity-panel">
<button class="entity-panel__close" @click="selected = null">×</button>
<h3 class="entity-panel__title">{{ selected.canonicalName }}</h3>
<span class="entity-panel__type" :style="{ background: typeColor(selected.type) }">
{{ selected.type }}
</span>
<span class="entity-panel__count">{{ selected.mentionCount || 0 }} {{ t('wiki.graph.mentions') }}</span>
<p v-if="selected.description" class="entity-panel__desc">{{ selected.description }}</p>
<div v-if="selected.aliases && selected.aliases.length" class="entity-panel__section">
<h4>{{ t('wiki.graph.aliases') }}</h4>
<div class="entity-panel__tags">
<span v-for="a in selected.aliases" :key="a" class="entity-panel__tag">{{ a }}</span>
</div>
</div>
<div v-if="egoEdges.length" class="entity-panel__section">
<h4>{{ t('wiki.graph.relations') }}</h4>
<ul class="entity-panel__list">
<li v-for="(r, i) in egoEdges" :key="i">
<span class="entity-panel__pred">{{ r.predicate }}</span>
<span class="entity-panel__rel-target">{{ r.label }}</span>
</li>
</ul>
</div>
<div v-if="egoPages.length" class="entity-panel__section">
<h4>{{ t('wiki.graph.mentionedIn') }}</h4>
<ul class="entity-panel__list">
<li v-for="p in egoPages" :key="p.pageId">
<a class="entity-panel__link" @click="emit('open-page', p.slug)">{{ p.title }}</a>
</li>
</ul>
</div>
</div>
<!-- Empty state -->
<div v-if="!loading && nodes.length === 0" class="graph-empty">
<svg width="48" height="48" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1">
<circle cx="7" cy="7" r="3"/><circle cx="17" cy="17" r="3"/><line x1="9.5" y1="9.5" x2="14.5" y2="14.5"/>
</svg>
<p>{{ t('wiki.graph.entityEmpty') }}</p>
</div>
</div>
</template>
<script setup lang="ts">
import { ref, watch, onMounted, onBeforeUnmount, nextTick } from 'vue'
import { useI18n } from 'vue-i18n'
import * as echarts from 'echarts/core'
import { GraphChart } from 'echarts/charts'
import { TooltipComponent, LegendComponent } from 'echarts/components'
import { CanvasRenderer } from 'echarts/renderers'
import { wikiApi } from '@/api'
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
const { t } = useI18n()
interface EntityNode {
id: number | string
canonicalName: string
type: string
description?: string
aliases?: string[]
salience?: number
mentionCount?: number
}
interface EntityEdge {
id?: number | string
subjectEntityId: number | string
predicate: string
objectEntityId: number | string
}
const props = defineProps<{ kbId: number | string | null; isFullscreen?: boolean }>()
const emit = defineEmits<{
(e: 'open-page', slug: string): void
(e: 'stats', v: { nodes: number; edges: number }): void
}>()
const chartEl = ref<HTMLDivElement | null>(null)
let chart: echarts.ECharts | null = null
const nodes = ref<EntityNode[]>([])
const edges = ref<EntityEdge[]>([])
const loading = ref(false)
const selected = ref<EntityNode | null>(null)
const egoEdges = ref<{ predicate: string; label: string }[]>([])
const egoPages = ref<{ pageId: number | string; slug: string; title: string }[]>([])
// Stable color per entity type via a small hash palette.
const PALETTE = ['#5b8ff9', '#5ad8a6', '#f6bd16', '#e8684a', '#6dc8ec', '#9270ca', '#ff9d4d', '#269a99']
function typeColor(type: string): string {
let h = 0
for (let i = 0; i < (type || '').length; i++) h = (h * 31 + type.charCodeAt(i)) >>> 0
return PALETTE[h % PALETTE.length]
}
async function load() {
if (props.kbId == null) return
loading.value = true
try {
// The axios response interceptor returns the raw body for non-enveloped
// responses, so `res` is already { center, nodes, edges, pages }.
const res: any = await wikiApi.getEntityGraph(props.kbId, 150)
nodes.value = res?.nodes || []
edges.value = res?.edges || []
emit('stats', { nodes: nodes.value.length, edges: edges.value.length })
renderChart()
} finally {
loading.value = false
}
}
function buildOption() {
// Keep IDs as strings throughout backend issues Snowflake IDs that lose
// precision if coerced to Number.
const idSet = new Set(nodes.value.map(n => String(n.id)))
const nodeList = nodes.value.map(n => {
const size = Math.max(12, Math.min(46, 12 + (n.mentionCount || 0) * 3))
return {
id: String(n.id),
name: n.canonicalName,
symbolSize: size,
itemStyle: { color: typeColor(n.type) },
label: { show: size > 22, position: 'right' as const, fontSize: 10, color: 'var(--mc-text-secondary)', distance: 4 },
}
})
const edgeList = edges.value
.filter(e => idSet.has(String(e.subjectEntityId)) && idSet.has(String(e.objectEntityId)))
.map(e => ({
source: String(e.subjectEntityId),
target: String(e.objectEntityId),
value: e.predicate,
lineStyle: { color: 'rgba(150,150,150,0.3)', width: 1 },
}))
return {
backgroundColor: 'transparent',
tooltip: {
trigger: 'item',
formatter: (params: any) => {
if (params.dataType === 'edge') return params.data.value || ''
const node = nodes.value.find(n => String(n.id) === String(params.data.id))
if (!node) return ''
const desc = (node.description || '').substring(0, 80)
return `<div style="max-width:220px;white-space:normal"><strong>${node.canonicalName}</strong>`
+ `<small style="color:#999;display:block">${node.type}</small>`
+ (desc ? `<span style="font-size:11px">${desc}</span>` : '') + '</div>'
},
},
series: [{
type: 'graph',
layout: 'force',
data: nodeList,
links: edgeList,
roam: true,
force: { repulsion: 240, gravity: 0.05, edgeLength: [70, 200], friction: 0.55 },
emphasis: { focus: 'adjacency', lineStyle: { width: 2 } },
lineStyle: { color: 'rgba(150,150,150,0.3)', curveness: 0.1 },
edgeSymbol: ['none', 'arrow'],
edgeSymbolSize: 6,
}],
}
}
function renderChart() {
if (!chartEl.value) return
if (!chart) {
chart = echarts.init(chartEl.value, undefined, { renderer: 'canvas' })
chart.on('click', (params: any) => {
if (params.dataType === 'node') {
const node = nodes.value.find(n => String(n.id) === String(params.data.id))
if (node) openEntity(node)
}
})
}
chart.setOption(buildOption(), { notMerge: true, lazyUpdate: true })
}
async function openEntity(node: EntityNode) {
selected.value = node
egoEdges.value = []
egoPages.value = []
if (props.kbId == null) return
try {
const res: any = await wikiApi.getEntityEgo(props.kbId, node.id, 50)
const data = res || {}
const nodeById = new Map<string, string>()
for (const n of data.nodes || []) nodeById.set(String(n.id), n.canonicalName)
nodeById.set(String(node.id), node.canonicalName)
egoEdges.value = (data.edges || []).map((e: any) => {
const otherId = String(e.subjectEntityId) === String(node.id) ? e.objectEntityId : e.subjectEntityId
return { predicate: e.predicate, label: nodeById.get(String(otherId)) || '' }
})
egoPages.value = data.pages || []
} catch {
/* panel still shows the basic entity info */
}
}
const resizeObserver = new ResizeObserver(() => chart?.resize())
onMounted(async () => {
await nextTick()
if (chartEl.value) resizeObserver.observe(chartEl.value)
load()
})
onBeforeUnmount(() => {
resizeObserver.disconnect()
chart?.dispose()
chart = null
})
watch(() => props.kbId, () => { selected.value = null; load() })
watch(() => props.isFullscreen, () => nextTick(() => chart?.resize()))
defineExpose({ reload: load })
</script>
<style scoped>
.entity-graph { position: relative; flex: 1; min-height: 0; width: 100%; display: flex; }
.graph-canvas { flex: 1; min-height: 0; width: 100%; }
.graph-empty {
position: absolute; inset: 0; display: flex; flex-direction: column;
align-items: center; justify-content: center; gap: 12px;
color: var(--mc-text-tertiary); pointer-events: none;
}
.graph-empty p { font-size: 14px; max-width: 320px; text-align: center; }
.entity-panel {
position: absolute; top: 12px; right: 12px; width: 280px; max-height: calc(100% - 24px);
overflow-y: auto; padding: 16px; border-radius: 10px;
background: var(--mc-bg-elevated, #fff); border: 1px solid var(--mc-border, #e5e7eb);
box-shadow: 0 4px 20px rgba(0,0,0,0.12);
}
.entity-panel__close {
position: absolute; top: 8px; right: 10px; border: none; background: none;
font-size: 20px; line-height: 1; cursor: pointer; color: var(--mc-text-tertiary);
}
.entity-panel__title { margin: 0 20px 8px 0; font-size: 15px; font-weight: 600; }
.entity-panel__type { display: inline-block; padding: 1px 8px; border-radius: 10px; font-size: 11px; color: #fff; }
.entity-panel__count { margin-left: 8px; font-size: 11px; color: var(--mc-text-tertiary); }
.entity-panel__desc { margin: 10px 0 0; font-size: 12px; line-height: 1.5; color: var(--mc-text-secondary); }
.entity-panel__section { margin-top: 14px; }
.entity-panel__section h4 { margin: 0 0 6px; font-size: 11px; text-transform: uppercase; color: var(--mc-text-tertiary); }
.entity-panel__tags { display: flex; flex-wrap: wrap; gap: 4px; }
.entity-panel__tag { padding: 1px 6px; border-radius: 6px; font-size: 11px; background: var(--mc-bg-subtle, #f3f4f6); }
.entity-panel__list { margin: 0; padding: 0; list-style: none; font-size: 12px; }
.entity-panel__list li { padding: 2px 0; }
.entity-panel__pred { color: var(--mc-text-tertiary); margin-right: 6px; }
.entity-panel__rel-target { font-weight: 500; }
.entity-panel__link { color: var(--mc-primary, #5b8ff9); cursor: pointer; }
.entity-panel__link:hover { text-decoration: underline; }
</style>

View File

@ -13,7 +13,7 @@
</svg>
{{ edgeCount }} {{ t('wiki.graph.edges') }}
</span>
<span v-if="orphanCount > 0" class="stat-item stat-warn">
<span v-if="mode === 'pages' && orphanCount > 0" class="stat-item stat-warn">
<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5">
<circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="12"/><line x1="12" y1="16" x2="12.01" y2="16"/>
</svg>
@ -22,11 +22,23 @@
</div>
<div class="graph-controls">
<label class="filter-label">
<div class="mode-toggle">
<button
class="mode-btn"
:class="{ 'mode-btn--active': mode === 'pages' }"
@click="emit('update:mode', 'pages')"
>{{ t('wiki.graph.modePages') }}</button>
<button
class="mode-btn"
:class="{ 'mode-btn--active': mode === 'entities' }"
@click="emit('update:mode', 'entities')"
>{{ t('wiki.graph.modeEntities') }}</button>
</div>
<label v-if="mode === 'pages'" class="filter-label">
<input :checked="showOrphans" type="checkbox" @change="emit('update:showOrphans', ($event.target as HTMLInputElement).checked)" />
{{ t('wiki.graph.showOrphans') }}
</label>
<select :value="typeFilter" class="type-select" @change="emit('update:typeFilter', ($event.target as HTMLSelectElement).value)">
<select v-if="mode === 'pages'" :value="typeFilter" class="type-select" @change="emit('update:typeFilter', ($event.target as HTMLSelectElement).value)">
<option value="">{{ t('wiki.graph.allTypes') }}</option>
<option v-for="type in availableTypes" :key="type" :value="type">
{{ formatPageTypeLabel(type) }}
@ -73,11 +85,13 @@ defineProps<{
typeFilter: string
availableTypes: string[]
isFullscreen: boolean
mode: 'pages' | 'entities'
}>()
const emit = defineEmits<{
(e: 'update:showOrphans', val: boolean): void
(e: 'update:typeFilter', val: string): void
(e: 'update:mode', val: 'pages' | 'entities'): void
(e: 'reset'): void
(e: 'toggleFullscreen'): void
}>()
@ -106,6 +120,13 @@ const emit = defineEmits<{
.stat-warn { color: var(--mc-danger, #f56c6c); }
.graph-controls { display: flex; align-items: center; gap: 8px; }
.mode-toggle { display: inline-flex; border: 1px solid var(--mc-border-light); border-radius: 7px; overflow: hidden; }
.mode-btn {
padding: 3px 10px; font-size: 11px; border: none; cursor: pointer;
background: var(--mc-bg-elevated); color: var(--mc-text-secondary);
}
.mode-btn--active { background: var(--mc-primary, #5b8ff9); color: #fff; }
.filter-label { display: flex; align-items: center; gap: 4px; font-size: 11px; color: var(--mc-text-secondary); cursor: pointer; }
.type-select {

View File

@ -2,23 +2,33 @@
<div ref="graphViewEl" class="graph-view" :class="{ 'graph-view--fullscreen': isFullscreen }">
<!-- Toolbar sub-component -->
<WikiGraphToolbar
:node-count="nodes.length"
:edge-count="edges.length"
:node-count="graphMode === 'entities' ? entityStats.nodes : nodes.length"
:edge-count="graphMode === 'entities' ? entityStats.edges : edges.length"
:orphan-count="orphanCount"
v-model:show-orphans="showOrphans"
v-model:type-filter="typeFilter"
v-model:mode="graphMode"
:available-types="availableTypes"
:is-fullscreen="isFullscreen"
@reset="resetChart"
@toggle-fullscreen="toggleFullscreen"
/>
<!-- ECharts canvas -->
<div ref="chartEl" class="graph-canvas" />
<!-- Entity-level knowledge graph -->
<WikiEntityGraphView
v-if="graphMode === 'entities'"
:kb-id="kbId"
:is-fullscreen="isFullscreen"
@open-page="emit('open-page', $event)"
@stats="entityStats = $event"
/>
<!-- Page-level graph (ECharts canvas) -->
<div v-show="graphMode === 'pages'" ref="chartEl" class="graph-canvas" />
<!-- Node detail panel sub-component -->
<WikiGraphNodePanel
v-if="selectedNode"
v-if="graphMode === 'pages' && selectedNode"
:page="selectedNode"
:linked-pages="selectedNodeLinks"
@close="selectedNode = null"
@ -26,7 +36,7 @@
/>
<!-- Empty state -->
<div v-if="nodes.length === 0" class="graph-empty">
<div v-if="graphMode === 'pages' && nodes.length === 0" class="graph-empty">
<svg width="48" height="48" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1">
<circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3"/>
<line x1="5" y1="5" x2="19" y2="19" stroke-width="0.5"/>
@ -47,6 +57,7 @@ import type { WikiPage } from '@/stores/useWikiStore'
import { useWikiPageType } from '@/composables/useWikiPageType'
import WikiGraphToolbar from './WikiGraphToolbar.vue'
import WikiGraphNodePanel from './WikiGraphNodePanel.vue'
import WikiEntityGraphView from './WikiEntityGraphView.vue'
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
@ -64,6 +75,17 @@ const showOrphans = ref(true)
const typeFilter = ref('')
const selectedNode = ref<WikiPage | null>(null)
// 'pages' = page-link graph (default); 'entities' = named-entity graph.
const graphMode = ref<'pages' | 'entities'>('pages')
const entityStats = ref<{ nodes: number; edges: number }>({ nodes: 0, edges: 0 })
// KB id for entity-graph fetches every page carries its kbId; keep it a
// string to avoid Snowflake precision loss.
const kbId = computed<string | number | null>(() => {
const id = props.pages[0]?.kbId
return id != null ? id : null
})
// Parse outgoing links JSON string slug[]
function parseLinks(outgoingLinks: string | null | undefined): string[] {
@ -340,6 +362,14 @@ onBeforeUnmount(() => {
watch([nodes, edges], () => {
scheduleRender()
})
// Returning to the page graph: the canvas was hidden (v-show), so let the DOM
// settle then tell ECharts to re-measure.
watch(graphMode, (mode) => {
if (mode === 'pages') {
nextTick(() => chart?.resize())
}
})
</script>
<style scoped>