mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(wiki): entity-level knowledge graph extraction (#336)
Add an opt-in named-entity extraction pass so the wiki knowledge graph captures fine-grained entities (people, organizations, locations, ...) and their relations, not just page-level link relations. - new tables mate_wiki_entity / _mention / _relation (h2/mysql/kingbase) - structured LLM extraction per chunk with entity resolution (normalized-key dedup + embedding near-merge), mention/relation persistence and page linking via chunk citations - per-KB opt-in toggle (off by default); async dispatch after embedding - read API: entity list, KB graph, entity ego-graph, manual extract - UI: entity-layer toggle in the graph view + KB config toggle - replace inline fully-qualified class names with imports in WikiProcessingService Closes #336
This commit is contained in:
parent
fe68f22aa8
commit
6e7c137154
@ -0,0 +1,76 @@
|
||||
package vip.mate.wiki.controller;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import vip.mate.wiki.dto.WikiEntityGraphView;
|
||||
import vip.mate.wiki.dto.WikiEntityView;
|
||||
import vip.mate.wiki.service.WikiEntityExtractionService;
|
||||
import vip.mate.wiki.service.WikiEntityGraphService;
|
||||
import vip.mate.wiki.service.WikiProcessingService;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Read and manual-trigger endpoints for the entity-level knowledge graph.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@RestController
|
||||
@RequestMapping("/api/v1/wiki")
|
||||
@RequiredArgsConstructor
|
||||
public class WikiEntityController {
|
||||
|
||||
private final WikiEntityGraphService graphService;
|
||||
private final WikiEntityExtractionService extractionService;
|
||||
|
||||
/** List entities in a KB, optionally filtered by type, ranked by salience. */
|
||||
@GetMapping("/kb/{kbId}/entities")
|
||||
public List<WikiEntityView> listEntities(@PathVariable Long kbId,
|
||||
@RequestParam(required = false) String type,
|
||||
@RequestParam(defaultValue = "100") int limit) {
|
||||
return graphService.listEntities(kbId, type, limit);
|
||||
}
|
||||
|
||||
/** Whole-KB entity graph: top entities by salience plus the edges among them. */
|
||||
@GetMapping("/kb/{kbId}/entity-graph")
|
||||
public WikiEntityGraphView kbEntityGraph(@PathVariable Long kbId,
|
||||
@RequestParam(defaultValue = "150") int limit) {
|
||||
return graphService.graph(kbId, limit);
|
||||
}
|
||||
|
||||
/** Ego-graph around a single entity: neighbors, edges, and mentioning pages. */
|
||||
@GetMapping("/kb/{kbId}/entities/{entityId}/graph")
|
||||
public WikiEntityGraphView entityGraph(@PathVariable Long kbId,
|
||||
@PathVariable Long entityId,
|
||||
@RequestParam(defaultValue = "50") int limit) {
|
||||
return graphService.ego(kbId, entityId, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Manually trigger an entity-extraction pass for a KB. Runs on the wiki
|
||||
* executor so the request returns immediately.
|
||||
*
|
||||
* @param force when true, re-extract chunks that already have mentions
|
||||
*/
|
||||
@PostMapping("/kb/{kbId}/entities/extract")
|
||||
public Map<String, Object> extract(@PathVariable Long kbId,
|
||||
@RequestParam(defaultValue = "false") boolean force) {
|
||||
WikiProcessingService.WIKI_EXECUTOR.submit(() -> {
|
||||
try {
|
||||
int count = extractionService.extractForKb(kbId, force);
|
||||
log.info("[WikiEntity] Manual extraction completed: kbId={}, entities={}", kbId, count);
|
||||
} catch (Exception e) {
|
||||
log.warn("[WikiEntity] Manual extraction failed for kbId={}: {}", kbId, e.getMessage());
|
||||
}
|
||||
});
|
||||
return Map.of("status", "started", "kbId", kbId);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package vip.mate.wiki.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Structured output schema for a single entity-extraction LLM call over one
|
||||
* source chunk. Bound via {@code BeanOutputConverter} and tolerant of a model
|
||||
* returning either field empty.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
public class EntityExtractionResult {
|
||||
|
||||
/** Named entities found in the chunk. */
|
||||
private List<ExtractedEntity> entities = new ArrayList<>();
|
||||
|
||||
/** Subject → predicate → object triples between the extracted entities. */
|
||||
private List<ExtractedRelation> relations = new ArrayList<>();
|
||||
|
||||
@Data
|
||||
public static class ExtractedEntity {
|
||||
/** Canonical surface form of the entity as it appears in the text. */
|
||||
private String name;
|
||||
/** One of the requested types: person | organization | location | event | product | concept | other. */
|
||||
private String type;
|
||||
/** Alternate names / spellings for the same entity, if any. */
|
||||
private List<String> aliases = new ArrayList<>();
|
||||
/** One-line description grounded in the chunk. */
|
||||
private String description;
|
||||
/** Short verbatim quote evidencing the entity. */
|
||||
private String evidence;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class ExtractedRelation {
|
||||
/** Subject entity name (should match an entry in {@link #entities}). */
|
||||
private String subject;
|
||||
/** Relation label, e.g. "works_for", "located_in", "founded". */
|
||||
private String predicate;
|
||||
/** Object entity name (should match an entry in {@link #entities}). */
|
||||
private String object;
|
||||
/** Short verbatim quote evidencing the relation. */
|
||||
private String evidence;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
package vip.mate.wiki.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Ego-graph around one entity: the center node, its neighbor entity nodes,
|
||||
* the relation edges connecting them, and the wiki pages that mention the
|
||||
* center entity (the bridge from the entity layer to the page layer).
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
public class WikiEntityGraphView {
|
||||
|
||||
private WikiEntityView center;
|
||||
private List<WikiEntityView> nodes = new ArrayList<>();
|
||||
private List<Edge> edges = new ArrayList<>();
|
||||
private List<PageRef> pages = new ArrayList<>();
|
||||
|
||||
@Data
|
||||
public static class Edge {
|
||||
private Long id;
|
||||
private Long subjectEntityId;
|
||||
private String predicate;
|
||||
private Long objectEntityId;
|
||||
private String evidence;
|
||||
private BigDecimal confidence;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class PageRef {
|
||||
private Long pageId;
|
||||
private String slug;
|
||||
private String title;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
package vip.mate.wiki.dto;
|
||||
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* API-facing projection of a canonical entity node. Excludes the raw
|
||||
* embedding vector and other internal columns.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
public class WikiEntityView {
|
||||
private Long id;
|
||||
private Long kbId;
|
||||
private String canonicalName;
|
||||
private String type;
|
||||
private List<String> aliases;
|
||||
private String description;
|
||||
private BigDecimal salience;
|
||||
private Integer mentionCount;
|
||||
}
|
||||
@ -1,9 +1,8 @@
|
||||
package vip.mate.wiki.job;
|
||||
|
||||
/**
|
||||
* RFC-030: Logical steps within a wiki processing job,
|
||||
* used for per-step model routing.
|
||||
* Logical steps within a wiki processing job, used for per-step model routing.
|
||||
*/
|
||||
public enum WikiJobStep {
|
||||
ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY
|
||||
ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY, ENTITY_EXTRACTION
|
||||
}
|
||||
|
||||
@ -57,4 +57,21 @@ public class WikiKbConfig {
|
||||
* pageType to be granted explicitly per agent.
|
||||
*/
|
||||
private String defaultReadPolicy;
|
||||
|
||||
/**
|
||||
* Opt-in for entity-level knowledge graph extraction on this KB. When
|
||||
* {@code true}, an extraction pass runs after ingest/embedding to pull
|
||||
* named entities (person, organization, location, ...) and their
|
||||
* relations from source chunks into the {@code mate_wiki_entity*} tables.
|
||||
* {@code null} or {@code false} keeps the legacy behaviour (page graph
|
||||
* only). Off by default because extraction adds LLM calls per chunk.
|
||||
*/
|
||||
private Boolean entityExtractionEnabled;
|
||||
|
||||
/**
|
||||
* Optional whitelist of entity types to extract, e.g.
|
||||
* {@code ["person","organization","location"]}. {@code null} or empty
|
||||
* lets the extractor use its built-in default type set.
|
||||
*/
|
||||
private List<String> entityTypes;
|
||||
}
|
||||
|
||||
@ -0,0 +1,72 @@
|
||||
package vip.mate.wiki.model;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.FieldFill;
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableLogic;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* Canonical named-entity node extracted and de-duplicated from source chunks.
|
||||
*
|
||||
* <p>Mention-granularity counterpart to {@link WikiPageEntity} (which models
|
||||
* document/topic granularity). Entities link to their source occurrences via
|
||||
* {@link WikiEntityMentionEntity} and to one another via
|
||||
* {@link WikiEntityRelationEntity}, forming an entity-level knowledge graph
|
||||
* beneath the page graph cached in {@link WikiRelationEntity}.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
@TableName("mate_wiki_entity")
|
||||
public class WikiEntityEntity {
|
||||
|
||||
@TableId(type = IdType.ASSIGN_ID)
|
||||
private Long id;
|
||||
|
||||
private Long kbId;
|
||||
|
||||
/** Display name chosen for the merged entity. */
|
||||
private String canonicalName;
|
||||
|
||||
/** Case/whitespace-folded key used for exact-match de-duplication. */
|
||||
private String normalizedKey;
|
||||
|
||||
/** Entity taxonomy: person | organization | location | event | product | concept | other. */
|
||||
private String type;
|
||||
|
||||
/** JSON array of surface forms merged into this entity. */
|
||||
private String aliasesJson;
|
||||
|
||||
/** One-line summary synthesized from the mentions. */
|
||||
private String description;
|
||||
|
||||
/** 0..1 importance score derived from mention frequency / distribution. */
|
||||
private BigDecimal salience;
|
||||
|
||||
/** Number of mentions resolved to this entity. */
|
||||
private Integer mentionCount;
|
||||
|
||||
/** Float32 little-endian name/description vector used for near-duplicate merge. */
|
||||
private byte[] embedding;
|
||||
|
||||
/** Model name that produced {@link #embedding}; used for re-embed detection. */
|
||||
private String embeddingModel;
|
||||
|
||||
/** Fingerprint of the inputs that produced this row; used for cache invalidation. */
|
||||
private String computedHash;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT)
|
||||
private LocalDateTime createTime;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT_UPDATE)
|
||||
private LocalDateTime updateTime;
|
||||
|
||||
@TableLogic
|
||||
private Integer deleted;
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
package vip.mate.wiki.model;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.FieldFill;
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableLogic;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* Links a canonical {@link WikiEntityEntity} to a single source occurrence.
|
||||
*
|
||||
* <p>One row per (entity, chunk) occurrence. {@link #pageId} is back-filled
|
||||
* from the chunk's citing pages so the entity layer connects to the page
|
||||
* layer: entity → mention → chunk → citing page.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
@TableName("mate_wiki_entity_mention")
|
||||
public class WikiEntityMentionEntity {
|
||||
|
||||
@TableId(type = IdType.ASSIGN_ID)
|
||||
private Long id;
|
||||
|
||||
private Long kbId;
|
||||
|
||||
/** The resolved canonical entity. */
|
||||
private Long entityId;
|
||||
|
||||
/** Source chunk the mention was found in. */
|
||||
private Long chunkId;
|
||||
|
||||
/** A wiki page that cites {@link #chunkId}, when known; null otherwise. */
|
||||
private Long pageId;
|
||||
|
||||
/** The exact text as it appeared in the source. */
|
||||
private String surfaceForm;
|
||||
|
||||
/** Character offset of the mention within the chunk, when known. */
|
||||
private Integer charOffset;
|
||||
|
||||
/** 0..1 extraction confidence. */
|
||||
private BigDecimal confidence;
|
||||
|
||||
/** Short surrounding quote (≤ 500 chars enforced in Java layer). */
|
||||
private String evidence;
|
||||
|
||||
/** Provenance tag: llm-extracted | manual. */
|
||||
private String source;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT)
|
||||
private LocalDateTime createTime;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT_UPDATE)
|
||||
private LocalDateTime updateTime;
|
||||
|
||||
@TableLogic
|
||||
private Integer deleted;
|
||||
}
|
||||
@ -0,0 +1,64 @@
|
||||
package vip.mate.wiki.model;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.FieldFill;
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableField;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import com.baomidou.mybatisplus.annotation.TableLogic;
|
||||
import com.baomidou.mybatisplus.annotation.TableName;
|
||||
import lombok.Data;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
/**
|
||||
* Directed subject → predicate → object triple between two canonical entities;
|
||||
* the edges of the entity-level knowledge graph.
|
||||
*
|
||||
* <p>Distinct from {@link WikiRelationEntity}, which scores page-to-page edges.
|
||||
* A row here is one fact triple connecting two {@link WikiEntityEntity} nodes.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Data
|
||||
@TableName("mate_wiki_entity_relation")
|
||||
public class WikiEntityRelationEntity {
|
||||
|
||||
@TableId(type = IdType.ASSIGN_ID)
|
||||
private Long id;
|
||||
|
||||
private Long kbId;
|
||||
|
||||
/** Head entity. */
|
||||
private Long subjectEntityId;
|
||||
|
||||
/** Free-text relation label, e.g. {@code works_for}, {@code located_in}. */
|
||||
private String predicate;
|
||||
|
||||
/** Tail entity. */
|
||||
private Long objectEntityId;
|
||||
|
||||
/** Short justification quote (≤ 500 chars enforced in Java layer). */
|
||||
private String evidence;
|
||||
|
||||
/** 0..1 extraction confidence. */
|
||||
private BigDecimal confidence;
|
||||
|
||||
/** Provenance tag: llm-extracted | inferred | manual. */
|
||||
private String source;
|
||||
|
||||
/** Source chunk the triple was extracted from, when known. */
|
||||
private Long evidenceChunkId;
|
||||
|
||||
/** Fingerprint of the inputs that produced this row; used for cache invalidation. */
|
||||
private String computedHash;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT)
|
||||
private LocalDateTime createTime;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT_UPDATE)
|
||||
private LocalDateTime updateTime;
|
||||
|
||||
@TableLogic
|
||||
private Integer deleted;
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package vip.mate.wiki.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import vip.mate.wiki.model.WikiEntityEntity;
|
||||
|
||||
/**
|
||||
* Mapper for canonical named-entity nodes.
|
||||
*
|
||||
* <p>Write paths upsert keyed by ({@code kb_id}, {@code normalized_key},
|
||||
* {@code type}); read paths list by {@code kb_id} ordered by {@code salience}.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Mapper
|
||||
public interface WikiEntityMapper extends BaseMapper<WikiEntityEntity> {
|
||||
}
|
||||
@ -0,0 +1,17 @@
|
||||
package vip.mate.wiki.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import vip.mate.wiki.model.WikiEntityMentionEntity;
|
||||
|
||||
/**
|
||||
* Mapper for entity-to-source occurrence links.
|
||||
*
|
||||
* <p>Read paths fetch mentions by {@code entity_id} or by {@code page_id};
|
||||
* cache invalidation soft-deletes by {@code chunk_id} or {@code kb_id}.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Mapper
|
||||
public interface WikiEntityMentionMapper extends BaseMapper<WikiEntityMentionEntity> {
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
package vip.mate.wiki.repository;
|
||||
|
||||
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
|
||||
import org.apache.ibatis.annotations.Mapper;
|
||||
import vip.mate.wiki.model.WikiEntityRelationEntity;
|
||||
|
||||
/**
|
||||
* Mapper for entity-to-entity fact triples.
|
||||
*
|
||||
* <p>Read paths traverse the ego-graph by {@code subject_entity_id} /
|
||||
* {@code object_entity_id}; write paths upsert keyed by the triple
|
||||
* ({@code kb_id}, {@code subject_entity_id}, {@code predicate},
|
||||
* {@code object_entity_id}).
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Mapper
|
||||
public interface WikiEntityRelationMapper extends BaseMapper<WikiEntityRelationEntity> {
|
||||
}
|
||||
@ -0,0 +1,457 @@
|
||||
package vip.mate.wiki.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.converter.BeanOutputConverter;
|
||||
import org.springframework.stereotype.Service;
|
||||
import vip.mate.llm.service.ModelConfigService;
|
||||
import vip.mate.wiki.dto.EntityExtractionResult;
|
||||
import vip.mate.wiki.job.WikiJobStep;
|
||||
import vip.mate.wiki.job.WikiKbConfig;
|
||||
import vip.mate.wiki.job.WikiKbConfigParser;
|
||||
import vip.mate.wiki.job.WikiModelRoutingService;
|
||||
import vip.mate.wiki.model.WikiChunkEntity;
|
||||
import vip.mate.wiki.model.WikiEntityEntity;
|
||||
import vip.mate.wiki.model.WikiEntityMentionEntity;
|
||||
import vip.mate.wiki.model.WikiEntityRelationEntity;
|
||||
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
|
||||
import vip.mate.wiki.repository.WikiEntityMapper;
|
||||
import vip.mate.wiki.repository.WikiEntityMentionMapper;
|
||||
import vip.mate.wiki.repository.WikiEntityRelationMapper;
|
||||
import vip.mate.wiki.repository.WikiPageCitationMapper;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Extracts a named-entity knowledge graph from source chunks: it pulls
|
||||
* entities (person, organization, location, ...) and subject→predicate→object
|
||||
* relations out of each chunk via a structured LLM call, resolves entities to
|
||||
* canonical nodes (exact-key dedup plus optional embedding near-merge), and
|
||||
* persists nodes, mentions, and edges into the {@code mate_wiki_entity*}
|
||||
* tables.
|
||||
*
|
||||
* <p>This is an opt-in pass gated by {@link WikiKbConfig#getEntityExtractionEnabled()};
|
||||
* it runs after ingest/embedding and never blocks the page-generation pipeline.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class WikiEntityExtractionService {
|
||||
|
||||
private static final List<String> DEFAULT_ENTITY_TYPES =
|
||||
List.of("person", "organization", "location", "event", "product", "concept");
|
||||
|
||||
/** Cosine threshold above which a new entity is merged into an existing same-type node. */
|
||||
private static final float MERGE_THRESHOLD = 0.92f;
|
||||
|
||||
/** Max chunk characters sent to the model per call, to bound token spend. */
|
||||
private static final int MAX_CHUNK_CHARS = 6000;
|
||||
|
||||
/** Evidence column is capped at 500 chars in the schema. */
|
||||
private static final int MAX_EVIDENCE = 500;
|
||||
|
||||
private final WikiKnowledgeBaseService kbService;
|
||||
private final WikiChunkService chunkService;
|
||||
private final WikiEmbeddingService embeddingService;
|
||||
private final WikiModelRoutingService routingService;
|
||||
private final ModelConfigService modelConfigService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private final WikiEntityMapper entityMapper;
|
||||
private final WikiEntityMentionMapper mentionMapper;
|
||||
private final WikiEntityRelationMapper relationMapper;
|
||||
private final WikiPageCitationMapper citationMapper;
|
||||
|
||||
/** Extract entities from every not-yet-processed chunk of one raw material. */
|
||||
public int extractForRaw(Long kbId, Long rawId) {
|
||||
return extract(kbId, chunkService.listByRawId(rawId), false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract entities across the whole KB.
|
||||
*
|
||||
* @param force when {@code true}, re-extract chunks that already have
|
||||
* mentions (used for a manual full rebuild); otherwise skip
|
||||
* chunks already processed
|
||||
*/
|
||||
public int extractForKb(Long kbId, boolean force) {
|
||||
return extract(kbId, chunkService.listByKbId(kbId), force);
|
||||
}
|
||||
|
||||
private int extract(Long kbId, List<WikiChunkEntity> chunks, boolean force) {
|
||||
WikiKnowledgeBaseEntity kb = kbService.getById(kbId);
|
||||
if (kb == null || chunks == null || chunks.isEmpty()) {
|
||||
return 0;
|
||||
}
|
||||
ChatModel chatModel = resolveChatModel(kbId);
|
||||
if (chatModel == null) {
|
||||
log.warn("[WikiEntity] No chat model available for kbId={}, skipping extraction", kbId);
|
||||
return 0;
|
||||
}
|
||||
|
||||
List<String> types = resolveEntityTypes(kb);
|
||||
BeanOutputConverter<EntityExtractionResult> converter =
|
||||
new BeanOutputConverter<>(EntityExtractionResult.class);
|
||||
String systemPrompt = buildSystemPrompt(types);
|
||||
|
||||
// Per-run resolution cache: type+normalizedKey → entityId. Seeded lazily
|
||||
// from the DB so entities resolve consistently within and across chunks.
|
||||
Map<String, Long> resolved = new HashMap<>();
|
||||
// Same-type existing entities with embeddings, for near-duplicate merge.
|
||||
EntityIndex index = new EntityIndex(kbId);
|
||||
|
||||
for (WikiChunkEntity chunk : chunks) {
|
||||
if (chunk.getContent() == null || chunk.getContent().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
if (!force && hasMentions(chunk.getId())) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
EntityExtractionResult result = callExtract(chatModel, converter, systemPrompt, chunk);
|
||||
if (result == null) {
|
||||
continue;
|
||||
}
|
||||
persistChunk(kbId, chunk, result, resolved, index);
|
||||
} catch (Exception e) {
|
||||
log.warn("[WikiEntity] Extraction failed for chunkId={} kbId={}: {}",
|
||||
chunk.getId(), kbId, e.getMessage());
|
||||
}
|
||||
}
|
||||
return resolved.size();
|
||||
}
|
||||
|
||||
// ---- LLM call ---------------------------------------------------------
|
||||
|
||||
private EntityExtractionResult callExtract(ChatModel chatModel,
|
||||
BeanOutputConverter<EntityExtractionResult> converter,
|
||||
String systemPrompt,
|
||||
WikiChunkEntity chunk) {
|
||||
String content = chunk.getContent();
|
||||
if (content.length() > MAX_CHUNK_CHARS) {
|
||||
content = content.substring(0, MAX_CHUNK_CHARS);
|
||||
}
|
||||
String userPrompt = "Source text:\n\"\"\"\n" + content + "\n\"\"\"\n\n" + converter.getFormat();
|
||||
Prompt prompt = new Prompt(List.of(
|
||||
new SystemMessage(systemPrompt),
|
||||
new UserMessage(userPrompt)));
|
||||
ChatResponse response = chatModel.call(prompt);
|
||||
String text = response.getResult().getOutput().getText();
|
||||
if (text == null || text.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return converter.convert(text);
|
||||
} catch (Exception e) {
|
||||
log.debug("[WikiEntity] Structured parse failed, skipping chunk: {}", e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String buildSystemPrompt(List<String> types) {
|
||||
return "You are a knowledge-graph entity extractor. From the given source text, "
|
||||
+ "extract named entities and the factual relations between them.\n"
|
||||
+ "Entity types to use: " + String.join(", ", types) + ".\n"
|
||||
+ "Rules:\n"
|
||||
+ "- Only extract entities explicitly named in the text; do not invent any.\n"
|
||||
+ "- Use the most complete surface form as the name; list shorter forms as aliases.\n"
|
||||
+ "- For each relation, subject and object must both appear in the entities list.\n"
|
||||
+ "- Keep predicates short and snake_case (e.g. works_for, located_in, founded).\n"
|
||||
+ "- Provide a short verbatim evidence quote for each entity and relation.\n"
|
||||
+ "- If nothing relevant is present, return empty lists.";
|
||||
}
|
||||
|
||||
// ---- persistence ------------------------------------------------------
|
||||
|
||||
private void persistChunk(Long kbId, WikiChunkEntity chunk, EntityExtractionResult result,
|
||||
Map<String, Long> resolved, EntityIndex index) {
|
||||
Long pageId = firstCitingPage(chunk.getId());
|
||||
|
||||
// Resolve each entity to a canonical id, persist its mention for this chunk.
|
||||
Map<String, Long> localByName = new HashMap<>();
|
||||
if (result.getEntities() != null) {
|
||||
for (EntityExtractionResult.ExtractedEntity e : result.getEntities()) {
|
||||
if (e == null || e.getName() == null || e.getName().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
String type = normalizeType(e.getType());
|
||||
Long entityId = resolveEntity(kbId, e, type, resolved, index);
|
||||
if (entityId == null) {
|
||||
continue;
|
||||
}
|
||||
localByName.put(normalize(e.getName()), entityId);
|
||||
if (e.getAliases() != null) {
|
||||
for (String alias : e.getAliases()) {
|
||||
if (alias != null && !alias.isBlank()) {
|
||||
localByName.put(normalize(alias), entityId);
|
||||
}
|
||||
}
|
||||
}
|
||||
insertMention(kbId, entityId, chunk.getId(), pageId, e.getName(), e.getEvidence());
|
||||
bumpMentionCount(entityId);
|
||||
}
|
||||
}
|
||||
|
||||
// Persist relations whose endpoints both resolved.
|
||||
if (result.getRelations() != null) {
|
||||
for (EntityExtractionResult.ExtractedRelation r : result.getRelations()) {
|
||||
if (r == null || r.getSubject() == null || r.getObject() == null
|
||||
|| r.getPredicate() == null || r.getPredicate().isBlank()) {
|
||||
continue;
|
||||
}
|
||||
Long subjectId = localByName.get(normalize(r.getSubject()));
|
||||
Long objectId = localByName.get(normalize(r.getObject()));
|
||||
if (subjectId == null || objectId == null || subjectId.equals(objectId)) {
|
||||
continue;
|
||||
}
|
||||
upsertRelation(kbId, subjectId, objectId, normalizePredicate(r.getPredicate()),
|
||||
r.getEvidence(), chunk.getId());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an extracted entity to a canonical node id: run cache → exact
|
||||
* key match in DB → embedding near-match → create new.
|
||||
*/
|
||||
private Long resolveEntity(Long kbId, EntityExtractionResult.ExtractedEntity e, String type,
|
||||
Map<String, Long> resolved, EntityIndex index) {
|
||||
String key = normalize(e.getName());
|
||||
String cacheKey = type + "" + key;
|
||||
Long cached = resolved.get(cacheKey);
|
||||
if (cached != null) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
WikiEntityEntity existing = entityMapper.selectOne(new LambdaQueryWrapper<WikiEntityEntity>()
|
||||
.eq(WikiEntityEntity::getKbId, kbId)
|
||||
.eq(WikiEntityEntity::getNormalizedKey, key)
|
||||
.eq(WikiEntityEntity::getType, type)
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
resolved.put(cacheKey, existing.getId());
|
||||
return existing.getId();
|
||||
}
|
||||
|
||||
// Embedding near-duplicate merge across spellings/languages.
|
||||
float[] vec = embedName(kbId, e);
|
||||
if (vec != null) {
|
||||
Long near = index.findNearest(type, vec);
|
||||
if (near != null) {
|
||||
resolved.put(cacheKey, near);
|
||||
return near;
|
||||
}
|
||||
}
|
||||
|
||||
WikiEntityEntity created = new WikiEntityEntity();
|
||||
created.setKbId(kbId);
|
||||
created.setCanonicalName(e.getName().trim());
|
||||
created.setNormalizedKey(key);
|
||||
created.setType(type);
|
||||
created.setAliasesJson(writeJson(e.getAliases()));
|
||||
created.setDescription(truncate(e.getDescription(), MAX_EVIDENCE));
|
||||
created.setMentionCount(0);
|
||||
created.setSalience(BigDecimal.ZERO);
|
||||
if (vec != null) {
|
||||
created.setEmbedding(WikiEmbeddingService.floatsToBytes(vec));
|
||||
}
|
||||
entityMapper.insert(created);
|
||||
resolved.put(cacheKey, created.getId());
|
||||
index.add(type, created.getId(), vec);
|
||||
return created.getId();
|
||||
}
|
||||
|
||||
private void insertMention(Long kbId, Long entityId, Long chunkId, Long pageId,
|
||||
String surfaceForm, String evidence) {
|
||||
WikiEntityMentionEntity m = new WikiEntityMentionEntity();
|
||||
m.setKbId(kbId);
|
||||
m.setEntityId(entityId);
|
||||
m.setChunkId(chunkId);
|
||||
m.setPageId(pageId);
|
||||
m.setSurfaceForm(truncate(surfaceForm, 256));
|
||||
m.setConfidence(BigDecimal.valueOf(0.9));
|
||||
m.setEvidence(truncate(evidence, MAX_EVIDENCE));
|
||||
m.setSource("llm-extracted");
|
||||
mentionMapper.insert(m);
|
||||
}
|
||||
|
||||
private void bumpMentionCount(Long entityId) {
|
||||
WikiEntityEntity e = entityMapper.selectById(entityId);
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
int count = (e.getMentionCount() == null ? 0 : e.getMentionCount()) + 1;
|
||||
e.setMentionCount(count);
|
||||
// Saturating popularity score in [0,1): count / (count + 5).
|
||||
e.setSalience(BigDecimal.valueOf((double) count / (count + 5.0))
|
||||
.setScale(4, RoundingMode.HALF_UP));
|
||||
entityMapper.updateById(e);
|
||||
}
|
||||
|
||||
private void upsertRelation(Long kbId, Long subjectId, Long objectId, String predicate,
|
||||
String evidence, Long chunkId) {
|
||||
WikiEntityRelationEntity existing = relationMapper.selectOne(
|
||||
new LambdaQueryWrapper<WikiEntityRelationEntity>()
|
||||
.eq(WikiEntityRelationEntity::getKbId, kbId)
|
||||
.eq(WikiEntityRelationEntity::getSubjectEntityId, subjectId)
|
||||
.eq(WikiEntityRelationEntity::getPredicate, predicate)
|
||||
.eq(WikiEntityRelationEntity::getObjectEntityId, objectId)
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
return;
|
||||
}
|
||||
WikiEntityRelationEntity rel = new WikiEntityRelationEntity();
|
||||
rel.setKbId(kbId);
|
||||
rel.setSubjectEntityId(subjectId);
|
||||
rel.setPredicate(predicate);
|
||||
rel.setObjectEntityId(objectId);
|
||||
rel.setEvidence(truncate(evidence, MAX_EVIDENCE));
|
||||
rel.setConfidence(BigDecimal.valueOf(0.8));
|
||||
rel.setSource("llm-extracted");
|
||||
rel.setEvidenceChunkId(chunkId);
|
||||
relationMapper.insert(rel);
|
||||
}
|
||||
|
||||
// ---- helpers ----------------------------------------------------------
|
||||
|
||||
private boolean hasMentions(Long chunkId) {
|
||||
Long count = mentionMapper.selectCount(new LambdaQueryWrapper<WikiEntityMentionEntity>()
|
||||
.eq(WikiEntityMentionEntity::getChunkId, chunkId));
|
||||
return count != null && count > 0;
|
||||
}
|
||||
|
||||
private Long firstCitingPage(Long chunkId) {
|
||||
List<Long> pages = citationMapper.listPageIdsByChunkId(chunkId);
|
||||
return (pages == null || pages.isEmpty()) ? null : pages.get(0);
|
||||
}
|
||||
|
||||
private float[] embedName(Long kbId, EntityExtractionResult.ExtractedEntity e) {
|
||||
try {
|
||||
String text = e.getName() + (e.getDescription() == null ? "" : ". " + e.getDescription());
|
||||
return embeddingService.embedQuery(kbId, text);
|
||||
} catch (Exception ex) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private ChatModel resolveChatModel(Long kbId) {
|
||||
try {
|
||||
Long modelId = routingService.selectModelId(kbId, "heavy_ingest", WikiJobStep.ENTITY_EXTRACTION);
|
||||
if (modelId != null) {
|
||||
return routingService.buildChatModel(modelId);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[WikiEntity] Model routing failed for kbId={}, using default: {}", kbId, e.getMessage());
|
||||
}
|
||||
var def = modelConfigService.getDefaultModel();
|
||||
if (def == null) {
|
||||
return null;
|
||||
}
|
||||
return routingService.buildChatModel(def.getId());
|
||||
}
|
||||
|
||||
private List<String> resolveEntityTypes(WikiKnowledgeBaseEntity kb) {
|
||||
if (kb.getConfigContent() != null) {
|
||||
WikiKbConfig config = WikiKbConfigParser.parse(objectMapper, kb.getConfigContent());
|
||||
if (config != null && config.getEntityTypes() != null && !config.getEntityTypes().isEmpty()) {
|
||||
return config.getEntityTypes();
|
||||
}
|
||||
}
|
||||
return DEFAULT_ENTITY_TYPES;
|
||||
}
|
||||
|
||||
private String normalize(String s) {
|
||||
return s == null ? "" : s.trim().toLowerCase(Locale.ROOT).replaceAll("\\s+", " ");
|
||||
}
|
||||
|
||||
private String normalizeType(String type) {
|
||||
String t = type == null ? "" : type.trim().toLowerCase(Locale.ROOT);
|
||||
return t.isEmpty() ? "other" : t;
|
||||
}
|
||||
|
||||
private String normalizePredicate(String predicate) {
|
||||
String p = predicate.trim().toLowerCase(Locale.ROOT).replaceAll("\\s+", "_");
|
||||
return p.length() > 64 ? p.substring(0, 64) : p;
|
||||
}
|
||||
|
||||
private String truncate(String s, int max) {
|
||||
if (s == null) {
|
||||
return null;
|
||||
}
|
||||
String t = s.trim();
|
||||
return t.length() > max ? t.substring(0, max) : t;
|
||||
}
|
||||
|
||||
private String writeJson(Object value) {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory index of same-type entity embeddings for near-duplicate merge
|
||||
* within a single extraction run. Bounded by the KB's existing entity count.
|
||||
*/
|
||||
private final class EntityIndex {
|
||||
private final Map<String, List<float[]>> vectorsByType = new HashMap<>();
|
||||
private final Map<String, List<Long>> idsByType = new HashMap<>();
|
||||
|
||||
EntityIndex(Long kbId) {
|
||||
List<WikiEntityEntity> existing = entityMapper.selectList(
|
||||
new LambdaQueryWrapper<WikiEntityEntity>()
|
||||
.eq(WikiEntityEntity::getKbId, kbId)
|
||||
.isNotNull(WikiEntityEntity::getEmbedding));
|
||||
for (WikiEntityEntity e : existing) {
|
||||
if (e.getEmbedding() != null) {
|
||||
add(e.getType(), e.getId(), WikiEmbeddingService.bytesToFloats(e.getEmbedding()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void add(String type, Long id, float[] vec) {
|
||||
if (vec == null) {
|
||||
return;
|
||||
}
|
||||
vectorsByType.computeIfAbsent(type, k -> new ArrayList<>()).add(vec);
|
||||
idsByType.computeIfAbsent(type, k -> new ArrayList<>()).add(id);
|
||||
}
|
||||
|
||||
Long findNearest(String type, float[] vec) {
|
||||
List<float[]> vectors = vectorsByType.get(type);
|
||||
List<Long> ids = idsByType.get(type);
|
||||
if (vectors == null || vectors.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
float best = MERGE_THRESHOLD;
|
||||
Long bestId = null;
|
||||
for (int i = 0; i < vectors.size(); i++) {
|
||||
float sim = WikiEmbeddingService.cosine(vec, vectors.get(i));
|
||||
if (sim >= best) {
|
||||
best = sim;
|
||||
bestId = ids.get(i);
|
||||
}
|
||||
}
|
||||
return bestId;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,177 @@
|
||||
package vip.mate.wiki.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
import vip.mate.wiki.dto.WikiEntityGraphView;
|
||||
import vip.mate.wiki.dto.WikiEntityView;
|
||||
import vip.mate.wiki.model.WikiEntityEntity;
|
||||
import vip.mate.wiki.model.WikiEntityMentionEntity;
|
||||
import vip.mate.wiki.model.WikiEntityRelationEntity;
|
||||
import vip.mate.wiki.model.WikiPageEntity;
|
||||
import vip.mate.wiki.repository.WikiEntityMapper;
|
||||
import vip.mate.wiki.repository.WikiEntityMentionMapper;
|
||||
import vip.mate.wiki.repository.WikiEntityRelationMapper;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Read-side queries over the entity-level knowledge graph: entity listing and
|
||||
* single-entity ego-graph assembly for the wiki graph view.
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class WikiEntityGraphService {
|
||||
|
||||
private final WikiEntityMapper entityMapper;
|
||||
private final WikiEntityMentionMapper mentionMapper;
|
||||
private final WikiEntityRelationMapper relationMapper;
|
||||
private final WikiPageService pageService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
/** List entities in a KB, optionally filtered by type, ranked by salience. */
|
||||
public List<WikiEntityView> listEntities(Long kbId, String type, int limit) {
|
||||
LambdaQueryWrapper<WikiEntityEntity> q = new LambdaQueryWrapper<WikiEntityEntity>()
|
||||
.eq(WikiEntityEntity::getKbId, kbId)
|
||||
.orderByDesc(WikiEntityEntity::getSalience)
|
||||
.last("LIMIT " + Math.max(1, Math.min(limit, 500)));
|
||||
if (type != null && !type.isBlank()) {
|
||||
q.eq(WikiEntityEntity::getType, type.trim().toLowerCase());
|
||||
}
|
||||
List<WikiEntityView> out = new ArrayList<>();
|
||||
for (WikiEntityEntity e : entityMapper.selectList(q)) {
|
||||
out.add(toView(e));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the whole-KB entity graph: the top entities by salience plus the
|
||||
* relation edges that connect any two of them.
|
||||
*/
|
||||
public WikiEntityGraphView graph(Long kbId, int limit) {
|
||||
WikiEntityGraphView view = new WikiEntityGraphView();
|
||||
List<WikiEntityView> nodes = listEntities(kbId, null, limit);
|
||||
view.setNodes(nodes);
|
||||
if (nodes.isEmpty()) {
|
||||
return view;
|
||||
}
|
||||
Set<Long> ids = new LinkedHashSet<>();
|
||||
for (WikiEntityView n : nodes) {
|
||||
ids.add(n.getId());
|
||||
}
|
||||
List<WikiEntityRelationEntity> rels = relationMapper.selectList(
|
||||
new LambdaQueryWrapper<WikiEntityRelationEntity>()
|
||||
.eq(WikiEntityRelationEntity::getKbId, kbId)
|
||||
.last("LIMIT " + Math.max(1, Math.min(limit * 5, 2000))));
|
||||
for (WikiEntityRelationEntity r : rels) {
|
||||
if (!ids.contains(r.getSubjectEntityId()) || !ids.contains(r.getObjectEntityId())) {
|
||||
continue;
|
||||
}
|
||||
WikiEntityGraphView.Edge edge = new WikiEntityGraphView.Edge();
|
||||
edge.setId(r.getId());
|
||||
edge.setSubjectEntityId(r.getSubjectEntityId());
|
||||
edge.setPredicate(r.getPredicate());
|
||||
edge.setObjectEntityId(r.getObjectEntityId());
|
||||
edge.setEvidence(r.getEvidence());
|
||||
edge.setConfidence(r.getConfidence());
|
||||
view.getEdges().add(edge);
|
||||
}
|
||||
return view;
|
||||
}
|
||||
|
||||
/** Assemble the ego-graph around one entity. */
|
||||
public WikiEntityGraphView ego(Long kbId, Long entityId, int limit) {
|
||||
WikiEntityGraphView view = new WikiEntityGraphView();
|
||||
WikiEntityEntity center = entityMapper.selectById(entityId);
|
||||
if (center == null || !center.getKbId().equals(kbId)) {
|
||||
return view;
|
||||
}
|
||||
view.setCenter(toView(center));
|
||||
|
||||
int edgeLimit = Math.max(1, Math.min(limit, 200));
|
||||
List<WikiEntityRelationEntity> edges = relationMapper.selectList(
|
||||
new LambdaQueryWrapper<WikiEntityRelationEntity>()
|
||||
.eq(WikiEntityRelationEntity::getKbId, kbId)
|
||||
.and(w -> w.eq(WikiEntityRelationEntity::getSubjectEntityId, entityId)
|
||||
.or().eq(WikiEntityRelationEntity::getObjectEntityId, entityId))
|
||||
.last("LIMIT " + edgeLimit));
|
||||
|
||||
Set<Long> neighborIds = new LinkedHashSet<>();
|
||||
for (WikiEntityRelationEntity r : edges) {
|
||||
WikiEntityGraphView.Edge edge = new WikiEntityGraphView.Edge();
|
||||
edge.setId(r.getId());
|
||||
edge.setSubjectEntityId(r.getSubjectEntityId());
|
||||
edge.setPredicate(r.getPredicate());
|
||||
edge.setObjectEntityId(r.getObjectEntityId());
|
||||
edge.setEvidence(r.getEvidence());
|
||||
edge.setConfidence(r.getConfidence());
|
||||
view.getEdges().add(edge);
|
||||
neighborIds.add(r.getSubjectEntityId());
|
||||
neighborIds.add(r.getObjectEntityId());
|
||||
}
|
||||
neighborIds.remove(entityId);
|
||||
if (!neighborIds.isEmpty()) {
|
||||
for (WikiEntityEntity n : entityMapper.selectBatchIds(neighborIds)) {
|
||||
view.getNodes().add(toView(n));
|
||||
}
|
||||
}
|
||||
|
||||
// Pages that mention the center entity — the bridge to the page layer.
|
||||
List<WikiEntityMentionEntity> mentions = mentionMapper.selectList(
|
||||
new LambdaQueryWrapper<WikiEntityMentionEntity>()
|
||||
.eq(WikiEntityMentionEntity::getEntityId, entityId)
|
||||
.isNotNull(WikiEntityMentionEntity::getPageId)
|
||||
.last("LIMIT 200"));
|
||||
Set<Long> pageIds = new LinkedHashSet<>();
|
||||
for (WikiEntityMentionEntity m : mentions) {
|
||||
pageIds.add(m.getPageId());
|
||||
}
|
||||
for (Long pageId : pageIds) {
|
||||
WikiPageEntity page = pageService.getById(pageId);
|
||||
if (page == null) {
|
||||
continue;
|
||||
}
|
||||
WikiEntityGraphView.PageRef ref = new WikiEntityGraphView.PageRef();
|
||||
ref.setPageId(page.getId());
|
||||
ref.setSlug(page.getSlug());
|
||||
ref.setTitle(page.getTitle());
|
||||
view.getPages().add(ref);
|
||||
}
|
||||
return view;
|
||||
}
|
||||
|
||||
private WikiEntityView toView(WikiEntityEntity e) {
|
||||
WikiEntityView v = new WikiEntityView();
|
||||
v.setId(e.getId());
|
||||
v.setKbId(e.getKbId());
|
||||
v.setCanonicalName(e.getCanonicalName());
|
||||
v.setType(e.getType());
|
||||
v.setDescription(e.getDescription());
|
||||
v.setSalience(e.getSalience());
|
||||
v.setMentionCount(e.getMentionCount());
|
||||
v.setAliases(parseAliases(e.getAliasesJson()));
|
||||
return v;
|
||||
}
|
||||
|
||||
private List<String> parseAliases(String json) {
|
||||
if (json == null || json.isBlank()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
try {
|
||||
return objectMapper.readValue(json, new TypeReference<List<String>>() {});
|
||||
} catch (Exception e) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,7 +1,9 @@
|
||||
package vip.mate.wiki.service;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
@ -10,24 +12,49 @@ import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.converter.BeanOutputConverter;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.context.annotation.Lazy;
|
||||
import org.springframework.dao.DuplicateKeyException;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
import vip.mate.agent.AgentGraphBuilder;
|
||||
import vip.mate.agent.prompt.PromptLoader;
|
||||
import vip.mate.llm.failover.ProviderHealthTracker;
|
||||
import vip.mate.llm.model.ModelConfigEntity;
|
||||
import vip.mate.llm.service.ModelConfigService;
|
||||
import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.wiki.WikiProperties;
|
||||
import vip.mate.wiki.dto.RouteResult;
|
||||
import vip.mate.wiki.dto.RoutedPageMeta;
|
||||
import vip.mate.wiki.dto.WikiChunkDraft;
|
||||
import vip.mate.wiki.event.WikiFactPageUpdatedEvent;
|
||||
import vip.mate.wiki.event.WikiKbDirtyEvent;
|
||||
import vip.mate.wiki.event.WikiPageCreatedEvent;
|
||||
import vip.mate.wiki.event.WikiProcessingEvent;
|
||||
import vip.mate.wiki.job.WikiEmbeddingProviderFailingException;
|
||||
import vip.mate.wiki.job.WikiJobStage;
|
||||
import vip.mate.wiki.job.WikiJobStep;
|
||||
import vip.mate.wiki.job.WikiKbConfig;
|
||||
import vip.mate.wiki.job.WikiKbConfigParser;
|
||||
import vip.mate.wiki.job.WikiModelRoutingService;
|
||||
import vip.mate.wiki.job.WikiProcessingJobService;
|
||||
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
|
||||
import vip.mate.wiki.model.WikiPageEntity;
|
||||
import vip.mate.wiki.model.WikiRawMaterialEntity;
|
||||
import vip.mate.wiki.profile.WikiMetadataValidator;
|
||||
import vip.mate.wiki.profile.WikiPageTypeDef;
|
||||
import vip.mate.wiki.profile.WikiPageTypeProfile;
|
||||
import vip.mate.wiki.profile.WikiPageTypeProfileService;
|
||||
import vip.mate.wiki.sse.WikiProgressBus;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@ -60,23 +87,24 @@ public class WikiProcessingService {
|
||||
private final ObjectMapper objectMapper;
|
||||
private final WikiProgressBus progressBus;
|
||||
private final WikiCitationService citationService;
|
||||
private final org.springframework.context.ApplicationEventPublisher eventPublisher;
|
||||
private final ApplicationEventPublisher eventPublisher;
|
||||
private final WikiEntityExtractionService entityExtractionService;
|
||||
|
||||
/**
|
||||
* Optional KB pageType profile. Field-injected (not a constructor arg) so
|
||||
* existing instantiations are unaffected; when absent the batch-create
|
||||
* prompt falls back to the legacy hardcoded pageType enum.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private vip.mate.wiki.profile.WikiPageTypeProfileService pageTypeProfileService;
|
||||
@Autowired(required = false)
|
||||
private WikiPageTypeProfileService pageTypeProfileService;
|
||||
|
||||
/** Optional metadata validator, paired with {@link #pageTypeProfileService}. */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private vip.mate.wiki.profile.WikiMetadataValidator metadataValidator;
|
||||
@Autowired(required = false)
|
||||
private WikiMetadataValidator metadataValidator;
|
||||
|
||||
/** Optional dependency/stale engine for layered-knowledge wiring. */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private vip.mate.wiki.service.WikiDependencyService dependencyService;
|
||||
@Autowired(required = false)
|
||||
private WikiDependencyService dependencyService;
|
||||
|
||||
/**
|
||||
* Read-the-failover-chain handle. Optional so the existing constructors and
|
||||
@ -84,8 +112,8 @@ public class WikiProcessingService {
|
||||
* fallback hop iterates {@code listEnabledModels} in DB order — same
|
||||
* behavior as before this PR.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private vip.mate.llm.service.ModelProviderService modelProviderService;
|
||||
@Autowired(required = false)
|
||||
private ModelProviderService modelProviderService;
|
||||
|
||||
/**
|
||||
* Per-provider failure counter / cooldown bookkeeping. Optional for the
|
||||
@ -94,12 +122,12 @@ public class WikiProcessingService {
|
||||
* successful call we clear the failure counter for the provider that
|
||||
* actually responded.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker;
|
||||
@Autowired(required = false)
|
||||
private ProviderHealthTracker providerHealthTracker;
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@org.springframework.context.annotation.Lazy
|
||||
private vip.mate.wiki.job.WikiProcessingJobService wikiJobService;
|
||||
@Autowired(required = false)
|
||||
@Lazy
|
||||
private WikiProcessingJobService wikiJobService;
|
||||
|
||||
/**
|
||||
* RFC-051 PR-1c: optional preprocessor that fills chunk metadata
|
||||
@ -107,7 +135,7 @@ public class WikiProcessingService {
|
||||
* Marked optional so unit tests that construct this service directly
|
||||
* (without Spring) can opt out without exploding.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@Autowired(required = false)
|
||||
private DocumentPreprocessService preprocessService;
|
||||
|
||||
/**
|
||||
@ -115,7 +143,7 @@ public class WikiProcessingService {
|
||||
* the KB before each ingest. Optional so the older lazy-only unit tests
|
||||
* don't need to wire it.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@Autowired(required = false)
|
||||
private WikiScaffoldService scaffoldService;
|
||||
|
||||
/**
|
||||
@ -124,7 +152,7 @@ public class WikiProcessingService {
|
||||
* without Spring don't need to supply it — when absent, the post-ingestion
|
||||
* auto-scan is simply skipped.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@Autowired(required = false)
|
||||
private WikiLintJobService lintJobService;
|
||||
|
||||
/**
|
||||
@ -133,14 +161,14 @@ public class WikiProcessingService {
|
||||
* routing chain (stepModels[step] -> wikiDefaultModelId -> system
|
||||
* default) for a model rather than always pulling the system default.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private vip.mate.wiki.job.WikiModelRoutingService modelRoutingService;
|
||||
@Autowired(required = false)
|
||||
private WikiModelRoutingService modelRoutingService;
|
||||
|
||||
/** RFC-051 PR-2b/2c: optional overview rebuilder + log appender. */
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@Autowired(required = false)
|
||||
private WikiOverviewService overviewService;
|
||||
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@Autowired(required = false)
|
||||
private WikiLogService logService;
|
||||
|
||||
/**
|
||||
@ -148,8 +176,8 @@ public class WikiProcessingService {
|
||||
* of the KB's apply-default transformation templates. Missing in the
|
||||
* legacy unit tests that wire this service directly.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
@org.springframework.context.annotation.Lazy
|
||||
@Autowired(required = false)
|
||||
@Lazy
|
||||
private WikiTransformationExecutor transformationExecutor;
|
||||
|
||||
/** Parallel chunk / material processing executor (JDK 21 virtual threads) */
|
||||
@ -203,7 +231,7 @@ public class WikiProcessingService {
|
||||
|
||||
/** KBs with a reclassify pass currently running, used to reject concurrent
|
||||
* re-triggers (which would double LLM spend and race page-type writes). */
|
||||
private final java.util.Set<Long> reclassifyInFlight = ConcurrentHashMap.newKeySet();
|
||||
private final Set<Long> reclassifyInFlight = ConcurrentHashMap.newKeySet();
|
||||
|
||||
/**
|
||||
* Process one raw material.
|
||||
@ -272,7 +300,7 @@ public class WikiProcessingService {
|
||||
try {
|
||||
var job = wikiJobService.createHeavyIngest(kb.getId(), rawId);
|
||||
jobId = job.getId();
|
||||
wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.ROUTING);
|
||||
wikiJobService.transition(jobId, WikiJobStage.ROUTING);
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Failed to create heavy ingest job record for raw={}: {}", rawId, e.getMessage());
|
||||
}
|
||||
@ -284,7 +312,7 @@ public class WikiProcessingService {
|
||||
|
||||
// RFC-012 M3:广播 raw.started(前端切到 indeterminate 进度条)
|
||||
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_STARTED,
|
||||
java.util.Map.of("rawId", rawId, "phase", "route"));
|
||||
Map.of("rawId", rawId, "phase", "route"));
|
||||
|
||||
try {
|
||||
// Phase 1: 获取文本内容
|
||||
@ -323,7 +351,7 @@ public class WikiProcessingService {
|
||||
|
||||
// Transition job to phase_a (chunk processing begins)
|
||||
if (wikiJobService != null && jobId != null) {
|
||||
try { wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.PHASE_A_RUNNING); } catch (Exception ignored) {}
|
||||
try { wikiJobService.transition(jobId, WikiJobStage.PHASE_A_RUNNING); } catch (Exception ignored) {}
|
||||
}
|
||||
|
||||
// Phase 3: LLM 消化
|
||||
@ -416,10 +444,10 @@ public class WikiProcessingService {
|
||||
// RFC-012 M3:广播终态
|
||||
if ("failed".equals(finalStatus)) {
|
||||
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED,
|
||||
java.util.Map.of("rawId", rawId, "error", finalDetail == null ? "" : finalDetail));
|
||||
Map.of("rawId", rawId, "error", finalDetail == null ? "" : finalDetail));
|
||||
} else {
|
||||
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED,
|
||||
java.util.Map.of(
|
||||
Map.of(
|
||||
"rawId", rawId,
|
||||
"status", finalStatus,
|
||||
"totalPages", totalPages,
|
||||
@ -430,10 +458,10 @@ public class WikiProcessingService {
|
||||
if (wikiJobService != null && jobId != null) {
|
||||
try {
|
||||
var terminalStage = switch (finalStatus) {
|
||||
case "failed" -> vip.mate.wiki.job.WikiJobStage.FAILED;
|
||||
case "partial" -> vip.mate.wiki.job.WikiJobStage.PARTIAL;
|
||||
case "cancelled" -> vip.mate.wiki.job.WikiJobStage.CANCELLED;
|
||||
default -> vip.mate.wiki.job.WikiJobStage.COMPLETED;
|
||||
case "failed" -> WikiJobStage.FAILED;
|
||||
case "partial" -> WikiJobStage.PARTIAL;
|
||||
case "cancelled" -> WikiJobStage.CANCELLED;
|
||||
default -> WikiJobStage.COMPLETED;
|
||||
};
|
||||
wikiJobService.transition(jobId, terminalStage);
|
||||
} catch (Exception ignored) {}
|
||||
@ -460,7 +488,7 @@ public class WikiProcessingService {
|
||||
// schedule (debounced) an LLM-generated overview narrative refresh.
|
||||
// Stats rebuild above is sync; narrative regen runs after-commit.
|
||||
if (nonTerminalSideEffects) {
|
||||
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kb.getId()));
|
||||
eventPublisher.publishEvent(new WikiKbDirtyEvent(this, kb.getId()));
|
||||
}
|
||||
|
||||
// Run apply-default transformation templates against the newly
|
||||
@ -495,7 +523,7 @@ public class WikiProcessingService {
|
||||
if (embedded > 0) {
|
||||
log.info("[Wiki] Async embedding completed: kbId={}, embedded={}", fKbId, embedded);
|
||||
}
|
||||
} catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) {
|
||||
} catch (WikiEmbeddingProviderFailingException ex) {
|
||||
// Circuit-breaker tripped — the provider has consistently failed.
|
||||
// The exception's own log line in WikiEmbeddingService is enough;
|
||||
// emit a calmer notice here instead of a generic failure log.
|
||||
@ -507,6 +535,26 @@ public class WikiProcessingService {
|
||||
});
|
||||
}
|
||||
|
||||
// Entity-level knowledge graph extraction — opt-in per KB. Runs as a
|
||||
// separate async pass so it never blocks (or fails) the ingest pipeline;
|
||||
// its inputs (chunks, citations) are already committed at this point.
|
||||
if (totalChunks > 0 && !"cancelled".equals(finalStatus)
|
||||
&& isEntityExtractionEnabled(kb)) {
|
||||
final Long fKbId = kb.getId();
|
||||
final Long fRawId = rawId;
|
||||
WIKI_EXECUTOR.submit(() -> {
|
||||
try {
|
||||
int count = entityExtractionService.extractForRaw(fKbId, fRawId);
|
||||
if (count > 0) {
|
||||
log.info("[Wiki] Async entity extraction completed: kbId={}, rawId={}, entities={}",
|
||||
fKbId, fRawId, count);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
log.warn("[Wiki] Async entity extraction failed for kbId={}: {}", fKbId, ex.getMessage());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
// If the user requested cancellation while this run was in flight,
|
||||
// surface the abort as 'cancelled' rather than 'failed' even when
|
||||
@ -527,8 +575,8 @@ public class WikiProcessingService {
|
||||
if (wikiJobService != null && jobId != null) {
|
||||
try {
|
||||
wikiJobService.transition(jobId, cancelled
|
||||
? vip.mate.wiki.job.WikiJobStage.CANCELLED
|
||||
: vip.mate.wiki.job.WikiJobStage.FAILED);
|
||||
? WikiJobStage.CANCELLED
|
||||
: WikiJobStage.FAILED);
|
||||
} catch (Exception ignored) {}
|
||||
}
|
||||
// Broadcast: cancelled rows reuse the COMPLETED event with status="cancelled"
|
||||
@ -536,10 +584,10 @@ public class WikiProcessingService {
|
||||
// go through RAW_FAILED (which the UI surfaces as a red banner).
|
||||
if (cancelled) {
|
||||
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED,
|
||||
java.util.Map.of("rawId", rawId, "status", "cancelled"));
|
||||
Map.of("rawId", rawId, "status", "cancelled"));
|
||||
} else {
|
||||
progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED,
|
||||
java.util.Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage()));
|
||||
Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage()));
|
||||
}
|
||||
} finally {
|
||||
// RFC-012 M2 v2 UI v2:写入最终进度并清理共享计数器
|
||||
@ -665,7 +713,7 @@ public class WikiProcessingService {
|
||||
if (modelId != null && modelRoutingService != null) {
|
||||
chatModel = modelRoutingService.buildChatModel(modelId);
|
||||
} else {
|
||||
chatModel = resolveChatModel(kbId, vip.mate.wiki.job.WikiJobStep.CREATE_PAGE).chatModel;
|
||||
chatModel = resolveChatModel(kbId, WikiJobStep.CREATE_PAGE).chatModel;
|
||||
}
|
||||
systemPrompt = PromptLoader.loadPrompt("wiki/classify-page-system")
|
||||
.replace("{allowed_page_types}", pageTypeProfileService.describeForPrompt(kbId));
|
||||
@ -714,13 +762,13 @@ public class WikiProcessingService {
|
||||
page.getId(), kbId, e.getMessage());
|
||||
} finally {
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
|
||||
java.util.Map.of("kind", "reclassify", "done", done, "total", total));
|
||||
Map.of("kind", "reclassify", "done", done, "total", total));
|
||||
}
|
||||
}
|
||||
log.info("[Wiki] reclassifyKB done kbId={} pages={} changed={} failed={}",
|
||||
kbId, total, changed, failed);
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED,
|
||||
java.util.Map.of("kind", "reclassify", "done", total, "total", total,
|
||||
Map.of("kind", "reclassify", "done", total, "total", total,
|
||||
"changed", changed, "failed", failed));
|
||||
} finally {
|
||||
reclassifyInFlight.remove(kbId);
|
||||
@ -950,7 +998,7 @@ public class WikiProcessingService {
|
||||
));
|
||||
if (isAborted(raw.getId(), "single-chunk legacy")) return 0;
|
||||
String llmResponse = callLlmWithResilientRetry(prompt, "chunk of raw=" + raw.getId(),
|
||||
kb.getId(), vip.mate.wiki.job.WikiJobStep.CREATE_PAGE);
|
||||
kb.getId(), WikiJobStep.CREATE_PAGE);
|
||||
|
||||
return applyLlmResponse(kb.getId(), raw.getId(), llmResponse);
|
||||
}
|
||||
@ -1004,9 +1052,9 @@ public class WikiProcessingService {
|
||||
// LLM produces strict RouteResult JSON. KB config wins; falls back to global
|
||||
// mate.wiki.use-structured-route default when the KB hasn't expressed a preference.
|
||||
boolean useStructured = resolveStructuredRouteFlag(kb);
|
||||
org.springframework.ai.converter.BeanOutputConverter<vip.mate.wiki.dto.RouteResult> routeConverter =
|
||||
BeanOutputConverter<RouteResult> routeConverter =
|
||||
useStructured
|
||||
? new org.springframework.ai.converter.BeanOutputConverter<>(vip.mate.wiki.dto.RouteResult.class)
|
||||
? new BeanOutputConverter<>(RouteResult.class)
|
||||
: null;
|
||||
if (routeConverter != null) {
|
||||
routeUser = routeUser + "\n\n" + routeConverter.getFormat();
|
||||
@ -1018,7 +1066,7 @@ public class WikiProcessingService {
|
||||
));
|
||||
if (isAborted(rawId, "route phase")) return 0;
|
||||
String routeResponse = callLlmWithResilientRetry(routePrompt, "route chunk of raw=" + rawId,
|
||||
kbId, vip.mate.wiki.job.WikiJobStep.ROUTE);
|
||||
kbId, WikiJobStep.ROUTE);
|
||||
|
||||
// RFC-012 follow-up #3:phase B 现在并行执行,计数必须是 atomic
|
||||
AtomicInteger created = new AtomicInteger(0);
|
||||
@ -1031,12 +1079,12 @@ public class WikiProcessingService {
|
||||
boolean structuredOk = false;
|
||||
if (routeConverter != null) {
|
||||
try {
|
||||
vip.mate.wiki.dto.RouteResult bound = routeConverter.convert(routeResponse);
|
||||
RouteResult bound = routeConverter.convert(routeResponse);
|
||||
if (bound != null) {
|
||||
for (vip.mate.wiki.dto.RoutedPageMeta meta : bound.create()) {
|
||||
for (RoutedPageMeta meta : bound.create()) {
|
||||
if (meta == null || meta.slug() == null || meta.slug().isBlank()
|
||||
|| meta.title() == null || meta.title().isBlank()) continue;
|
||||
com.fasterxml.jackson.databind.node.ObjectNode node = objectMapper.createObjectNode();
|
||||
ObjectNode node = objectMapper.createObjectNode();
|
||||
node.put("slug", meta.slug());
|
||||
node.put("title", meta.title());
|
||||
if (meta.summary() != null) node.put("summary", meta.summary());
|
||||
@ -1079,7 +1127,7 @@ public class WikiProcessingService {
|
||||
));
|
||||
String retryResponse = callLlmWithResilientRetry(retryPrompt,
|
||||
"route chunk RETRY of raw=" + rawId,
|
||||
kbId, vip.mate.wiki.job.WikiJobStep.ROUTE);
|
||||
kbId, WikiJobStep.ROUTE);
|
||||
routeJson = parseJsonResponse(retryResponse);
|
||||
if (routeJson == null) {
|
||||
log.warn("[Wiki] Route phase: failed to parse JSON for kbId={}, rawId={}, responseLen={}, first200={}",
|
||||
@ -1112,7 +1160,7 @@ public class WikiProcessingService {
|
||||
// so no content is silently dropped (mirrors llm_wiki source-summary guarantee).
|
||||
if (totalPlanned == 0 && textContent.length() >= properties.getChunkFallbackMinChars()) {
|
||||
String overviewSlug = WikiPageService.toSlug(rawTitle) + "-overview";
|
||||
com.fasterxml.jackson.databind.node.ObjectNode fallbackMeta =
|
||||
ObjectNode fallbackMeta =
|
||||
objectMapper.createObjectNode();
|
||||
fallbackMeta.put("slug", overviewSlug);
|
||||
fallbackMeta.put("title", rawTitle + " 概述");
|
||||
@ -1133,7 +1181,7 @@ public class WikiProcessingService {
|
||||
log.info("[Wiki] Progress: switching to phase-b for raw={}", rawId);
|
||||
// RFC-012 M3:route 完成、phase-b 启动 → 通知前端确定进度(可显示 0/N)
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_ROUTE_DONE,
|
||||
java.util.Map.of(
|
||||
Map.of(
|
||||
"rawId", rawId,
|
||||
"phase", "phase-b",
|
||||
"done", pc.done.get(),
|
||||
@ -1195,7 +1243,7 @@ public class WikiProcessingService {
|
||||
if (!ok) pc.failed.incrementAndGet();
|
||||
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
|
||||
java.util.Map.of(
|
||||
Map.of(
|
||||
"rawId", rawId,
|
||||
"kind", "merge",
|
||||
"ok", ok,
|
||||
@ -1288,7 +1336,7 @@ public class WikiProcessingService {
|
||||
String batchResponse = callLlmWithResilientRetry(batchPrompt,
|
||||
"batch-create " + subBatch.size() + " pages of raw=" + rawId
|
||||
+ " subBatch=" + (bStart / batchSize + 1),
|
||||
kbId, vip.mate.wiki.job.WikiJobStep.CREATE_PAGE);
|
||||
kbId, WikiJobStep.CREATE_PAGE);
|
||||
|
||||
List<WikiBatchCreateParser.ParsedPage> parsedPages = batchParser.parse(batchResponse);
|
||||
|
||||
@ -1328,7 +1376,7 @@ public class WikiProcessingService {
|
||||
pc.failed.incrementAndGet();
|
||||
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
|
||||
java.util.Map.of("rawId", rawId, "kind", "create",
|
||||
Map.of("rawId", rawId, "kind", "create",
|
||||
"ok", false, "done", d, "total", pc.total.get()));
|
||||
}
|
||||
continue;
|
||||
@ -1377,7 +1425,7 @@ public class WikiProcessingService {
|
||||
pc.failed.incrementAndGet();
|
||||
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
|
||||
java.util.Map.of("rawId", rawId, "kind", "create-retry",
|
||||
Map.of("rawId", rawId, "kind", "create-retry",
|
||||
"ok", false, "done", d, "total", pc.total.get()));
|
||||
}
|
||||
continue;
|
||||
@ -1422,7 +1470,7 @@ public class WikiProcessingService {
|
||||
if (!ok) pc.failed.incrementAndGet();
|
||||
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
|
||||
java.util.Map.of(
|
||||
Map.of(
|
||||
"rawId", rawId,
|
||||
"kind", "create",
|
||||
"ok", ok,
|
||||
@ -1433,7 +1481,7 @@ public class WikiProcessingService {
|
||||
|
||||
// Retry any pages that LLM omitted from the batch response
|
||||
int subBatchNum = bStart / batchSize + 1;
|
||||
java.util.Set<String> returnedSlugs = new java.util.HashSet<>();
|
||||
Set<String> returnedSlugs = new HashSet<>();
|
||||
for (WikiBatchCreateParser.ParsedPage pp : parsedPages) {
|
||||
returnedSlugs.add(pp.slug());
|
||||
}
|
||||
@ -1481,7 +1529,7 @@ public class WikiProcessingService {
|
||||
));
|
||||
if (isAborted(raw.getId(), "retry-create slug=" + slug)) return null;
|
||||
return callLlmWithResilientRetry(prompt, "retry-create slug=" + slug + " of raw=" + raw.getId(),
|
||||
kb.getId(), vip.mate.wiki.job.WikiJobStep.CREATE_PAGE);
|
||||
kb.getId(), WikiJobStep.CREATE_PAGE);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1572,7 +1620,7 @@ public class WikiProcessingService {
|
||||
if (delta < 0) pc.failed.incrementAndGet();
|
||||
rawService.updateProgress(rawId, "phase-b", d, pc.total.get());
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_CHUNK_DONE,
|
||||
java.util.Map.of("rawId", rawId, "kind", "create-retry",
|
||||
Map.of("rawId", rawId, "kind", "create-retry",
|
||||
"ok", delta >= 0, "done", d, "total", pc.total.get()));
|
||||
}
|
||||
return delta;
|
||||
@ -1723,7 +1771,7 @@ public class WikiProcessingService {
|
||||
log.info("[Wiki] Phase B create page slug='{}' done (created)", slug);
|
||||
citationService.buildCitationsAsync(created.getId(), kbId);
|
||||
return true;
|
||||
} catch (org.springframework.dao.DuplicateKeyException e) {
|
||||
} catch (DuplicateKeyException e) {
|
||||
// Fallback 2: concurrent INSERT race — degrade to update
|
||||
pageService.updatePageByAi(kbId, slug, content, pageSummary, rawId);
|
||||
WikiPageEntity raced = pageService.getBySlug(kbId, slug);
|
||||
@ -1756,13 +1804,13 @@ public class WikiProcessingService {
|
||||
// transactions), so the count is accurate. Idempotent + dedup-guarded
|
||||
// downstream, so firing on update paths is safe.
|
||||
if (eventPublisher != null && pageType != null && !pageType.isBlank()) {
|
||||
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiPageCreatedEvent(kbId, pageType, pageId));
|
||||
eventPublisher.publishEvent(new WikiPageCreatedEvent(kbId, pageType, pageId));
|
||||
}
|
||||
// When an existing fact page is updated, propagate staleness to the
|
||||
// experience pages depending on it (async, off the ingest thread).
|
||||
if (isUpdate && eventPublisher != null && dependencyService != null
|
||||
&& pageTypeProfileService != null && !pageTypeProfileService.isExperience(kbId, pageType)) {
|
||||
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiFactPageUpdatedEvent(
|
||||
eventPublisher.publishEvent(new WikiFactPageUpdatedEvent(
|
||||
kbId, pageId, "fact page updated during ingest"));
|
||||
}
|
||||
}
|
||||
@ -1792,7 +1840,7 @@ public class WikiProcessingService {
|
||||
if (!pageTypeProfileService.isExperience(kbId, pageType)) {
|
||||
return; // only experience pages declare fact dependencies
|
||||
}
|
||||
java.util.List<Long> depIds = new java.util.ArrayList<>();
|
||||
List<Long> depIds = new ArrayList<>();
|
||||
for (JsonNode n : dependsOnNode) {
|
||||
String slug = n.asText("");
|
||||
if (slug.isBlank()) continue;
|
||||
@ -1802,7 +1850,7 @@ public class WikiProcessingService {
|
||||
}
|
||||
}
|
||||
try {
|
||||
java.util.List<String> rejected = dependencyService.setDependencies(kbId, pageId, depIds);
|
||||
List<String> rejected = dependencyService.setDependencies(kbId, pageId, depIds);
|
||||
if (!rejected.isEmpty()) {
|
||||
log.warn("[Wiki] page {} dependency warnings: {}", pageId, rejected);
|
||||
}
|
||||
@ -1853,11 +1901,11 @@ public class WikiProcessingService {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
vip.mate.wiki.profile.WikiPageTypeProfile profile = pageTypeProfileService.resolveProfile(kbId);
|
||||
vip.mate.wiki.profile.WikiPageTypeDef def = profile.get(pageType);
|
||||
WikiPageTypeProfile profile = pageTypeProfileService.resolveProfile(kbId);
|
||||
WikiPageTypeDef def = profile.get(pageType);
|
||||
@SuppressWarnings("unchecked")
|
||||
java.util.Map<String, Object> raw = objectMapper.convertValue(metadataNode, java.util.Map.class);
|
||||
vip.mate.wiki.profile.WikiMetadataValidator.ValidationResult result =
|
||||
Map<String, Object> raw = objectMapper.convertValue(metadataNode, Map.class);
|
||||
WikiMetadataValidator.ValidationResult result =
|
||||
metadataValidator.validate(def, raw, profile.isAllowAdditionalFields(), "create");
|
||||
String metadataJson = objectMapper.writeValueAsString(result.getCleaned());
|
||||
String validationJson = result.getWarnings().isEmpty()
|
||||
@ -1925,7 +1973,7 @@ public class WikiProcessingService {
|
||||
if (isAborted(rawId, "merge slug=" + slug)) return false;
|
||||
String response = callLlmWithResilientRetry(prompt,
|
||||
"merge page slug=" + slug + " of raw=" + rawId,
|
||||
kbId, vip.mate.wiki.job.WikiJobStep.MERGE_PAGE);
|
||||
kbId, WikiJobStep.MERGE_PAGE);
|
||||
JsonNode mergeJson = parseJsonResponse(response);
|
||||
if (mergeJson == null) {
|
||||
log.warn("[Wiki] Phase B merge page slug='{}' returned unparseable JSON, skipping", slug);
|
||||
@ -2053,7 +2101,7 @@ public class WikiProcessingService {
|
||||
try {
|
||||
if (isAborted(raw.getId(), "doc analysis")) return "";
|
||||
String response = callLlmWithResilientRetry(prompt, "analyze doc raw=" + raw.getId(),
|
||||
kb.getId(), vip.mate.wiki.job.WikiJobStep.ROUTE);
|
||||
kb.getId(), WikiJobStep.ROUTE);
|
||||
JsonNode json = parseJsonResponse(response);
|
||||
if (json != null) {
|
||||
// Validate related_pages against the active KB slug set BEFORE
|
||||
@ -2120,7 +2168,7 @@ public class WikiProcessingService {
|
||||
JsonNode relatedNode = analysisJson.path("related_pages");
|
||||
if (!relatedNode.isArray() || relatedNode.size() == 0) return analysisJson;
|
||||
|
||||
java.util.Set<String> activeSlugs;
|
||||
Set<String> activeSlugs;
|
||||
try {
|
||||
activeSlugs = linkService.lowercaseSlugSet(pageService.listSummaries(kbId));
|
||||
} catch (RuntimeException e) {
|
||||
@ -2134,12 +2182,12 @@ public class WikiProcessingService {
|
||||
return result;
|
||||
}
|
||||
|
||||
com.fasterxml.jackson.databind.node.ArrayNode keptArray = objectMapper.createArrayNode();
|
||||
java.util.List<String> dropped = new java.util.ArrayList<>();
|
||||
ArrayNode keptArray = objectMapper.createArrayNode();
|
||||
List<String> dropped = new ArrayList<>();
|
||||
for (JsonNode el : relatedNode) {
|
||||
String slug = el.asText("").trim();
|
||||
if (slug.isEmpty()) continue;
|
||||
if (activeSlugs.contains(slug.toLowerCase(java.util.Locale.ROOT))) {
|
||||
if (activeSlugs.contains(slug.toLowerCase(Locale.ROOT))) {
|
||||
keptArray.add(slug);
|
||||
} else {
|
||||
dropped.add(slug);
|
||||
@ -2257,7 +2305,7 @@ public class WikiProcessingService {
|
||||
* is available. Falls back to the system default on any lookup failure
|
||||
* so a misconfigured KB never blocks ingest.
|
||||
*/
|
||||
private ChatModel buildChatModelFor(Long kbId, vip.mate.wiki.job.WikiJobStep step) {
|
||||
private ChatModel buildChatModelFor(Long kbId, WikiJobStep step) {
|
||||
if (modelRoutingService != null && kbId != null && step != null) {
|
||||
try {
|
||||
Long modelId = modelRoutingService.selectModelId(kbId, "heavy_ingest", step);
|
||||
@ -2295,7 +2343,7 @@ public class WikiProcessingService {
|
||||
* pick the routed chat model; passing {@code null} for either reproduces
|
||||
* the legacy behavior (system default model).
|
||||
*/
|
||||
private String callLlmWithResilientRetry(Prompt prompt, String ctx, Long kbId, vip.mate.wiki.job.WikiJobStep step) {
|
||||
private String callLlmWithResilientRetry(Prompt prompt, String ctx, Long kbId, WikiJobStep step) {
|
||||
long backoffMs = 1000;
|
||||
final long maxBackoffMs = 60_000;
|
||||
final int maxAttempts = Math.max(1, properties.getLlmMaxAttempts());
|
||||
@ -2434,7 +2482,7 @@ public class WikiProcessingService {
|
||||
/** Pair of modelId + built ChatModel — null modelId means we used the system default. */
|
||||
private record ResolvedChatModel(Long modelId, ChatModel chatModel) {}
|
||||
|
||||
private ResolvedChatModel resolveChatModel(Long kbId, vip.mate.wiki.job.WikiJobStep step) {
|
||||
private ResolvedChatModel resolveChatModel(Long kbId, WikiJobStep step) {
|
||||
if (modelRoutingService != null && kbId != null && step != null) {
|
||||
try {
|
||||
Long modelId = modelRoutingService.selectModelId(kbId, "heavy_ingest", step);
|
||||
@ -2466,7 +2514,7 @@ public class WikiProcessingService {
|
||||
* stable, never random.
|
||||
*
|
||||
* <p>Skips providers currently in cooldown ({@link
|
||||
* vip.mate.llm.failover.ProviderHealthTracker}) so a flapping provider
|
||||
* ProviderHealthTracker}) so a flapping provider
|
||||
* doesn't keep getting tried while we wait for it to recover.
|
||||
*/
|
||||
private ResolvedChatModel pickFallbackChatModel(Long failedModelId) {
|
||||
@ -2758,8 +2806,8 @@ public class WikiProcessingService {
|
||||
));
|
||||
if (isAborted(raw.getId(), "repair page=" + page.getSlug())) return;
|
||||
String response = callLlmWithResilientRetry(prompt, "repair page=" + page.getSlug(),
|
||||
kb.getId(), vip.mate.wiki.job.WikiJobStep.MERGE_PAGE);
|
||||
com.fasterxml.jackson.databind.JsonNode pageJson = parseJsonResponse(response);
|
||||
kb.getId(), WikiJobStep.MERGE_PAGE);
|
||||
JsonNode pageJson = parseJsonResponse(response);
|
||||
if (pageJson == null) return;
|
||||
|
||||
String content = pageJson.path("content").asText("");
|
||||
@ -2773,7 +2821,7 @@ public class WikiProcessingService {
|
||||
private List<Long> parseSourceRawIds(String json) {
|
||||
if (json == null || json.isBlank()) return List.of();
|
||||
try {
|
||||
return objectMapper.readValue(json, new com.fasterxml.jackson.core.type.TypeReference<List<Long>>() {});
|
||||
return objectMapper.readValue(json, new TypeReference<List<Long>>() {});
|
||||
} catch (Exception e) {
|
||||
return List.of();
|
||||
}
|
||||
@ -2843,6 +2891,17 @@ public class WikiProcessingService {
|
||||
return config != null ? config.getIngestMode() : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the {@code entityExtractionEnabled} opt-in from KB config. Defaults
|
||||
* to {@code false} on any parse error or missing field — extraction is an
|
||||
* opt-in cost and must never be turned on implicitly.
|
||||
*/
|
||||
private boolean isEntityExtractionEnabled(WikiKnowledgeBaseEntity kb) {
|
||||
if (kb == null || kb.getConfigContent() == null) return false;
|
||||
WikiKbConfig config = WikiKbConfigParser.parse(objectMapper, kb.getConfigContent());
|
||||
return config != null && Boolean.TRUE.equals(config.getEntityExtractionEnabled());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns {@code true} when the caller should bail out of an in-flight
|
||||
* processing path because the raw material has been deleted.
|
||||
@ -2911,7 +2970,7 @@ public class WikiProcessingService {
|
||||
|
||||
rawService.updateProgress(rawId, "lazy", 0, 0);
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_STARTED,
|
||||
java.util.Map.of("rawId", rawId, "phase", "lazy"));
|
||||
Map.of("rawId", rawId, "phase", "lazy"));
|
||||
|
||||
try {
|
||||
String textContent = rawService.getTextContent(raw);
|
||||
@ -2919,7 +2978,7 @@ public class WikiProcessingService {
|
||||
rawService.updateProcessingStatus(rawId, "failed", "No text content available");
|
||||
kbService.updateStatus(kbId, "active");
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED,
|
||||
java.util.Map.of("rawId", rawId, "error", "No text content available"));
|
||||
Map.of("rawId", rawId, "error", "No text content available"));
|
||||
return;
|
||||
}
|
||||
|
||||
@ -2957,7 +3016,7 @@ public class WikiProcessingService {
|
||||
if (embedded > 0) {
|
||||
log.info("[Wiki] Lazy async embedding completed: kbId={}, embedded={}", fKbId, embedded);
|
||||
}
|
||||
} catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) {
|
||||
} catch (WikiEmbeddingProviderFailingException ex) {
|
||||
log.warn("[Wiki] Lazy async embedding aborted by circuit-breaker for kbId={}: {}",
|
||||
fKbId, ex.getMessage());
|
||||
} catch (Exception ex) {
|
||||
@ -2975,7 +3034,7 @@ public class WikiProcessingService {
|
||||
kbService.updateStatus(kbId, "active");
|
||||
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_COMPLETED,
|
||||
java.util.Map.of(
|
||||
Map.of(
|
||||
"rawId", rawId,
|
||||
"status", "completed",
|
||||
"totalPages", 0,
|
||||
@ -2993,7 +3052,7 @@ public class WikiProcessingService {
|
||||
// RFC-051 PR-2b: refresh overview stats.
|
||||
if (overviewService != null) overviewService.rebuild(kbId);
|
||||
// Tier 2: dirty event drives the LLM-narrated overview section.
|
||||
eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kbId));
|
||||
eventPublisher.publishEvent(new WikiKbDirtyEvent(this, kbId));
|
||||
|
||||
log.info("[Wiki] Lazy processing completed for raw={}, kbId={}, chunks={}",
|
||||
rawId, kbId, totalChunks);
|
||||
@ -3002,7 +3061,7 @@ public class WikiProcessingService {
|
||||
rawService.updateProcessingStatus(rawId, "failed", e.getMessage());
|
||||
kbService.updateStatus(kbId, "active");
|
||||
progressBus.broadcast(kbId, WikiProgressBus.EVENT_RAW_FAILED,
|
||||
java.util.Map.of("rawId", rawId,
|
||||
Map.of("rawId", rawId,
|
||||
"error", e.getMessage() == null ? "unknown" : e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,56 @@
|
||||
-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks.
|
||||
--
|
||||
-- Distinct from mate_wiki_page (document/topic granularity) — a row here is a
|
||||
-- mention-granularity named entity (person, organization, location, event, ...)
|
||||
-- resolved and de-duplicated across the knowledge base. The entity layer sits
|
||||
-- beneath the page layer: entities are linked to their source chunks (and, via
|
||||
-- citing pages, to wiki pages) through mate_wiki_entity_mention, and to each
|
||||
-- other through mate_wiki_entity_relation.
|
||||
--
|
||||
-- canonical_name display name chosen for the merged entity
|
||||
-- normalized_key case/whitespace-folded key used for exact-match dedup
|
||||
-- type entity taxonomy: person | organization | location |
|
||||
-- event | product | concept | other
|
||||
-- aliases_json JSON array of surface forms merged into this entity
|
||||
-- description one-line summary synthesized from the mentions
|
||||
-- salience 0..1 importance score (mention frequency / distribution)
|
||||
-- mention_count number of mentions resolved to this entity
|
||||
-- embedding float32 little-endian name/description vector used for
|
||||
-- near-duplicate merge across spellings/languages
|
||||
-- computed_hash fingerprint of the inputs that produced this row
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity (
|
||||
id BIGINT PRIMARY KEY,
|
||||
kb_id BIGINT NOT NULL,
|
||||
|
||||
canonical_name VARCHAR(256) NOT NULL,
|
||||
normalized_key VARCHAR(256) NOT NULL,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
|
||||
aliases_json CLOB,
|
||||
description CLOB,
|
||||
salience DECIMAL(5, 4),
|
||||
mention_count INT NOT NULL DEFAULT 0,
|
||||
|
||||
embedding BLOB,
|
||||
embedding_model VARCHAR(64),
|
||||
|
||||
computed_hash VARCHAR(64),
|
||||
|
||||
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted INT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- Exact-match dedup identity within a KB (deleted included so soft-deleted
|
||||
-- rows can coexist with re-inserted ones during re-extract cycles).
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uk_we_key
|
||||
ON mate_wiki_entity (kb_id, normalized_key, type, deleted);
|
||||
|
||||
-- KB-wide listing and "top entities by salience".
|
||||
CREATE INDEX IF NOT EXISTS idx_we_kb
|
||||
ON mate_wiki_entity (kb_id, deleted);
|
||||
CREATE INDEX IF NOT EXISTS idx_we_salience
|
||||
ON mate_wiki_entity (kb_id, salience DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_we_type
|
||||
ON mate_wiki_entity (kb_id, type, deleted);
|
||||
@ -0,0 +1,38 @@
|
||||
-- mate_wiki_entity_mention: links a canonical entity to a source occurrence.
|
||||
--
|
||||
-- One row per (entity, chunk) occurrence. page_id is back-filled from the
|
||||
-- chunk's citing pages so the entity layer connects to the page layer:
|
||||
-- entity -> mention -> chunk -> citing page. A NULL page_id means the source
|
||||
-- chunk is not yet cited by any generated page.
|
||||
--
|
||||
-- entity_id the resolved canonical entity (mate_wiki_entity.id)
|
||||
-- chunk_id source chunk the mention was found in
|
||||
-- page_id a wiki page that cites that chunk, when known
|
||||
-- surface_form the exact text as it appeared in the source
|
||||
-- char_offset character offset of the mention within the chunk, when known
|
||||
-- confidence 0..1 extraction confidence
|
||||
-- evidence short surrounding quote (<= 500 chars enforced in Java)
|
||||
-- source provenance tag: llm-extracted | manual
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention (
|
||||
id BIGINT PRIMARY KEY,
|
||||
kb_id BIGINT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
chunk_id BIGINT,
|
||||
page_id BIGINT,
|
||||
|
||||
surface_form VARCHAR(256),
|
||||
char_offset INT,
|
||||
confidence DECIMAL(4, 3),
|
||||
evidence CLOB,
|
||||
source VARCHAR(32),
|
||||
|
||||
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted INT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_entity ON mate_wiki_entity_mention (entity_id, deleted);
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_chunk ON mate_wiki_entity_mention (chunk_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_page ON mate_wiki_entity_mention (page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_kb ON mate_wiki_entity_mention (kb_id, deleted);
|
||||
@ -0,0 +1,42 @@
|
||||
-- mate_wiki_entity_relation: directed subject -> predicate -> object triples
|
||||
-- between canonical entities (the entity-level knowledge graph edges).
|
||||
--
|
||||
-- Distinct from mate_wiki_relation, which scores page-to-page edges. A row
|
||||
-- here is one fact triple connecting two mate_wiki_entity nodes.
|
||||
--
|
||||
-- subject_entity_id head entity
|
||||
-- predicate free-text relation label (e.g. "works_for", "located_in")
|
||||
-- object_entity_id tail entity
|
||||
-- evidence short justification quote (<= 500 chars enforced in Java)
|
||||
-- confidence 0..1 extraction confidence
|
||||
-- source provenance tag: llm-extracted | inferred | manual
|
||||
-- evidence_chunk_id source chunk the triple was extracted from, when known
|
||||
-- computed_hash fingerprint of the inputs that produced this row
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation (
|
||||
id BIGINT PRIMARY KEY,
|
||||
kb_id BIGINT NOT NULL,
|
||||
|
||||
subject_entity_id BIGINT NOT NULL,
|
||||
predicate VARCHAR(64) NOT NULL,
|
||||
object_entity_id BIGINT NOT NULL,
|
||||
|
||||
evidence CLOB,
|
||||
confidence DECIMAL(4, 3),
|
||||
source VARCHAR(32),
|
||||
evidence_chunk_id BIGINT,
|
||||
computed_hash VARCHAR(64),
|
||||
|
||||
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted INT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
-- Unique triple identity within a KB.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uk_wer_triple
|
||||
ON mate_wiki_entity_relation (kb_id, subject_entity_id, predicate, object_entity_id, deleted);
|
||||
|
||||
-- Ego-graph traversal in both directions.
|
||||
CREATE INDEX IF NOT EXISTS idx_wer_subject ON mate_wiki_entity_relation (kb_id, subject_entity_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wer_object ON mate_wiki_entity_relation (kb_id, object_entity_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wer_kb ON mate_wiki_entity_relation (kb_id, deleted);
|
||||
@ -0,0 +1,31 @@
|
||||
-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks.
|
||||
-- See the H2 file for the design rationale.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity (
|
||||
id BIGINT PRIMARY KEY,
|
||||
kb_id BIGINT NOT NULL,
|
||||
|
||||
canonical_name VARCHAR(256) NOT NULL,
|
||||
normalized_key VARCHAR(256) NOT NULL,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
|
||||
aliases_json TEXT,
|
||||
description TEXT,
|
||||
salience DECIMAL(5, 4),
|
||||
mention_count INT NOT NULL DEFAULT 0,
|
||||
|
||||
embedding BYTEA,
|
||||
embedding_model VARCHAR(64),
|
||||
|
||||
computed_hash VARCHAR(64),
|
||||
|
||||
create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted SMALLINT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uk_we_key
|
||||
ON mate_wiki_entity (kb_id, normalized_key, type, deleted);
|
||||
CREATE INDEX IF NOT EXISTS idx_we_kb ON mate_wiki_entity (kb_id, deleted);
|
||||
CREATE INDEX IF NOT EXISTS idx_we_salience ON mate_wiki_entity (kb_id, salience DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_we_type ON mate_wiki_entity (kb_id, type, deleted);
|
||||
@ -0,0 +1,25 @@
|
||||
-- mate_wiki_entity_mention: links a canonical entity to a source occurrence.
|
||||
-- See the H2 file for the design rationale.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention (
|
||||
id BIGINT PRIMARY KEY,
|
||||
kb_id BIGINT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
chunk_id BIGINT,
|
||||
page_id BIGINT,
|
||||
|
||||
surface_form VARCHAR(256),
|
||||
char_offset INT,
|
||||
confidence DECIMAL(4, 3),
|
||||
evidence TEXT,
|
||||
source VARCHAR(32),
|
||||
|
||||
create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted SMALLINT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_entity ON mate_wiki_entity_mention (entity_id, deleted);
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_chunk ON mate_wiki_entity_mention (chunk_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_page ON mate_wiki_entity_mention (page_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wem_kb ON mate_wiki_entity_mention (kb_id, deleted);
|
||||
@ -0,0 +1,27 @@
|
||||
-- mate_wiki_entity_relation: directed subject -> predicate -> object triples
|
||||
-- between canonical entities. See the H2 file for the design rationale.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation (
|
||||
id BIGINT PRIMARY KEY,
|
||||
kb_id BIGINT NOT NULL,
|
||||
|
||||
subject_entity_id BIGINT NOT NULL,
|
||||
predicate VARCHAR(64) NOT NULL,
|
||||
object_entity_id BIGINT NOT NULL,
|
||||
|
||||
evidence TEXT,
|
||||
confidence DECIMAL(4, 3),
|
||||
source VARCHAR(32),
|
||||
evidence_chunk_id BIGINT,
|
||||
computed_hash VARCHAR(64),
|
||||
|
||||
create_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
deleted SMALLINT NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uk_wer_triple
|
||||
ON mate_wiki_entity_relation (kb_id, subject_entity_id, predicate, object_entity_id, deleted);
|
||||
CREATE INDEX IF NOT EXISTS idx_wer_subject ON mate_wiki_entity_relation (kb_id, subject_entity_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wer_object ON mate_wiki_entity_relation (kb_id, object_entity_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wer_kb ON mate_wiki_entity_relation (kb_id, deleted);
|
||||
@ -0,0 +1,32 @@
|
||||
-- mate_wiki_entity: canonical named-entity nodes extracted from source chunks.
|
||||
-- See the H2 file for the design rationale.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity (
|
||||
id BIGINT NOT NULL,
|
||||
kb_id BIGINT NOT NULL,
|
||||
|
||||
canonical_name VARCHAR(256) NOT NULL,
|
||||
normalized_key VARCHAR(256) NOT NULL,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
|
||||
aliases_json LONGTEXT,
|
||||
description LONGTEXT,
|
||||
salience DECIMAL(5, 4),
|
||||
mention_count INT NOT NULL DEFAULT 0,
|
||||
|
||||
embedding BLOB,
|
||||
embedding_model VARCHAR(64),
|
||||
|
||||
computed_hash VARCHAR(64),
|
||||
|
||||
create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
deleted TINYINT NOT NULL DEFAULT 0,
|
||||
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uk_we_key (kb_id, normalized_key, type, deleted),
|
||||
KEY idx_we_kb (kb_id, deleted),
|
||||
KEY idx_we_salience (kb_id, salience DESC),
|
||||
KEY idx_we_type (kb_id, type, deleted)
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
|
||||
COMMENT = 'Canonical named-entity nodes extracted and de-duplicated from source chunks.';
|
||||
@ -0,0 +1,27 @@
|
||||
-- mate_wiki_entity_mention: links a canonical entity to a source occurrence.
|
||||
-- See the H2 file for the design rationale.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity_mention (
|
||||
id BIGINT NOT NULL,
|
||||
kb_id BIGINT NOT NULL,
|
||||
entity_id BIGINT NOT NULL,
|
||||
chunk_id BIGINT,
|
||||
page_id BIGINT,
|
||||
|
||||
surface_form VARCHAR(256),
|
||||
char_offset INT,
|
||||
confidence DECIMAL(4, 3),
|
||||
evidence TEXT,
|
||||
source VARCHAR(32),
|
||||
|
||||
create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
deleted TINYINT NOT NULL DEFAULT 0,
|
||||
|
||||
PRIMARY KEY (id),
|
||||
KEY idx_wem_entity (entity_id, deleted),
|
||||
KEY idx_wem_chunk (chunk_id),
|
||||
KEY idx_wem_page (page_id),
|
||||
KEY idx_wem_kb (kb_id, deleted)
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
|
||||
COMMENT = 'Entity-to-source occurrence links connecting the entity layer to chunks and pages.';
|
||||
@ -0,0 +1,28 @@
|
||||
-- mate_wiki_entity_relation: directed subject -> predicate -> object triples
|
||||
-- between canonical entities. See the H2 file for the design rationale.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mate_wiki_entity_relation (
|
||||
id BIGINT NOT NULL,
|
||||
kb_id BIGINT NOT NULL,
|
||||
|
||||
subject_entity_id BIGINT NOT NULL,
|
||||
predicate VARCHAR(64) NOT NULL,
|
||||
object_entity_id BIGINT NOT NULL,
|
||||
|
||||
evidence TEXT,
|
||||
confidence DECIMAL(4, 3),
|
||||
source VARCHAR(32),
|
||||
evidence_chunk_id BIGINT,
|
||||
computed_hash VARCHAR(64),
|
||||
|
||||
create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
deleted TINYINT NOT NULL DEFAULT 0,
|
||||
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uk_wer_triple (kb_id, subject_entity_id, predicate, object_entity_id, deleted),
|
||||
KEY idx_wer_subject (kb_id, subject_entity_id),
|
||||
KEY idx_wer_object (kb_id, object_entity_id),
|
||||
KEY idx_wer_kb (kb_id, deleted)
|
||||
) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
|
||||
COMMENT = 'Entity-to-entity fact triples forming the entity-level knowledge graph edges.';
|
||||
@ -0,0 +1,179 @@
|
||||
package vip.mate.wiki.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.Wrapper;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.metadata.ChatGenerationMetadata;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import vip.mate.llm.service.ModelConfigService;
|
||||
import vip.mate.wiki.job.WikiJobStep;
|
||||
import vip.mate.wiki.job.WikiModelRoutingService;
|
||||
import vip.mate.wiki.model.WikiChunkEntity;
|
||||
import vip.mate.wiki.model.WikiEntityEntity;
|
||||
import vip.mate.wiki.model.WikiEntityMentionEntity;
|
||||
import vip.mate.wiki.model.WikiEntityRelationEntity;
|
||||
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
|
||||
import vip.mate.wiki.repository.WikiEntityMapper;
|
||||
import vip.mate.wiki.repository.WikiEntityMentionMapper;
|
||||
import vip.mate.wiki.repository.WikiEntityRelationMapper;
|
||||
import vip.mate.wiki.repository.WikiPageCitationMapper;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Verifies the entity-extraction runtime logic against a mocked model + DB:
|
||||
* the structured LLM output is parsed, entities are de-duplicated across
|
||||
* chunks via the run cache, mentions are written per occurrence, and a
|
||||
* resolved relation triple is persisted. This exercises the real
|
||||
* {@link WikiEntityExtractionService} control flow, not just a stubbed call.
|
||||
*/
|
||||
class WikiEntityExtractionServiceTest {
|
||||
|
||||
private static final Long KB_ID = 1L;
|
||||
private static final Long RAW_ID = 100L;
|
||||
|
||||
private static final String LLM_JSON = """
|
||||
{
|
||||
"entities": [
|
||||
{"name": "Alice", "type": "person", "aliases": [], "description": "An engineer", "evidence": "Alice works at Acme"},
|
||||
{"name": "Acme", "type": "organization", "aliases": [], "description": "A company", "evidence": "Acme Corp"}
|
||||
],
|
||||
"relations": [
|
||||
{"subject": "Alice", "predicate": "works for", "object": "Acme", "evidence": "Alice works at Acme"}
|
||||
]
|
||||
}
|
||||
""";
|
||||
|
||||
private WikiKnowledgeBaseService kbService;
|
||||
private WikiChunkService chunkService;
|
||||
private WikiEmbeddingService embeddingService;
|
||||
private WikiModelRoutingService routingService;
|
||||
private ModelConfigService modelConfigService;
|
||||
private WikiEntityMapper entityMapper;
|
||||
private WikiEntityMentionMapper mentionMapper;
|
||||
private WikiEntityRelationMapper relationMapper;
|
||||
private WikiPageCitationMapper citationMapper;
|
||||
|
||||
private WikiEntityExtractionService service;
|
||||
|
||||
/** Simulated entity store so selectById sees what insert assigned. */
|
||||
private final Map<Long, WikiEntityEntity> store = new HashMap<>();
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
kbService = mock(WikiKnowledgeBaseService.class);
|
||||
chunkService = mock(WikiChunkService.class);
|
||||
embeddingService = mock(WikiEmbeddingService.class);
|
||||
routingService = mock(WikiModelRoutingService.class);
|
||||
modelConfigService = mock(ModelConfigService.class);
|
||||
entityMapper = mock(WikiEntityMapper.class);
|
||||
mentionMapper = mock(WikiEntityMentionMapper.class);
|
||||
relationMapper = mock(WikiEntityRelationMapper.class);
|
||||
citationMapper = mock(WikiPageCitationMapper.class);
|
||||
|
||||
service = new WikiEntityExtractionService(
|
||||
kbService, chunkService, embeddingService, routingService,
|
||||
modelConfigService, new ObjectMapper(),
|
||||
entityMapper, mentionMapper, relationMapper, citationMapper);
|
||||
|
||||
WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity();
|
||||
kb.setId(KB_ID);
|
||||
when(kbService.getById(KB_ID)).thenReturn(kb);
|
||||
|
||||
// Model routing → a mock ChatModel that always returns the canned JSON.
|
||||
ChatModel canned = cannedModel(LLM_JSON);
|
||||
when(routingService.selectModelId(eq(KB_ID), any(), eq(WikiJobStep.ENTITY_EXTRACTION)))
|
||||
.thenReturn(7L);
|
||||
when(routingService.buildChatModel(7L)).thenReturn(canned);
|
||||
|
||||
// No prior entities (with embeddings), no embedding vectors → exercise
|
||||
// the exact-key dedup path (no embedding merge).
|
||||
when(entityMapper.selectList(any())).thenReturn(Collections.emptyList());
|
||||
when(entityMapper.selectOne(any())).thenReturn(null);
|
||||
when(embeddingService.embedQuery(anyLong(), any())).thenReturn(null);
|
||||
|
||||
// No existing mentions → every chunk gets processed.
|
||||
when(mentionMapper.selectCount(any(Wrapper.class))).thenReturn(0L);
|
||||
when(citationMapper.listPageIdsByChunkId(anyLong())).thenReturn(Collections.emptyList());
|
||||
when(relationMapper.selectOne(any())).thenReturn(null);
|
||||
|
||||
// insert assigns a snowflake-like id and records the row so selectById works.
|
||||
AtomicLong seq = new AtomicLong(1000L);
|
||||
when(entityMapper.insert(any(WikiEntityEntity.class))).thenAnswer(inv -> {
|
||||
WikiEntityEntity e = inv.getArgument(0);
|
||||
e.setId(seq.getAndIncrement());
|
||||
store.put(e.getId(), e);
|
||||
return 1;
|
||||
});
|
||||
when(entityMapper.selectById(anyLong())).thenAnswer(inv -> store.get(inv.getArgument(0)));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("extractForRaw: dedups entities across chunks, writes mentions and a relation")
|
||||
void extractForRaw_buildsGraph() {
|
||||
when(chunkService.listByRawId(RAW_ID)).thenReturn(List.of(
|
||||
chunk(1L, "Alice works at Acme."),
|
||||
chunk(2L, "Acme promoted Alice.")));
|
||||
|
||||
int touched = service.extractForRaw(KB_ID, RAW_ID);
|
||||
|
||||
// Two distinct canonical entities despite two chunks naming them.
|
||||
assertEquals(2, touched, "should resolve exactly two canonical entities");
|
||||
verify(entityMapper, times(2)).insert(any(WikiEntityEntity.class));
|
||||
|
||||
// One mention per (entity, chunk) occurrence → 2 entities * 2 chunks.
|
||||
verify(mentionMapper, times(4)).insert(any(WikiEntityMentionEntity.class));
|
||||
|
||||
// The works_for triple is persisted once per chunk it appears in.
|
||||
verify(relationMapper, times(2)).insert(any(WikiEntityRelationEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("extractForRaw: skips chunks that already have mentions")
|
||||
void extractForRaw_skipsProcessedChunks() {
|
||||
when(chunkService.listByRawId(RAW_ID)).thenReturn(List.of(chunk(1L, "Alice works at Acme.")));
|
||||
when(mentionMapper.selectCount(any(Wrapper.class))).thenReturn(3L);
|
||||
|
||||
int touched = service.extractForRaw(KB_ID, RAW_ID);
|
||||
|
||||
assertEquals(0, touched);
|
||||
verify(entityMapper, times(0)).insert(any(WikiEntityEntity.class));
|
||||
}
|
||||
|
||||
private WikiChunkEntity chunk(Long id, String content) {
|
||||
WikiChunkEntity c = new WikiChunkEntity();
|
||||
c.setId(id);
|
||||
c.setKbId(KB_ID);
|
||||
c.setRawId(RAW_ID);
|
||||
c.setContent(content);
|
||||
return c;
|
||||
}
|
||||
|
||||
private ChatModel cannedModel(String body) {
|
||||
ChatModel model = mock(ChatModel.class);
|
||||
when(model.call(any(Prompt.class))).thenReturn(new ChatResponse(List.of(
|
||||
new Generation(new AssistantMessage(body),
|
||||
ChatGenerationMetadata.builder().finishReason("STOP").build()))));
|
||||
return model;
|
||||
}
|
||||
}
|
||||
@ -68,7 +68,8 @@ class WikiProcessingFallbackTest {
|
||||
om,
|
||||
mock(WikiProgressBus.class),
|
||||
mock(WikiCitationService.class),
|
||||
mock(ApplicationEventPublisher.class));
|
||||
mock(ApplicationEventPublisher.class),
|
||||
mock(WikiEntityExtractionService.class));
|
||||
|
||||
// Inject the optional fields via reflection — Spring would do this
|
||||
// post-construction in production, but the test instantiates directly.
|
||||
|
||||
@ -67,7 +67,8 @@ class WikiProcessingServiceLazyTest {
|
||||
new WikiLinkService(om),
|
||||
properties, modelConfigService, agentGraphBuilder, om,
|
||||
progressBus, citationService,
|
||||
mock(org.springframework.context.ApplicationEventPublisher.class));
|
||||
mock(org.springframework.context.ApplicationEventPublisher.class),
|
||||
mock(WikiEntityExtractionService.class));
|
||||
}
|
||||
|
||||
private WikiRawMaterialEntity raw() {
|
||||
|
||||
@ -846,6 +846,16 @@ export const wikiApi = {
|
||||
getPageCitations: (kbId: number | string, pageId: number | string) =>
|
||||
http.get(`/wiki/kb/${kbId}/pages/${pageId}/citations`),
|
||||
|
||||
// Entity-level knowledge graph
|
||||
listEntities: (kbId: number | string, params?: { type?: string; limit?: number }) =>
|
||||
http.get(`/wiki/kb/${kbId}/entities`, { params }),
|
||||
getEntityGraph: (kbId: number | string, limit = 150) =>
|
||||
http.get(`/wiki/kb/${kbId}/entity-graph`, { params: { limit } }),
|
||||
getEntityEgo: (kbId: number | string, entityId: number | string, limit = 50) =>
|
||||
http.get(`/wiki/kb/${kbId}/entities/${entityId}/graph`, { params: { limit } }),
|
||||
extractEntities: (kbId: number | string, force = false) =>
|
||||
http.post(`/wiki/kb/${kbId}/entities/extract`, null, { params: { force } }),
|
||||
|
||||
// RFC-030: Jobs
|
||||
getWikiJobs: (kbId: number, rawId: number) =>
|
||||
http.get(`/wiki/kb/${kbId}/jobs`, { params: { rawId } }),
|
||||
|
||||
@ -2442,6 +2442,11 @@ export default {
|
||||
ingestModeLazy: 'Lazy',
|
||||
ingestModeEagerHint: 'Upload runs the full LLM pipeline to generate wiki pages immediately.',
|
||||
ingestModeLazyHint: 'Upload only extracts, chunks, and embeds. No page generation; search works right away.',
|
||||
entityExtraction: 'Entity extraction',
|
||||
entityExtractionHint: 'Extract named entities (people, organizations, locations, ...) and their relations from raw materials to build an entity-level knowledge graph. Adds processing time and token cost; off by default.',
|
||||
entityExtractionEnable: 'Enable entity extraction',
|
||||
entityExtractionRun: 'Extract now',
|
||||
entityExtractionRunning: 'Extracting…',
|
||||
modelStrategy: 'Model Strategy',
|
||||
globalDefault: 'Global default',
|
||||
selectModel: 'Select a model…',
|
||||
@ -2517,6 +2522,15 @@ export default {
|
||||
linksTo: 'Links to',
|
||||
openPage: 'Open page',
|
||||
empty: 'No graph data — process some raw materials first',
|
||||
modePages: 'Pages',
|
||||
modeEntities: 'Entities',
|
||||
entityEmpty: 'No entities yet — enable entity extraction in the KB config and reprocess',
|
||||
entityType: 'Type',
|
||||
mentions: 'mentions',
|
||||
mentionedIn: 'Mentioned in',
|
||||
relations: 'Relations',
|
||||
description: 'Description',
|
||||
aliases: 'Aliases',
|
||||
},
|
||||
},
|
||||
cronField: {
|
||||
|
||||
@ -2454,6 +2454,11 @@ export default {
|
||||
ingestModeLazy: '先入索引',
|
||||
ingestModeEagerHint: '上传后立即调用 LLM 生成完整 Wiki 页面。',
|
||||
ingestModeLazyHint: '上传仅抽取、切片、向量化,跳过页面生成;上传即可搜索,页面按需编译。',
|
||||
entityExtraction: '实体抽取',
|
||||
entityExtractionHint: '从原始材料中抽取命名实体(人物、组织、地点等)及其关系,构建实体级知识图谱。会增加处理时间与 Token 消耗,默认关闭。',
|
||||
entityExtractionEnable: '开启实体抽取',
|
||||
entityExtractionRun: '立即抽取',
|
||||
entityExtractionRunning: '抽取中…',
|
||||
modelStrategy: '模型策略',
|
||||
globalDefault: '跟随全局默认',
|
||||
selectModel: '选择可用模型…',
|
||||
@ -2529,6 +2534,15 @@ export default {
|
||||
linksTo: '链接到',
|
||||
openPage: '打开页面',
|
||||
empty: '暂无图谱数据,请先处理原始材料',
|
||||
modePages: '页面',
|
||||
modeEntities: '实体',
|
||||
entityEmpty: '暂无实体数据,请在知识库配置中开启实体抽取后重新处理',
|
||||
entityType: '类型',
|
||||
mentions: '提及',
|
||||
mentionedIn: '出现于页面',
|
||||
relations: '关系',
|
||||
description: '描述',
|
||||
aliases: '别名',
|
||||
},
|
||||
},
|
||||
cronField: {
|
||||
|
||||
@ -46,6 +46,33 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ①c Entity extraction -->
|
||||
<div class="config-card">
|
||||
<div class="config-card__head">
|
||||
<div>
|
||||
<div class="config-card__title">{{ t('wiki.configPanel.entityExtraction') }}</div>
|
||||
<div class="config-card__hint">{{ t('wiki.configPanel.entityExtractionHint') }}</div>
|
||||
</div>
|
||||
<button class="btn-save" @click="saveEntityExtraction" :disabled="savingEntityExtraction">
|
||||
{{ savingEntityExtraction ? t('wiki.saving') : t('common.save') }}
|
||||
</button>
|
||||
</div>
|
||||
<div class="ingest-mode-row">
|
||||
<label class="entity-toggle">
|
||||
<input type="checkbox" v-model="entityExtractionEnabled" :disabled="savingEntityExtraction" />
|
||||
<span class="entity-toggle__label">{{ t('wiki.configPanel.entityExtractionEnable') }}</span>
|
||||
</label>
|
||||
<button
|
||||
v-if="entityExtractionEnabled"
|
||||
class="btn-save btn-save--ghost"
|
||||
@click="runExtraction"
|
||||
:disabled="extracting"
|
||||
>
|
||||
{{ extracting ? t('wiki.configPanel.entityExtractionRunning') : t('wiki.configPanel.entityExtractionRun') }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ② Model strategy -->
|
||||
<div class="config-card config-card--clickable" @click="modelsOpen = true">
|
||||
<div class="config-card__row">
|
||||
@ -230,6 +257,42 @@ async function saveIngestMode() {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Entity extraction ──
|
||||
// Opt-in named-entity knowledge graph extraction. Off by default because it
|
||||
// adds an LLM call per chunk on top of the page pipeline.
|
||||
const entityExtractionEnabled = ref(false)
|
||||
const savingEntityExtraction = ref(false)
|
||||
const extracting = ref(false)
|
||||
|
||||
async function saveEntityExtraction() {
|
||||
if (!store.currentKB) return
|
||||
savingEntityExtraction.value = true
|
||||
try {
|
||||
let existingConfig: any = {}
|
||||
try {
|
||||
if (store.currentKB.configContent) existingConfig = JSON.parse(store.currentKB.configContent)
|
||||
} catch { /* config may be plain text rules */ }
|
||||
existingConfig.entityExtractionEnabled = entityExtractionEnabled.value ? true : undefined
|
||||
await wikiApi.updateConfig(store.currentKB.id, JSON.stringify(existingConfig, null, 2))
|
||||
} catch (e) {
|
||||
console.error('[WikiConfig] Failed to save entity extraction toggle', e)
|
||||
} finally {
|
||||
savingEntityExtraction.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function runExtraction() {
|
||||
if (!store.currentKB) return
|
||||
extracting.value = true
|
||||
try {
|
||||
await wikiApi.extractEntities(store.currentKB.id)
|
||||
} catch (e) {
|
||||
console.error('[WikiConfig] Failed to start entity extraction', e)
|
||||
} finally {
|
||||
extracting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// ── Model strategy ──
|
||||
const stepKeys = ['route', 'create_page', 'merge_page', 'enrich', 'summary']
|
||||
const stepModels = reactive<Record<string, string>>({})
|
||||
@ -246,6 +309,7 @@ function loadStepModels() {
|
||||
fallbackModelIds.value = []
|
||||
wikiGlobalModelId.value = ''
|
||||
ingestMode.value = 'eager'
|
||||
entityExtractionEnabled.value = false
|
||||
if (!store.currentKB) return
|
||||
try {
|
||||
const cfg = store.currentKB.configContent ? JSON.parse(store.currentKB.configContent) : null
|
||||
@ -258,6 +322,7 @@ function loadStepModels() {
|
||||
if (cfg?.fallbackModelIds) fallbackModelIds.value = cfg.fallbackModelIds.map(String)
|
||||
if (cfg?.wikiDefaultModelId) wikiGlobalModelId.value = String(cfg.wikiDefaultModelId)
|
||||
if (cfg?.ingestMode === 'lazy') ingestMode.value = 'lazy'
|
||||
if (cfg?.entityExtractionEnabled) entityExtractionEnabled.value = true
|
||||
} catch { /* not JSON */ }
|
||||
}
|
||||
|
||||
@ -475,6 +540,12 @@ loadProviderNames().then(() => {
|
||||
}
|
||||
.btn-save:hover { opacity: 0.88; }
|
||||
.btn-save:disabled { background: var(--mc-border); cursor: not-allowed; }
|
||||
.btn-save--ghost { background: transparent; color: var(--mc-primary); border: 1px solid var(--mc-primary); }
|
||||
.btn-save--ghost:disabled { background: transparent; color: var(--mc-text-tertiary); border-color: var(--mc-border); }
|
||||
|
||||
/* Entity extraction toggle */
|
||||
.entity-toggle { display: flex; align-items: center; gap: 6px; font-size: 13px; color: var(--mc-text-primary); cursor: pointer; }
|
||||
.entity-toggle__label { user-select: none; }
|
||||
|
||||
/* Ingest mode radio group */
|
||||
.ingest-mode-row { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
|
||||
264
mateclaw-ui/src/views/Wiki/components/WikiEntityGraphView.vue
Normal file
264
mateclaw-ui/src/views/Wiki/components/WikiEntityGraphView.vue
Normal file
@ -0,0 +1,264 @@
|
||||
<template>
|
||||
<div class="entity-graph">
|
||||
<div ref="chartEl" class="graph-canvas" />
|
||||
|
||||
<!-- Entity detail panel -->
|
||||
<div v-if="selected" class="entity-panel">
|
||||
<button class="entity-panel__close" @click="selected = null">×</button>
|
||||
<h3 class="entity-panel__title">{{ selected.canonicalName }}</h3>
|
||||
<span class="entity-panel__type" :style="{ background: typeColor(selected.type) }">
|
||||
{{ selected.type }}
|
||||
</span>
|
||||
<span class="entity-panel__count">{{ selected.mentionCount || 0 }} {{ t('wiki.graph.mentions') }}</span>
|
||||
|
||||
<p v-if="selected.description" class="entity-panel__desc">{{ selected.description }}</p>
|
||||
|
||||
<div v-if="selected.aliases && selected.aliases.length" class="entity-panel__section">
|
||||
<h4>{{ t('wiki.graph.aliases') }}</h4>
|
||||
<div class="entity-panel__tags">
|
||||
<span v-for="a in selected.aliases" :key="a" class="entity-panel__tag">{{ a }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="egoEdges.length" class="entity-panel__section">
|
||||
<h4>{{ t('wiki.graph.relations') }}</h4>
|
||||
<ul class="entity-panel__list">
|
||||
<li v-for="(r, i) in egoEdges" :key="i">
|
||||
<span class="entity-panel__pred">{{ r.predicate }}</span>
|
||||
<span class="entity-panel__rel-target">{{ r.label }}</span>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div v-if="egoPages.length" class="entity-panel__section">
|
||||
<h4>{{ t('wiki.graph.mentionedIn') }}</h4>
|
||||
<ul class="entity-panel__list">
|
||||
<li v-for="p in egoPages" :key="p.pageId">
|
||||
<a class="entity-panel__link" @click="emit('open-page', p.slug)">{{ p.title }}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Empty state -->
|
||||
<div v-if="!loading && nodes.length === 0" class="graph-empty">
|
||||
<svg width="48" height="48" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1">
|
||||
<circle cx="7" cy="7" r="3"/><circle cx="17" cy="17" r="3"/><line x1="9.5" y1="9.5" x2="14.5" y2="14.5"/>
|
||||
</svg>
|
||||
<p>{{ t('wiki.graph.entityEmpty') }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, watch, onMounted, onBeforeUnmount, nextTick } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import * as echarts from 'echarts/core'
|
||||
import { GraphChart } from 'echarts/charts'
|
||||
import { TooltipComponent, LegendComponent } from 'echarts/components'
|
||||
import { CanvasRenderer } from 'echarts/renderers'
|
||||
import { wikiApi } from '@/api'
|
||||
|
||||
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
interface EntityNode {
|
||||
id: number | string
|
||||
canonicalName: string
|
||||
type: string
|
||||
description?: string
|
||||
aliases?: string[]
|
||||
salience?: number
|
||||
mentionCount?: number
|
||||
}
|
||||
interface EntityEdge {
|
||||
id?: number | string
|
||||
subjectEntityId: number | string
|
||||
predicate: string
|
||||
objectEntityId: number | string
|
||||
}
|
||||
|
||||
const props = defineProps<{ kbId: number | string | null; isFullscreen?: boolean }>()
|
||||
const emit = defineEmits<{
|
||||
(e: 'open-page', slug: string): void
|
||||
(e: 'stats', v: { nodes: number; edges: number }): void
|
||||
}>()
|
||||
|
||||
const chartEl = ref<HTMLDivElement | null>(null)
|
||||
let chart: echarts.ECharts | null = null
|
||||
|
||||
const nodes = ref<EntityNode[]>([])
|
||||
const edges = ref<EntityEdge[]>([])
|
||||
const loading = ref(false)
|
||||
const selected = ref<EntityNode | null>(null)
|
||||
const egoEdges = ref<{ predicate: string; label: string }[]>([])
|
||||
const egoPages = ref<{ pageId: number | string; slug: string; title: string }[]>([])
|
||||
|
||||
// Stable color per entity type via a small hash → palette.
|
||||
const PALETTE = ['#5b8ff9', '#5ad8a6', '#f6bd16', '#e8684a', '#6dc8ec', '#9270ca', '#ff9d4d', '#269a99']
|
||||
function typeColor(type: string): string {
|
||||
let h = 0
|
||||
for (let i = 0; i < (type || '').length; i++) h = (h * 31 + type.charCodeAt(i)) >>> 0
|
||||
return PALETTE[h % PALETTE.length]
|
||||
}
|
||||
|
||||
async function load() {
|
||||
if (props.kbId == null) return
|
||||
loading.value = true
|
||||
try {
|
||||
// The axios response interceptor returns the raw body for non-enveloped
|
||||
// responses, so `res` is already { center, nodes, edges, pages }.
|
||||
const res: any = await wikiApi.getEntityGraph(props.kbId, 150)
|
||||
nodes.value = res?.nodes || []
|
||||
edges.value = res?.edges || []
|
||||
emit('stats', { nodes: nodes.value.length, edges: edges.value.length })
|
||||
renderChart()
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function buildOption() {
|
||||
// Keep IDs as strings throughout — backend issues Snowflake IDs that lose
|
||||
// precision if coerced to Number.
|
||||
const idSet = new Set(nodes.value.map(n => String(n.id)))
|
||||
const nodeList = nodes.value.map(n => {
|
||||
const size = Math.max(12, Math.min(46, 12 + (n.mentionCount || 0) * 3))
|
||||
return {
|
||||
id: String(n.id),
|
||||
name: n.canonicalName,
|
||||
symbolSize: size,
|
||||
itemStyle: { color: typeColor(n.type) },
|
||||
label: { show: size > 22, position: 'right' as const, fontSize: 10, color: 'var(--mc-text-secondary)', distance: 4 },
|
||||
}
|
||||
})
|
||||
const edgeList = edges.value
|
||||
.filter(e => idSet.has(String(e.subjectEntityId)) && idSet.has(String(e.objectEntityId)))
|
||||
.map(e => ({
|
||||
source: String(e.subjectEntityId),
|
||||
target: String(e.objectEntityId),
|
||||
value: e.predicate,
|
||||
lineStyle: { color: 'rgba(150,150,150,0.3)', width: 1 },
|
||||
}))
|
||||
|
||||
return {
|
||||
backgroundColor: 'transparent',
|
||||
tooltip: {
|
||||
trigger: 'item',
|
||||
formatter: (params: any) => {
|
||||
if (params.dataType === 'edge') return params.data.value || ''
|
||||
const node = nodes.value.find(n => String(n.id) === String(params.data.id))
|
||||
if (!node) return ''
|
||||
const desc = (node.description || '').substring(0, 80)
|
||||
return `<div style="max-width:220px;white-space:normal"><strong>${node.canonicalName}</strong>`
|
||||
+ `<small style="color:#999;display:block">${node.type}</small>`
|
||||
+ (desc ? `<span style="font-size:11px">${desc}</span>` : '') + '</div>'
|
||||
},
|
||||
},
|
||||
series: [{
|
||||
type: 'graph',
|
||||
layout: 'force',
|
||||
data: nodeList,
|
||||
links: edgeList,
|
||||
roam: true,
|
||||
force: { repulsion: 240, gravity: 0.05, edgeLength: [70, 200], friction: 0.55 },
|
||||
emphasis: { focus: 'adjacency', lineStyle: { width: 2 } },
|
||||
lineStyle: { color: 'rgba(150,150,150,0.3)', curveness: 0.1 },
|
||||
edgeSymbol: ['none', 'arrow'],
|
||||
edgeSymbolSize: 6,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
function renderChart() {
|
||||
if (!chartEl.value) return
|
||||
if (!chart) {
|
||||
chart = echarts.init(chartEl.value, undefined, { renderer: 'canvas' })
|
||||
chart.on('click', (params: any) => {
|
||||
if (params.dataType === 'node') {
|
||||
const node = nodes.value.find(n => String(n.id) === String(params.data.id))
|
||||
if (node) openEntity(node)
|
||||
}
|
||||
})
|
||||
}
|
||||
chart.setOption(buildOption(), { notMerge: true, lazyUpdate: true })
|
||||
}
|
||||
|
||||
async function openEntity(node: EntityNode) {
|
||||
selected.value = node
|
||||
egoEdges.value = []
|
||||
egoPages.value = []
|
||||
if (props.kbId == null) return
|
||||
try {
|
||||
const res: any = await wikiApi.getEntityEgo(props.kbId, node.id, 50)
|
||||
const data = res || {}
|
||||
const nodeById = new Map<string, string>()
|
||||
for (const n of data.nodes || []) nodeById.set(String(n.id), n.canonicalName)
|
||||
nodeById.set(String(node.id), node.canonicalName)
|
||||
egoEdges.value = (data.edges || []).map((e: any) => {
|
||||
const otherId = String(e.subjectEntityId) === String(node.id) ? e.objectEntityId : e.subjectEntityId
|
||||
return { predicate: e.predicate, label: nodeById.get(String(otherId)) || '' }
|
||||
})
|
||||
egoPages.value = data.pages || []
|
||||
} catch {
|
||||
/* panel still shows the basic entity info */
|
||||
}
|
||||
}
|
||||
|
||||
const resizeObserver = new ResizeObserver(() => chart?.resize())
|
||||
|
||||
onMounted(async () => {
|
||||
await nextTick()
|
||||
if (chartEl.value) resizeObserver.observe(chartEl.value)
|
||||
load()
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
resizeObserver.disconnect()
|
||||
chart?.dispose()
|
||||
chart = null
|
||||
})
|
||||
|
||||
watch(() => props.kbId, () => { selected.value = null; load() })
|
||||
watch(() => props.isFullscreen, () => nextTick(() => chart?.resize()))
|
||||
|
||||
defineExpose({ reload: load })
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.entity-graph { position: relative; flex: 1; min-height: 0; width: 100%; display: flex; }
|
||||
.graph-canvas { flex: 1; min-height: 0; width: 100%; }
|
||||
|
||||
.graph-empty {
|
||||
position: absolute; inset: 0; display: flex; flex-direction: column;
|
||||
align-items: center; justify-content: center; gap: 12px;
|
||||
color: var(--mc-text-tertiary); pointer-events: none;
|
||||
}
|
||||
.graph-empty p { font-size: 14px; max-width: 320px; text-align: center; }
|
||||
|
||||
.entity-panel {
|
||||
position: absolute; top: 12px; right: 12px; width: 280px; max-height: calc(100% - 24px);
|
||||
overflow-y: auto; padding: 16px; border-radius: 10px;
|
||||
background: var(--mc-bg-elevated, #fff); border: 1px solid var(--mc-border, #e5e7eb);
|
||||
box-shadow: 0 4px 20px rgba(0,0,0,0.12);
|
||||
}
|
||||
.entity-panel__close {
|
||||
position: absolute; top: 8px; right: 10px; border: none; background: none;
|
||||
font-size: 20px; line-height: 1; cursor: pointer; color: var(--mc-text-tertiary);
|
||||
}
|
||||
.entity-panel__title { margin: 0 20px 8px 0; font-size: 15px; font-weight: 600; }
|
||||
.entity-panel__type { display: inline-block; padding: 1px 8px; border-radius: 10px; font-size: 11px; color: #fff; }
|
||||
.entity-panel__count { margin-left: 8px; font-size: 11px; color: var(--mc-text-tertiary); }
|
||||
.entity-panel__desc { margin: 10px 0 0; font-size: 12px; line-height: 1.5; color: var(--mc-text-secondary); }
|
||||
.entity-panel__section { margin-top: 14px; }
|
||||
.entity-panel__section h4 { margin: 0 0 6px; font-size: 11px; text-transform: uppercase; color: var(--mc-text-tertiary); }
|
||||
.entity-panel__tags { display: flex; flex-wrap: wrap; gap: 4px; }
|
||||
.entity-panel__tag { padding: 1px 6px; border-radius: 6px; font-size: 11px; background: var(--mc-bg-subtle, #f3f4f6); }
|
||||
.entity-panel__list { margin: 0; padding: 0; list-style: none; font-size: 12px; }
|
||||
.entity-panel__list li { padding: 2px 0; }
|
||||
.entity-panel__pred { color: var(--mc-text-tertiary); margin-right: 6px; }
|
||||
.entity-panel__rel-target { font-weight: 500; }
|
||||
.entity-panel__link { color: var(--mc-primary, #5b8ff9); cursor: pointer; }
|
||||
.entity-panel__link:hover { text-decoration: underline; }
|
||||
</style>
|
||||
@ -13,7 +13,7 @@
|
||||
</svg>
|
||||
{{ edgeCount }} {{ t('wiki.graph.edges') }}
|
||||
</span>
|
||||
<span v-if="orphanCount > 0" class="stat-item stat-warn">
|
||||
<span v-if="mode === 'pages' && orphanCount > 0" class="stat-item stat-warn">
|
||||
<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5">
|
||||
<circle cx="12" cy="12" r="10"/><line x1="12" y1="8" x2="12" y2="12"/><line x1="12" y1="16" x2="12.01" y2="16"/>
|
||||
</svg>
|
||||
@ -22,11 +22,23 @@
|
||||
</div>
|
||||
|
||||
<div class="graph-controls">
|
||||
<label class="filter-label">
|
||||
<div class="mode-toggle">
|
||||
<button
|
||||
class="mode-btn"
|
||||
:class="{ 'mode-btn--active': mode === 'pages' }"
|
||||
@click="emit('update:mode', 'pages')"
|
||||
>{{ t('wiki.graph.modePages') }}</button>
|
||||
<button
|
||||
class="mode-btn"
|
||||
:class="{ 'mode-btn--active': mode === 'entities' }"
|
||||
@click="emit('update:mode', 'entities')"
|
||||
>{{ t('wiki.graph.modeEntities') }}</button>
|
||||
</div>
|
||||
<label v-if="mode === 'pages'" class="filter-label">
|
||||
<input :checked="showOrphans" type="checkbox" @change="emit('update:showOrphans', ($event.target as HTMLInputElement).checked)" />
|
||||
{{ t('wiki.graph.showOrphans') }}
|
||||
</label>
|
||||
<select :value="typeFilter" class="type-select" @change="emit('update:typeFilter', ($event.target as HTMLSelectElement).value)">
|
||||
<select v-if="mode === 'pages'" :value="typeFilter" class="type-select" @change="emit('update:typeFilter', ($event.target as HTMLSelectElement).value)">
|
||||
<option value="">{{ t('wiki.graph.allTypes') }}</option>
|
||||
<option v-for="type in availableTypes" :key="type" :value="type">
|
||||
{{ formatPageTypeLabel(type) }}
|
||||
@ -73,11 +85,13 @@ defineProps<{
|
||||
typeFilter: string
|
||||
availableTypes: string[]
|
||||
isFullscreen: boolean
|
||||
mode: 'pages' | 'entities'
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
(e: 'update:showOrphans', val: boolean): void
|
||||
(e: 'update:typeFilter', val: string): void
|
||||
(e: 'update:mode', val: 'pages' | 'entities'): void
|
||||
(e: 'reset'): void
|
||||
(e: 'toggleFullscreen'): void
|
||||
}>()
|
||||
@ -106,6 +120,13 @@ const emit = defineEmits<{
|
||||
.stat-warn { color: var(--mc-danger, #f56c6c); }
|
||||
|
||||
.graph-controls { display: flex; align-items: center; gap: 8px; }
|
||||
|
||||
.mode-toggle { display: inline-flex; border: 1px solid var(--mc-border-light); border-radius: 7px; overflow: hidden; }
|
||||
.mode-btn {
|
||||
padding: 3px 10px; font-size: 11px; border: none; cursor: pointer;
|
||||
background: var(--mc-bg-elevated); color: var(--mc-text-secondary);
|
||||
}
|
||||
.mode-btn--active { background: var(--mc-primary, #5b8ff9); color: #fff; }
|
||||
.filter-label { display: flex; align-items: center; gap: 4px; font-size: 11px; color: var(--mc-text-secondary); cursor: pointer; }
|
||||
|
||||
.type-select {
|
||||
|
||||
@ -2,23 +2,33 @@
|
||||
<div ref="graphViewEl" class="graph-view" :class="{ 'graph-view--fullscreen': isFullscreen }">
|
||||
<!-- Toolbar sub-component -->
|
||||
<WikiGraphToolbar
|
||||
:node-count="nodes.length"
|
||||
:edge-count="edges.length"
|
||||
:node-count="graphMode === 'entities' ? entityStats.nodes : nodes.length"
|
||||
:edge-count="graphMode === 'entities' ? entityStats.edges : edges.length"
|
||||
:orphan-count="orphanCount"
|
||||
v-model:show-orphans="showOrphans"
|
||||
v-model:type-filter="typeFilter"
|
||||
v-model:mode="graphMode"
|
||||
:available-types="availableTypes"
|
||||
:is-fullscreen="isFullscreen"
|
||||
@reset="resetChart"
|
||||
@toggle-fullscreen="toggleFullscreen"
|
||||
/>
|
||||
|
||||
<!-- ECharts canvas -->
|
||||
<div ref="chartEl" class="graph-canvas" />
|
||||
<!-- Entity-level knowledge graph -->
|
||||
<WikiEntityGraphView
|
||||
v-if="graphMode === 'entities'"
|
||||
:kb-id="kbId"
|
||||
:is-fullscreen="isFullscreen"
|
||||
@open-page="emit('open-page', $event)"
|
||||
@stats="entityStats = $event"
|
||||
/>
|
||||
|
||||
<!-- Page-level graph (ECharts canvas) -->
|
||||
<div v-show="graphMode === 'pages'" ref="chartEl" class="graph-canvas" />
|
||||
|
||||
<!-- Node detail panel sub-component -->
|
||||
<WikiGraphNodePanel
|
||||
v-if="selectedNode"
|
||||
v-if="graphMode === 'pages' && selectedNode"
|
||||
:page="selectedNode"
|
||||
:linked-pages="selectedNodeLinks"
|
||||
@close="selectedNode = null"
|
||||
@ -26,7 +36,7 @@
|
||||
/>
|
||||
|
||||
<!-- Empty state -->
|
||||
<div v-if="nodes.length === 0" class="graph-empty">
|
||||
<div v-if="graphMode === 'pages' && nodes.length === 0" class="graph-empty">
|
||||
<svg width="48" height="48" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1">
|
||||
<circle cx="12" cy="12" r="10"/><circle cx="12" cy="12" r="3"/>
|
||||
<line x1="5" y1="5" x2="19" y2="19" stroke-width="0.5"/>
|
||||
@ -47,6 +57,7 @@ import type { WikiPage } from '@/stores/useWikiStore'
|
||||
import { useWikiPageType } from '@/composables/useWikiPageType'
|
||||
import WikiGraphToolbar from './WikiGraphToolbar.vue'
|
||||
import WikiGraphNodePanel from './WikiGraphNodePanel.vue'
|
||||
import WikiEntityGraphView from './WikiEntityGraphView.vue'
|
||||
|
||||
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
|
||||
|
||||
@ -64,6 +75,17 @@ const showOrphans = ref(true)
|
||||
const typeFilter = ref('')
|
||||
const selectedNode = ref<WikiPage | null>(null)
|
||||
|
||||
// 'pages' = page-link graph (default); 'entities' = named-entity graph.
|
||||
const graphMode = ref<'pages' | 'entities'>('pages')
|
||||
const entityStats = ref<{ nodes: number; edges: number }>({ nodes: 0, edges: 0 })
|
||||
|
||||
// KB id for entity-graph fetches — every page carries its kbId; keep it a
|
||||
// string to avoid Snowflake precision loss.
|
||||
const kbId = computed<string | number | null>(() => {
|
||||
const id = props.pages[0]?.kbId
|
||||
return id != null ? id : null
|
||||
})
|
||||
|
||||
|
||||
// Parse outgoing links JSON string → slug[]
|
||||
function parseLinks(outgoingLinks: string | null | undefined): string[] {
|
||||
@ -340,6 +362,14 @@ onBeforeUnmount(() => {
|
||||
watch([nodes, edges], () => {
|
||||
scheduleRender()
|
||||
})
|
||||
|
||||
// Returning to the page graph: the canvas was hidden (v-show), so let the DOM
|
||||
// settle then tell ECharts to re-measure.
|
||||
watch(graphMode, (mode) => {
|
||||
if (mode === 'pages') {
|
||||
nextTick(() => chart?.resize())
|
||||
}
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user