diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java index 78641cc6..ba57e887 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java @@ -73,6 +73,20 @@ public class WikiPageEntity { */ private Integer archived; + /** + * Page-level embedding (float32 little-endian) used by the semantic + * retriever to surface pages whose generated content does not appear + * in any source raw's chunks — typically synthesis pages produced by + * a transformation. {@code null} = not yet embedded. + */ + private byte[] embedding; + + /** Model name that produced {@link #embedding}; used for re-embed detection. */ + private String embeddingModel; + + /** Input-format version for {@link #embedding}; bumped when the embedding builder changes. */ + private String embeddingTextVersion; + @TableField(fill = FieldFill.INSERT) private LocalDateTime createTime; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java index e63cf404..ac4f5eef 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java @@ -211,13 +211,15 @@ public class HybridRetriever { // ==================== Internal methods ==================== - /** Semantic search: chunk cosine → aggregate to page level */ + /** Semantic search: chunk cosine → aggregate to page level, then merged + * with direct page-level cosine when a page has its own embedding. + * The page-level signal covers synthesis pages whose vocabulary doesn't + * appear in any source raw's chunks. */ private List semanticSearch(Long kbId, String query, int limit) { float[] queryVec = embeddingService.embedQuery(kbId, query); if (queryVec == null) return List.of(); List allChunks = chunkService.listByKbId(kbId); - if (allChunks.isEmpty()) return List.of(); Map chunkScores = new HashMap<>(); for (WikiChunkEntity chunk : allChunks) { @@ -230,6 +232,14 @@ public class HybridRetriever { List allPages = pageService.listByKbId(kbId); Map pageScores = new HashMap<>(); for (WikiPageEntity page : allPages) { + // Direct page-level signal: the page carries its own embedding + // (typical for transformation synthesis pages). + if (page.getEmbedding() != null) { + float[] pageVec = WikiEmbeddingService.bytesToFloats(page.getEmbedding()); + float pageScore = WikiEmbeddingService.cosine(queryVec, pageVec); + pageScores.merge(page.getId(), (double) pageScore, Math::max); + } + // Transitive signal: chunks of any source raw this page references. String rawIds = page.getSourceRawIds(); if (rawIds == null) continue; for (String rawIdStr : rawIds.replaceAll("[\\[\\]\\s]", "").split(",")) { diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java index 3cd55eb9..b91458d9 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java @@ -17,7 +17,9 @@ import vip.mate.system.repository.SystemSettingMapper; import vip.mate.wiki.WikiProperties; import vip.mate.wiki.model.WikiChunkEntity; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.repository.WikiChunkMapper; +import vip.mate.wiki.repository.WikiPageMapper; import vip.mate.wiki.repository.WikiRawMaterialMapper; import java.nio.ByteBuffer; @@ -46,6 +48,7 @@ import java.util.Set; public class WikiEmbeddingService { private final WikiChunkMapper chunkMapper; + private final WikiPageMapper pageMapper; private final WikiRawMaterialMapper rawMaterialMapper; private final WikiProperties properties; private final EmbeddingModelFactory factory; @@ -451,6 +454,74 @@ public class WikiEmbeddingService { return end; // hard cut } + /** + * Embed a wiki page's content directly so the semantic retriever can match + * vocabulary that exists in the synthesised page but not in any source + * raw's chunks (typical for transformation-generated synthesis pages). + * Idempotent: skips when the stored embedding is already current for the + * resolved model + input version. + * + * @return {@code true} when the page row was updated with a fresh embedding + */ + public boolean embedPage(Long pageId) { + if (pageId == null) return false; + WikiPageEntity page = pageMapper.selectById(pageId); + if (page == null) { + log.warn("[WikiEmbedding] embedPage: page not found id={}", pageId); + return false; + } + Resolved r = resolveForKb(page.getKbId()); + if (r == null) { + log.debug("[WikiEmbedding] embedPage: no embedding model for kbId={}", page.getKbId()); + return false; + } + String inputVersion = currentInputVersion(); + // Short-circuit when this page is already embedded against the same + // model + input format — nothing to do. + if (page.getEmbedding() != null + && r.modelName().equals(page.getEmbeddingModel()) + && inputVersion.equals(page.getEmbeddingTextVersion())) { + return false; + } + + String input = buildPageEmbeddingInput(page); + if (input.isBlank()) return false; + int maxChars = Math.max(500, properties.getEmbeddingMaxChars()); + if (input.length() > maxChars) input = input.substring(0, maxChars); + + try { + EmbeddingResponse resp = r.model().call(new EmbeddingRequest(List.of(input), null)); + float[] vec = resp.getResults().get(0).getOutput(); + page.setEmbedding(floatsToBytes(vec)); + page.setEmbeddingModel(r.modelName()); + page.setEmbeddingTextVersion(inputVersion); + pageMapper.updateById(page); + log.info("[WikiEmbedding] Embedded page id={} kbId={} model={} ({} chars)", + pageId, page.getKbId(), r.modelName(), input.length()); + return true; + } catch (Exception e) { + log.warn("[WikiEmbedding] embedPage failed id={}: {}", pageId, e.getMessage()); + return false; + } + } + + /** Concatenates the fields that best capture a page's topic — title + + * summary + content prefix — so the embedding picks up both the + * vocabulary the LLM authored and the source-derived material. */ + private String buildPageEmbeddingInput(WikiPageEntity page) { + StringBuilder sb = new StringBuilder(); + if (page.getTitle() != null && !page.getTitle().isBlank()) { + sb.append("# ").append(page.getTitle()).append("\n\n"); + } + if (page.getSummary() != null && !page.getSummary().isBlank()) { + sb.append(page.getSummary()).append("\n\n"); + } + if (page.getContent() != null && !page.getContent().isBlank()) { + sb.append(page.getContent()); + } + return sb.toString(); + } + /** * 查询向量化（混合搜索时调用，需指定 KB 以便解析对应模型） */ diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java index 63848986..045a397c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java @@ -58,6 +58,12 @@ public class WikiTransformationExecutor { @Autowired(required = false) private WikiPageService pageService; + /** Optional. When wired, every persisted synthesis page is embedded so + * the semantic retriever can surface it on terms that exist only in the + * transformation output (not in any source raw's chunks). */ + @Autowired(required = false) + private WikiEmbeddingService embeddingService; + private final com.fasterxml.jackson.databind.ObjectMapper objectMapper = new com.fasterxml.jackson.databind.ObjectMapper(); @@ -375,6 +381,20 @@ public class WikiTransformationExecutor { log.info("[WikiTransformation] updated existing synthesis page slug={} pageId={} from run={}", slug, persisted.getId(), run.getId()); } + + // Fire-and-forget page-level embedding so semantic search can match + // vocabulary the LLM authored which isn't present in the source raw's + // chunks (e.g. "AM-GM", "柯西不等式" derived from a garbled OCR PDF). + if (embeddingService != null) { + final Long pid = persisted.getId(); + WORKER.submit(() -> { + try { embeddingService.embedPage(pid); } + catch (Exception ee) { + log.warn("[WikiTransformation] post-save embedPage failed pageId={}: {}", + pid, ee.getMessage()); + } + }); + } return persisted; } diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql b/mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql new file mode 100644 index 00000000..a6a4c7bb --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql @@ -0,0 +1,10 @@ +-- Page-level embedding so synthesis pages produced by transformations can be +-- surfaced by semantic search even when their generated content doesn't +-- appear in the source raw's chunks. The retriever combines chunk-level +-- cosine (via sourceRawIds) with these page-level vectors taking the max, +-- so a synthesis page that the LLM authored with vocabulary not present in +-- the original PDF can still match a user's natural-language query. + +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding BLOB DEFAULT NULL; +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding_model VARCHAR(64) DEFAULT NULL; +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding_text_version VARCHAR(32) DEFAULT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql new file mode 100644 index 00000000..a1be50d0 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql @@ -0,0 +1,24 @@ +-- Page-level embedding columns. See the h2 sibling for the prose +-- explanation. MySQL lacks `ADD COLUMN IF NOT EXISTS`, so each column +-- guarded by an INFORMATION_SCHEMA check + prepared statement. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'embedding'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding BLOB DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'embedding_model'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding_model VARCHAR(64) DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'embedding_text_version'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding_text_version VARCHAR(32) DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;