feat(wiki): page-level embedding so synthesis pages enter semantic search

mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java

@@ -73,6 +73,20 @@ public class WikiPageEntity {
      */
     private Integer archived;
 
+    /**
+     * Page-level embedding (float32 little-endian) used by the semantic
+     * retriever to surface pages whose generated content does not appear
+     * in any source raw's chunks — typically synthesis pages produced by
+     * a transformation. {@code null} = not yet embedded.
+     */
+    private byte[] embedding;
+
+    /** Model name that produced {@link #embedding}; used for re-embed detection. */
+    private String embeddingModel;
+
+    /** Input-format version for {@link #embedding}; bumped when the embedding builder changes. */
+    private String embeddingTextVersion;
+
     @TableField(fill = FieldFill.INSERT)
     private LocalDateTime createTime;
 

mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java

@@ -211,13 +211,15 @@ public class HybridRetriever {
 
     // ==================== Internal methods ====================
 
-    /** Semantic search: chunk cosine → aggregate to page level */
+    /** Semantic search: chunk cosine → aggregate to page level, then merged
+     *  with direct page-level cosine when a page has its own embedding.
+     *  The page-level signal covers synthesis pages whose vocabulary doesn't
+     *  appear in any source raw's chunks. */
     private List<RankedItem> semanticSearch(Long kbId, String query, int limit) {
         float[] queryVec = embeddingService.embedQuery(kbId, query);
         if (queryVec == null) return List.of();
 
         List<WikiChunkEntity> allChunks = chunkService.listByKbId(kbId);
-        if (allChunks.isEmpty()) return List.of();
 
         Map<Long, Float> chunkScores = new HashMap<>();
         for (WikiChunkEntity chunk : allChunks) {
@@ -230,6 +232,14 @@ public class HybridRetriever {
         List<WikiPageEntity> allPages = pageService.listByKbId(kbId);
         Map<Long, Double> pageScores = new HashMap<>();
         for (WikiPageEntity page : allPages) {
+            // Direct page-level signal: the page carries its own embedding
+            // (typical for transformation synthesis pages).
+            if (page.getEmbedding() != null) {
+                float[] pageVec = WikiEmbeddingService.bytesToFloats(page.getEmbedding());
+                float pageScore = WikiEmbeddingService.cosine(queryVec, pageVec);
+                pageScores.merge(page.getId(), (double) pageScore, Math::max);
+            }
+            // Transitive signal: chunks of any source raw this page references.
             String rawIds = page.getSourceRawIds();
             if (rawIds == null) continue;
             for (String rawIdStr : rawIds.replaceAll("[\\[\\]\\s]", "").split(",")) {

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java

@@ -17,7 +17,9 @@ import vip.mate.system.repository.SystemSettingMapper;
 import vip.mate.wiki.WikiProperties;
 import vip.mate.wiki.model.WikiChunkEntity;
 import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
+import vip.mate.wiki.model.WikiPageEntity;
 import vip.mate.wiki.repository.WikiChunkMapper;
+import vip.mate.wiki.repository.WikiPageMapper;
 import vip.mate.wiki.repository.WikiRawMaterialMapper;
 
 import java.nio.ByteBuffer;
@@ -46,6 +48,7 @@ import java.util.Set;
 public class WikiEmbeddingService {
 
     private final WikiChunkMapper chunkMapper;
+    private final WikiPageMapper pageMapper;
     private final WikiRawMaterialMapper rawMaterialMapper;
     private final WikiProperties properties;
     private final EmbeddingModelFactory factory;
@@ -451,6 +454,74 @@ public class WikiEmbeddingService {
         return end; // hard cut
     }
 
+    /**
+     * Embed a wiki page's content directly so the semantic retriever can match
+     * vocabulary that exists in the synthesised page but not in any source
+     * raw's chunks (typical for transformation-generated synthesis pages).
+     * Idempotent: skips when the stored embedding is already current for the
+     * resolved model + input version.
+     *
+     * @return {@code true} when the page row was updated with a fresh embedding
+     */
+    public boolean embedPage(Long pageId) {
+        if (pageId == null) return false;
+        WikiPageEntity page = pageMapper.selectById(pageId);
+        if (page == null) {
+            log.warn("[WikiEmbedding] embedPage: page not found id={}", pageId);
+            return false;
+        }
+        Resolved r = resolveForKb(page.getKbId());
+        if (r == null) {
+            log.debug("[WikiEmbedding] embedPage: no embedding model for kbId={}", page.getKbId());
+            return false;
+        }
+        String inputVersion = currentInputVersion();
+        // Short-circuit when this page is already embedded against the same
+        // model + input format — nothing to do.
+        if (page.getEmbedding() != null
+                && r.modelName().equals(page.getEmbeddingModel())
+                && inputVersion.equals(page.getEmbeddingTextVersion())) {
+            return false;
+        }
+
+        String input = buildPageEmbeddingInput(page);
+        if (input.isBlank()) return false;
+        int maxChars = Math.max(500, properties.getEmbeddingMaxChars());
+        if (input.length() > maxChars) input = input.substring(0, maxChars);
+
+        try {
+            EmbeddingResponse resp = r.model().call(new EmbeddingRequest(List.of(input), null));
+            float[] vec = resp.getResults().get(0).getOutput();
+            page.setEmbedding(floatsToBytes(vec));
+            page.setEmbeddingModel(r.modelName());
+            page.setEmbeddingTextVersion(inputVersion);
+            pageMapper.updateById(page);
+            log.info("[WikiEmbedding] Embedded page id={} kbId={} model={} ({} chars)",
+                    pageId, page.getKbId(), r.modelName(), input.length());
+            return true;
+        } catch (Exception e) {
+            log.warn("[WikiEmbedding] embedPage failed id={}: {}", pageId, e.getMessage());
+            return false;
+        }
+    }
+
+    /** Concatenates the fields that best capture a page's topic — title +
+     *  summary + content prefix — so the embedding picks up both the
+     *  vocabulary the LLM authored and the source-derived material. */
+    private String buildPageEmbeddingInput(WikiPageEntity page) {
+        StringBuilder sb = new StringBuilder();
+        if (page.getTitle() != null && !page.getTitle().isBlank()) {
+            sb.append("# ").append(page.getTitle()).append("\n\n");
+        }
+        if (page.getSummary() != null && !page.getSummary().isBlank()) {
+            sb.append(page.getSummary()).append("\n\n");
+        }
+        if (page.getContent() != null && !page.getContent().isBlank()) {
+            sb.append(page.getContent());
+        }
+        return sb.toString();
+    }
+
     /**
      * 查询向量化（混合搜索时调用，需指定 KB 以便解析对应模型）
      */

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java

@@ -58,6 +58,12 @@ public class WikiTransformationExecutor {
     @Autowired(required = false)
     private WikiPageService pageService;
 
+    /** Optional. When wired, every persisted synthesis page is embedded so
+     *  the semantic retriever can surface it on terms that exist only in the
+     *  transformation output (not in any source raw's chunks). */
+    @Autowired(required = false)
+    private WikiEmbeddingService embeddingService;
+
     private final com.fasterxml.jackson.databind.ObjectMapper objectMapper =
             new com.fasterxml.jackson.databind.ObjectMapper();
 
@@ -375,6 +381,20 @@ public class WikiTransformationExecutor {
             log.info("[WikiTransformation] updated existing synthesis page slug={} pageId={} from run={}",
                     slug, persisted.getId(), run.getId());
         }
+
+        // Fire-and-forget page-level embedding so semantic search can match
+        // vocabulary the LLM authored which isn't present in the source raw's
+        // chunks (e.g. "AM-GM", "柯西不等式" derived from a garbled OCR PDF).
+        if (embeddingService != null) {
+            final Long pid = persisted.getId();
+            WORKER.submit(() -> {
+                try { embeddingService.embedPage(pid); }
+                catch (Exception ee) {
+                    log.warn("[WikiTransformation] post-save embedPage failed pageId={}: {}",
+                            pid, ee.getMessage());
+                }
+            });
+        }
         return persisted;
     }
 

mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql

@@ -0,0 +1,10 @@
+-- Page-level embedding so synthesis pages produced by transformations can be
+-- surfaced by semantic search even when their generated content doesn't
+-- appear in the source raw's chunks. The retriever combines chunk-level
+-- cosine (via sourceRawIds) with these page-level vectors taking the max,
+-- so a synthesis page that the LLM authored with vocabulary not present in
+-- the original PDF can still match a user's natural-language query.
+
+ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding BLOB DEFAULT NULL;
+ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding_model VARCHAR(64) DEFAULT NULL;
+ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding_text_version VARCHAR(32) DEFAULT NULL;

mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql

@@ -0,0 +1,24 @@
+-- Page-level embedding columns. See the h2 sibling for the prose
+-- explanation. MySQL lacks `ADD COLUMN IF NOT EXISTS`, so each column
+-- guarded by an INFORMATION_SCHEMA check + prepared statement.
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+           WHERE TABLE_SCHEMA = DATABASE()
+             AND TABLE_NAME = 'mate_wiki_page'
+             AND COLUMN_NAME = 'embedding');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding BLOB DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+           WHERE TABLE_SCHEMA = DATABASE()
+             AND TABLE_NAME = 'mate_wiki_page'
+             AND COLUMN_NAME = 'embedding_model');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding_model VARCHAR(64) DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+           WHERE TABLE_SCHEMA = DATABASE()
+             AND TABLE_NAME = 'mate_wiki_page'
+             AND COLUMN_NAME = 'embedding_text_version');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding_text_version VARCHAR(32) DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;