feat(wiki): enrich chunk embedding input with raw title and structural metadata

mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java

@@ -130,6 +130,22 @@ public class WikiProperties {
      */
     private int embeddingMaxChars = 6000;
 
+    /**
+     * Expected embedding-input format version. The authoritative source is
+     * the builder's {@code CURRENT_INPUT_VERSION} constant; this property
+     * exists for staged rollouts and ops overrides.
+     * <p>
+     * Behavior on startup:
+     * <ul>
+     *   <li>Blank: use the builder version.</li>
+     *   <li>Less than builder version: WARN and continue, so a KB can be
+     *       embedded against an older format during a gradual rollback.</li>
+     *   <li>Greater than builder version: fail fast — this usually means the
+     *       config was deployed ahead of the code that implements that format.</li>
+     * </ul>
+     */
+    private String embeddingTextVersionCurrent = "";
+
     /**
      * Circuit-breaker threshold: abort an embedding pass after this many
      * consecutive batch failures (auth / rate-limit / network errors that

mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java

@@ -35,6 +35,7 @@ public class WikiRelationController {
     private final HybridRetriever hybridRetriever;
     private final ApplicationEventPublisher eventPublisher;
     private final ObjectMapper objectMapper;
+    private final WikiEmbeddingService embeddingService;
 
     // ==================== RFC-029: Relations ====================
 
@@ -104,11 +105,14 @@ public class WikiRelationController {
                 .filter(j -> "running".equals(j.getStatus()))
                 .count();
 
+        WikiEmbeddingService.EmbeddingDrift drift = embeddingService.describeDrift(kbId);
+
         return Map.of(
                 "pageCount", pageCount,
                 "enrichedPageCount", enrichedCount,
                 "failedJobCount", failedJobCount,
-                "runningJobCount", runningJobCount
+                "runningJobCount", runningJobCount,
+                "embeddingDrift", drift
         );
     }
 

mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java

@@ -51,6 +51,15 @@ public class WikiChunkEntity {
     /** RFC-011：生成该 embedding 的模型名称（切模型时需全量重嵌） */
     private String embeddingModel;
 
+    /**
+     * Identifies the input format used to produce the stored embedding.
+     * <p>
+     * Set to the embedding input builder's current version on every write.
+     * NULL signals a legacy content-only embedding from before the builder
+     * existed and is treated as stale on the next re-embed pass.
+     */
+    private String embeddingTextVersion;
+
     /** RFC-051: source page number (PDF/PPTX) when known; null otherwise. */
     private Integer pageNumber;
 

mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java

@@ -0,0 +1,15 @@
+package vip.mate.wiki.service;
+
+/**
+ * Resolves {@code rawId -> rawTitle} for embedding-time enrichment.
+ * <p>
+ * Implementations may be naive (one DB hit per call), batch-preloaded for a
+ * given job, or backed by an in-memory snapshot shared with an ingest-scope
+ * page index. Callers must tolerate {@code null} for unknown / deleted ids.
+ */
+@FunctionalInterface
+public interface RawTitleLookup {
+
+    /** @return raw material title, or {@code null} when the id is unknown */
+    String titleFor(Long rawId);
+}

mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java

@@ -0,0 +1,42 @@
+package vip.mate.wiki.service;
+
+import vip.mate.wiki.dto.RawTitleRef;
+import vip.mate.wiki.repository.WikiRawMaterialMapper;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Factory helpers for {@link RawTitleLookup}.
+ */
+public final class RawTitleLookups {
+
+    private RawTitleLookups() {}
+
+    /** Lookup that always returns {@code null}; useful for callers without raw context. */
+    public static RawTitleLookup empty() {
+        return id -> null;
+    }
+
+    /** Lookup backed by a pre-resolved map (e.g. from an ingest-scope snapshot). */
+    public static RawTitleLookup of(Map<Long, String> titlesById) {
+        Map<Long, String> snapshot = titlesById == null ? Map.of() : Map.copyOf(titlesById);
+        return snapshot::get;
+    }
+
+    /**
+     * Preload titles for the given ids in a single batch query and return a
+     * lookup over the resulting map. Unknown ids resolve to {@code null}.
+     */
+    public static RawTitleLookup preload(WikiRawMaterialMapper mapper, Collection<Long> rawIds) {
+        if (mapper == null || rawIds == null || rawIds.isEmpty()) {
+            return empty();
+        }
+        Map<Long, String> titles = new HashMap<>(rawIds.size());
+        for (RawTitleRef ref : mapper.selectBatchTitles(rawIds)) {
+            titles.put(ref.id(), ref.title());
+        }
+        return of(titles);
+    }
+}

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java

@@ -0,0 +1,82 @@
+package vip.mate.wiki.service;
+
+import org.springframework.stereotype.Component;
+import vip.mate.wiki.model.WikiChunkEntity;
+
+/**
+ * Produces the text fed to the embedding model for a chunk.
+ * <p>
+ * Naive content-only embeddings make short or context-poor chunks (e.g. a
+ * standalone sentence like "accuracy improved by 12%") near-indistinguishable
+ * in vector space. Prefixing the model input with already-available metadata —
+ * source title, header breadcrumb, source section, page number — preserves
+ * the semantic neighborhood the chunk came from without changing the storage
+ * model.
+ * <p>
+ * Bump {@link #CURRENT_INPUT_VERSION} whenever the prefix format changes. The
+ * embedding pass treats any chunk whose stored {@code embedding_text_version}
+ * differs from the current value as stale and re-embeds it.
+ */
+@Component
+public class WikiEmbeddingInputBuilder {
+
+    /**
+     * Version tag stamped onto every chunk that this builder embeds.
+     * Increment when the prefix format below changes in a way that should
+     * trigger a re-embed pass. The string is opaque; "v1", "v2", ... is fine.
+     */
+    public static final String CURRENT_INPUT_VERSION = "v1";
+
+    /**
+     * Build the embedding input string for a chunk. Metadata fields that are
+     * null or blank are skipped so empty values never produce stray headers.
+     * Falls back to the chunk content alone when no metadata is available.
+     */
+    public String build(WikiChunkEntity chunk, RawTitleLookup lookup) {
+        if (chunk == null) {
+            return "";
+        }
+        String content = chunk.getContent() == null ? "" : chunk.getContent();
+        String prefix = buildPrefix(chunk, lookup);
+        return prefix.isEmpty() ? content : prefix + content;
+    }
+
+    /**
+     * Build only the metadata prefix for a chunk. Useful when callers need to
+     * split content into sub-segments and prepend the prefix to each one so
+     * the metadata participates in every per-segment embedding before pooling.
+     * Returns an empty string when no metadata is available; otherwise ends
+     * with a blank line so the content reads as a separate paragraph.
+     */
+    public String buildPrefix(WikiChunkEntity chunk, RawTitleLookup lookup) {
+        if (chunk == null) {
+            return "";
+        }
+        StringBuilder sb = new StringBuilder();
+        String rawTitle = (lookup == null || chunk.getRawId() == null)
+                ? null : lookup.titleFor(chunk.getRawId());
+        appendLine(sb, "Source", rawTitle);
+        appendLine(sb, "Section", chunk.getHeaderBreadcrumb());
+        appendLine(sb, "Subsection", chunk.getSourceSection());
+        if (chunk.getPageNumber() != null) {
+            appendLine(sb, "Page", String.valueOf(chunk.getPageNumber()));
+        }
+        if (sb.length() == 0) {
+            return "";
+        }
+        sb.append('\n');
+        return sb.toString();
+    }
+
+    /** @return the version tag this builder stamps onto each chunk it embeds */
+    public String currentVersion() {
+        return CURRENT_INPUT_VERSION;
+    }
+
+    private static void appendLine(StringBuilder sb, String label, String value) {
+        if (value == null || value.isBlank()) {
+            return;
+        }
+        sb.append(label).append(": ").append(value.strip()).append('\n');
+    }
+}

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java

@@ -2,6 +2,7 @@ package vip.mate.wiki.service;
 
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
+import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.ai.embedding.EmbeddingModel;
@@ -17,11 +18,14 @@ import vip.mate.wiki.WikiProperties;
 import vip.mate.wiki.model.WikiChunkEntity;
 import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
 import vip.mate.wiki.repository.WikiChunkMapper;
+import vip.mate.wiki.repository.WikiRawMaterialMapper;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * RFC-011 + Embedding-UI-Config: Wiki 嵌入服务
@@ -42,16 +46,86 @@ import java.util.List;
 public class WikiEmbeddingService {
 
     private final WikiChunkMapper chunkMapper;
+    private final WikiRawMaterialMapper rawMaterialMapper;
     private final WikiProperties properties;
     private final EmbeddingModelFactory factory;
     private final ModelConfigService modelConfigService;
     private final WikiKnowledgeBaseService kbService;
     private final SystemSettingMapper systemSettingMapper;
     private final vip.mate.llm.service.ModelProviderService modelProviderService;
+    private final WikiEmbeddingInputBuilder inputBuilder;
 
     /** 系统默认 embedding 模型的 mate_system_setting key */
     public static final String SYSTEM_SETTING_DEFAULT_EMBEDDING_ID = "embedding.default.model.id";
 
+    /**
+     * Returns the embedding input format version this service stamps onto
+     * each chunk. The builder's constant is the source of truth; the
+     * {@code mate.wiki.embedding-text-version-current} property exists only
+     * to support ops overrides and is validated against the builder at
+     * startup ({@link #verifyConfiguredInputVersion()}).
+     */
+    public String currentInputVersion() {
+        String configured = properties.getEmbeddingTextVersionCurrent();
+        return (configured == null || configured.isBlank()) ? inputBuilder.currentVersion() : configured.trim();
+    }
+
+    /**
+     * Validate the configured embedding input version against the builder
+     * constant on startup. A blank config is normal (the builder version is
+     * used). A config below the builder is allowed with a WARN so a KB can
+     * be embedded against an older format during a gradual rollback. A
+     * config above the builder fails fast — it almost always means the
+     * config was deployed ahead of the code.
+     */
+    @PostConstruct
+    void verifyConfiguredInputVersion() {
+        String configured = properties.getEmbeddingTextVersionCurrent();
+        if (configured == null || configured.isBlank()) {
+            log.info("[WikiEmbedding] Embedding input version: {} (from builder)", inputBuilder.currentVersion());
+            return;
+        }
+        String builderVersion = inputBuilder.currentVersion();
+        int cmp = compareInputVersions(configured.trim(), builderVersion);
+        if (cmp == 0) {
+            log.info("[WikiEmbedding] Embedding input version: {} (matches builder)", configured);
+        } else if (cmp < 0) {
+            log.warn("[WikiEmbedding] Configured embedding input version {} is older than builder {}; "
+                    + "new embeddings will still be stamped with the configured value. "
+                    + "Clear mate.wiki.embedding-text-version-current to use the builder default.",
+                    configured, builderVersion);
+        } else {
+            throw new IllegalStateException(
+                    "Configured embedding input version " + configured + " is newer than builder version "
+                            + builderVersion + ". The builder code is older than the deployment config; "
+                            + "upgrade the application or clear mate.wiki.embedding-text-version-current.");
+        }
+    }
+
+    /**
+     * Compare version tags of the form {@code v\d+} numerically (so v2 > v10
+     * does not happen). Falls back to case-insensitive string compare when
+     * either side does not match the expected pattern.
+     */
+    static int compareInputVersions(String a, String b) {
+        Integer ai = parseNumericVersion(a);
+        Integer bi = parseNumericVersion(b);
+        if (ai != null && bi != null) {
+            return Integer.compare(ai, bi);
+        }
+        return a.compareToIgnoreCase(b);
+    }
+
+    private static Integer parseNumericVersion(String tag) {
+        if (tag == null || tag.length() < 2) return null;
+        if (tag.charAt(0) != 'v' && tag.charAt(0) != 'V') return null;
+        try {
+            return Integer.parseInt(tag.substring(1));
+        } catch (NumberFormatException e) {
+            return null;
+        }
+    }
+
     /**
      * 判断全局是否有可用的 embedding 能力（任何 enabled 的 embedding 模型配置）
      */
@@ -116,8 +190,10 @@ public class WikiEmbeddingService {
     /**
      * 批量嵌入指定 KB 中缺失 embedding 的 chunk。
      * <p>
-     * 只嵌入 embedding 为 NULL 或 embeddingModel 与当前解析出的模型不一致的 chunk。
-     * 模型切换时自动触发全量重嵌（通过 embedding_model 字段比对）。
+     * Pending criteria: embedding is NULL, the stored embedding_model differs
+     * from the currently-resolved model, or the stored embedding_text_version
+     * differs from the active builder version. Switching the embedding model
+     * or bumping the input format both trigger a full re-embed pass.
      */
     public int embedMissingChunks(Long kbId) {
         Resolved r = resolveForKb(kbId);
@@ -127,17 +203,21 @@ public class WikiEmbeddingService {
         }
 
         String modelName = r.modelName();
+        String inputVersion = currentInputVersion();
         List<WikiChunkEntity> pending = chunkMapper.selectList(
                 new LambdaQueryWrapper<WikiChunkEntity>()
                         .eq(WikiChunkEntity::getKbId, kbId)
                         .and(w -> w.isNull(WikiChunkEntity::getEmbedding)
-                                   .or().ne(WikiChunkEntity::getEmbeddingModel, modelName)));
+                                   .or().ne(WikiChunkEntity::getEmbeddingModel, modelName)
+                                   .or().isNull(WikiChunkEntity::getEmbeddingTextVersion)
+                                   .or().ne(WikiChunkEntity::getEmbeddingTextVersion, inputVersion)));
 
         if (pending.isEmpty()) {
             log.debug("[WikiEmbedding] No chunks need embedding for kbId={}", kbId);
             return 0;
         }
 
+        RawTitleLookup titleLookup = preloadTitlesFor(pending);
         int batchSize = Math.max(1, properties.getEmbeddingBatchSize());
         int maxChars = Math.max(500, properties.getEmbeddingMaxChars());
         int threshold = Math.max(1, properties.getEmbeddingConsecutiveFailureThreshold());
@@ -165,7 +245,7 @@ public class WikiEmbeddingService {
 
             // Short chunks: existing batch path
             if (!shortBatch.isEmpty()) {
-                int embedded = embedShortBatch(shortBatch, r.model(), modelName, kbId);
+                int embedded = embedShortBatch(shortBatch, r.model(), modelName, kbId, inputVersion, titleLookup);
                 total += embedded;
                 if (embedded == 0) {
                     consecutiveFailures++;
@@ -180,7 +260,7 @@ public class WikiEmbeddingService {
 
             // Long chunks: each goes through sub-segment split + mean pool
             for (WikiChunkEntity longChunk : longChunks) {
-                if (embedLongChunk(longChunk, r.model(), modelName, maxChars)) {
+                if (embedLongChunk(longChunk, r.model(), modelName, maxChars, inputVersion, titleLookup)) {
                     total++;
                     consecutiveFailures = 0;
                 } else {
@@ -218,15 +298,19 @@ public class WikiEmbeddingService {
      * Returns the number of chunks that were successfully embedded and persisted.
      */
     private int embedShortBatch(List<WikiChunkEntity> batch, EmbeddingModel model,
-                                 String modelName, Long kbId) {
+                                 String modelName, Long kbId,
+                                 String inputVersion, RawTitleLookup titleLookup) {
         try {
-            List<String> inputs = batch.stream().map(WikiChunkEntity::getContent).toList();
+            List<String> inputs = batch.stream()
+                    .map(c -> inputBuilder.build(c, titleLookup))
+                    .toList();
             EmbeddingResponse resp = model.call(new EmbeddingRequest(inputs, null));
             for (int i = 0; i < batch.size(); i++) {
                 float[] vec = resp.getResults().get(i).getOutput();
                 WikiChunkEntity chunk = batch.get(i);
                 chunk.setEmbedding(floatsToBytes(vec));
                 chunk.setEmbeddingModel(modelName);
+                chunk.setEmbeddingTextVersion(inputVersion);
                 chunkMapper.updateById(chunk);
             }
             return batch.size();
@@ -248,12 +332,24 @@ public class WikiEmbeddingService {
      * Returns true if at least one sub-segment succeeded and the chunk was persisted.
      */
     private boolean embedLongChunk(WikiChunkEntity chunk, EmbeddingModel model,
-                                    String modelName, int maxChars) {
-        List<String> segments = splitForEmbedding(chunk.getContent(), maxChars);
-        if (segments.isEmpty()) {
+                                    String modelName, int maxChars,
+                                    String inputVersion, RawTitleLookup titleLookup) {
+        // Prepend the metadata prefix to every sub-segment so the per-segment
+        // embeddings carry the same context before mean-pooling. The split
+        // budget is reduced by the prefix length to keep each enriched segment
+        // under the provider's per-input cap; the floor of 500 keeps the
+        // splitter from collapsing to single-char windows when a pathological
+        // metadata prefix appears.
+        String prefix = inputBuilder.buildPrefix(chunk, titleLookup);
+        int segmentBudget = Math.max(500, maxChars - prefix.length());
+        List<String> rawSegments = splitForEmbedding(chunk.getContent(), segmentBudget);
+        if (rawSegments.isEmpty()) {
             log.warn("[WikiEmbedding] Chunk {} produced no embeddable segments after split", chunk.getId());
             return false;
         }
+        List<String> segments = prefix.isEmpty()
+                ? rawSegments
+                : rawSegments.stream().map(s -> prefix + s).toList();
         log.info("[WikiEmbedding] Chunk {} ({} chars) split into {} sub-segments",
                 chunk.getId(), chunk.getContent().length(), segments.size());
 
@@ -286,6 +382,7 @@ public class WikiEmbeddingService {
         float[] pooled = averageAndNormalize(vectors);
         chunk.setEmbedding(floatsToBytes(pooled));
         chunk.setEmbeddingModel(modelName);
+        chunk.setEmbeddingTextVersion(inputVersion);
         chunkMapper.updateById(chunk);
         return true;
     }
@@ -374,6 +471,51 @@ public class WikiEmbeddingService {
         }
     }
 
+    /**
+     * Snapshot of how many chunks in a KB still need to be re-embedded
+     * against the current model + input version. Powers the admin "embedding
+     * drift" indicator without exposing internal pending logic.
+     */
+    public EmbeddingDrift describeDrift(Long kbId) {
+        String inputVersion = currentInputVersion();
+        Resolved r = resolveForKb(kbId);
+        String modelName = r == null ? null : r.modelName();
+
+        long totalEmbedded = chunkMapper.selectCount(
+                new LambdaQueryWrapper<WikiChunkEntity>()
+                        .eq(WikiChunkEntity::getKbId, kbId)
+                        .isNotNull(WikiChunkEntity::getEmbedding));
+
+        LambdaQueryWrapper<WikiChunkEntity> pendingQ = new LambdaQueryWrapper<WikiChunkEntity>()
+                .eq(WikiChunkEntity::getKbId, kbId)
+                .and(w -> {
+                    w.isNull(WikiChunkEntity::getEmbedding)
+                            .or().isNull(WikiChunkEntity::getEmbeddingTextVersion)
+                            .or().ne(WikiChunkEntity::getEmbeddingTextVersion, inputVersion);
+                    if (modelName != null) {
+                        w.or().ne(WikiChunkEntity::getEmbeddingModel, modelName);
+                    }
+                });
+        List<WikiChunkEntity> pending = chunkMapper.selectList(pendingQ);
+
+        long pendingChars = 0;
+        for (WikiChunkEntity c : pending) {
+            if (c.getContent() != null) pendingChars += c.getContent().length();
+        }
+        // Provider-agnostic token approximation; ~4 chars per token covers
+        // English and is conservative for Chinese (which is denser per token).
+        long pendingTokens = pendingChars / 4;
+
+        return new EmbeddingDrift(inputVersion, pending.size(), totalEmbedded, pendingTokens);
+    }
+
+    /** Result of {@link #describeDrift(Long)}; serialized into KB stats. */
+    public record EmbeddingDrift(
+            String currentEmbeddingTextVersion,
+            int pendingReembedChunks,
+            long totalEmbeddedChunks,
+            long pendingReembedEstimatedTokens) {}
+
     /**
      * 清空指定 KB 的所有 embedding（模型切换时调用）
      */
@@ -381,10 +523,22 @@ public class WikiEmbeddingService {
         chunkMapper.update(null, new LambdaUpdateWrapper<WikiChunkEntity>()
                 .eq(WikiChunkEntity::getKbId, kbId)
                 .set(WikiChunkEntity::getEmbedding, null)
-                .set(WikiChunkEntity::getEmbeddingModel, null));
+                .set(WikiChunkEntity::getEmbeddingModel, null)
+                .set(WikiChunkEntity::getEmbeddingTextVersion, null));
         log.info("[WikiEmbedding] Cleared all embeddings for kbId={}", kbId);
     }
 
+    private RawTitleLookup preloadTitlesFor(List<WikiChunkEntity> chunks) {
+        if (chunks == null || chunks.isEmpty()) {
+            return RawTitleLookups.empty();
+        }
+        Set<Long> rawIds = new HashSet<>();
+        for (WikiChunkEntity c : chunks) {
+            if (c.getRawId() != null) rawIds.add(c.getRawId());
+        }
+        return RawTitleLookups.preload(rawMaterialMapper, rawIds);
+    }
+
     // ==================== 私有 helper ====================
 
     private ModelConfigEntity safeGetModel(Long id) {

mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql

@@ -0,0 +1,6 @@
+-- V104: track which input format a chunk's stored embedding was generated against.
+-- The embedding input builder concatenates raw title / header breadcrumb / page
+-- number alongside chunk content; bumping the builder's CURRENT_INPUT_VERSION
+-- forces a re-embed pass without changing the model. NULL is treated as the
+-- legacy content-only format and re-embedded lazily on the next pass.
+ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS embedding_text_version VARCHAR(32) NULL;

mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql

@@ -0,0 +1,9 @@
+-- V104: track which input format a chunk's stored embedding was generated against.
+-- The embedding input builder concatenates raw title / header breadcrumb / page
+-- number alongside chunk content; bumping the builder's CURRENT_INPUT_VERSION
+-- forces a re-embed pass without changing the model. NULL is treated as the
+-- legacy content-only format and re-embedded lazily on the next pass.
+-- MySQL lacks `ADD COLUMN IF NOT EXISTS`; use INFORMATION_SCHEMA guard instead.
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'embedding_text_version');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN embedding_text_version VARCHAR(32) NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;