diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java index 5ec7d38e..4df212e6 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java @@ -130,6 +130,22 @@ public class WikiProperties { */ private int embeddingMaxChars = 6000; + /** + * Expected embedding-input format version. The authoritative source is + * the builder's {@code CURRENT_INPUT_VERSION} constant; this property + * exists for staged rollouts and ops overrides. + *

+ * Behavior on startup: + *

+ */ + private String embeddingTextVersionCurrent = ""; + /** * Circuit-breaker threshold: abort an embedding pass after this many * consecutive batch failures (auth / rate-limit / network errors that diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java index 854d9d3a..484be855 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java @@ -35,6 +35,7 @@ public class WikiRelationController { private final HybridRetriever hybridRetriever; private final ApplicationEventPublisher eventPublisher; private final ObjectMapper objectMapper; + private final WikiEmbeddingService embeddingService; // ==================== RFC-029: Relations ==================== @@ -104,11 +105,14 @@ public class WikiRelationController { .filter(j -> "running".equals(j.getStatus())) .count(); + WikiEmbeddingService.EmbeddingDrift drift = embeddingService.describeDrift(kbId); + return Map.of( "pageCount", pageCount, "enrichedPageCount", enrichedCount, "failedJobCount", failedJobCount, - "runningJobCount", runningJobCount + "runningJobCount", runningJobCount, + "embeddingDrift", drift ); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java index aef31ce6..6c7abf0f 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java @@ -51,6 +51,15 @@ public class WikiChunkEntity { /** RFC-011：生成该 embedding 的模型名称（切模型时需全量重嵌） */ private String embeddingModel; + /** + * Identifies the input format used to produce the stored embedding. + *

+ * Set to the embedding input builder's current version on every write. + * NULL signals a legacy content-only embedding from before the builder + * existed and is treated as stale on the next re-embed pass. + */ + private String embeddingTextVersion; + /** RFC-051: source page number (PDF/PPTX) when known; null otherwise. */ private Integer pageNumber; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java new file mode 100644 index 00000000..1ffe5f3b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java @@ -0,0 +1,15 @@ +package vip.mate.wiki.service; + +/** + * Resolves {@code rawId -> rawTitle} for embedding-time enrichment. + *

+ * Implementations may be naive (one DB hit per call), batch-preloaded for a + * given job, or backed by an in-memory snapshot shared with an ingest-scope + * page index. Callers must tolerate {@code null} for unknown / deleted ids. + */ +@FunctionalInterface +public interface RawTitleLookup { + + /** @return raw material title, or {@code null} when the id is unknown */ + String titleFor(Long rawId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java new file mode 100644 index 00000000..57aef55b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java @@ -0,0 +1,42 @@ +package vip.mate.wiki.service; + +import vip.mate.wiki.dto.RawTitleRef; +import vip.mate.wiki.repository.WikiRawMaterialMapper; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Factory helpers for {@link RawTitleLookup}. + */ +public final class RawTitleLookups { + + private RawTitleLookups() {} + + /** Lookup that always returns {@code null}; useful for callers without raw context. */ + public static RawTitleLookup empty() { + return id -> null; + } + + /** Lookup backed by a pre-resolved map (e.g. from an ingest-scope snapshot). */ + public static RawTitleLookup of(Map titlesById) { + Map snapshot = titlesById == null ? Map.of() : Map.copyOf(titlesById); + return snapshot::get; + } + + /** + * Preload titles for the given ids in a single batch query and return a + * lookup over the resulting map. Unknown ids resolve to {@code null}. + */ + public static RawTitleLookup preload(WikiRawMaterialMapper mapper, Collection rawIds) { + if (mapper == null || rawIds == null || rawIds.isEmpty()) { + return empty(); + } + Map titles = new HashMap<>(rawIds.size()); + for (RawTitleRef ref : mapper.selectBatchTitles(rawIds)) { + titles.put(ref.id(), ref.title()); + } + return of(titles); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java new file mode 100644 index 00000000..53428923 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java @@ -0,0 +1,82 @@ +package vip.mate.wiki.service; + +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiChunkEntity; + +/** + * Produces the text fed to the embedding model for a chunk. + *

+ * Naive content-only embeddings make short or context-poor chunks (e.g. a + * standalone sentence like "accuracy improved by 12%") near-indistinguishable + * in vector space. Prefixing the model input with already-available metadata — + * source title, header breadcrumb, source section, page number — preserves + * the semantic neighborhood the chunk came from without changing the storage + * model. + *

+ * Bump {@link #CURRENT_INPUT_VERSION} whenever the prefix format changes. The + * embedding pass treats any chunk whose stored {@code embedding_text_version} + * differs from the current value as stale and re-embeds it. + */ +@Component +public class WikiEmbeddingInputBuilder { + + /** + * Version tag stamped onto every chunk that this builder embeds. + * Increment when the prefix format below changes in a way that should + * trigger a re-embed pass. The string is opaque; "v1", "v2", ... is fine. + */ + public static final String CURRENT_INPUT_VERSION = "v1"; + + /** + * Build the embedding input string for a chunk. Metadata fields that are + * null or blank are skipped so empty values never produce stray headers. + * Falls back to the chunk content alone when no metadata is available. + */ + public String build(WikiChunkEntity chunk, RawTitleLookup lookup) { + if (chunk == null) { + return ""; + } + String content = chunk.getContent() == null ? "" : chunk.getContent(); + String prefix = buildPrefix(chunk, lookup); + return prefix.isEmpty() ? content : prefix + content; + } + + /** + * Build only the metadata prefix for a chunk. Useful when callers need to + * split content into sub-segments and prepend the prefix to each one so + * the metadata participates in every per-segment embedding before pooling. + * Returns an empty string when no metadata is available; otherwise ends + * with a blank line so the content reads as a separate paragraph. + */ + public String buildPrefix(WikiChunkEntity chunk, RawTitleLookup lookup) { + if (chunk == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + String rawTitle = (lookup == null || chunk.getRawId() == null) + ? null : lookup.titleFor(chunk.getRawId()); + appendLine(sb, "Source", rawTitle); + appendLine(sb, "Section", chunk.getHeaderBreadcrumb()); + appendLine(sb, "Subsection", chunk.getSourceSection()); + if (chunk.getPageNumber() != null) { + appendLine(sb, "Page", String.valueOf(chunk.getPageNumber())); + } + if (sb.length() == 0) { + return ""; + } + sb.append('\n'); + return sb.toString(); + } + + /** @return the version tag this builder stamps onto each chunk it embeds */ + public String currentVersion() { + return CURRENT_INPUT_VERSION; + } + + private static void appendLine(StringBuilder sb, String label, String value) { + if (value == null || value.isBlank()) { + return; + } + sb.append(label).append(": ").append(value.strip()).append('\n'); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java index b4ccf373..3cd55eb9 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java @@ -2,6 +2,7 @@ package vip.mate.wiki.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.embedding.EmbeddingModel; @@ -17,11 +18,14 @@ import vip.mate.wiki.WikiProperties; import vip.mate.wiki.model.WikiChunkEntity; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.repository.WikiChunkMapper; +import vip.mate.wiki.repository.WikiRawMaterialMapper; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * RFC-011 + Embedding-UI-Config: Wiki 嵌入服务 @@ -42,16 +46,86 @@ import java.util.List; public class WikiEmbeddingService { private final WikiChunkMapper chunkMapper; + private final WikiRawMaterialMapper rawMaterialMapper; private final WikiProperties properties; private final EmbeddingModelFactory factory; private final ModelConfigService modelConfigService; private final WikiKnowledgeBaseService kbService; private final SystemSettingMapper systemSettingMapper; private final vip.mate.llm.service.ModelProviderService modelProviderService; + private final WikiEmbeddingInputBuilder inputBuilder; /** 系统默认 embedding 模型的 mate_system_setting key */ public static final String SYSTEM_SETTING_DEFAULT_EMBEDDING_ID = "embedding.default.model.id"; + /** + * Returns the embedding input format version this service stamps onto + * each chunk. The builder's constant is the source of truth; the + * {@code mate.wiki.embedding-text-version-current} property exists only + * to support ops overrides and is validated against the builder at + * startup ({@link #verifyConfiguredInputVersion()}). + */ + public String currentInputVersion() { + String configured = properties.getEmbeddingTextVersionCurrent(); + return (configured == null || configured.isBlank()) ? inputBuilder.currentVersion() : configured.trim(); + } + + /** + * Validate the configured embedding input version against the builder + * constant on startup. A blank config is normal (the builder version is + * used). A config below the builder is allowed with a WARN so a KB can + * be embedded against an older format during a gradual rollback. A + * config above the builder fails fast — it almost always means the + * config was deployed ahead of the code. + */ + @PostConstruct + void verifyConfiguredInputVersion() { + String configured = properties.getEmbeddingTextVersionCurrent(); + if (configured == null || configured.isBlank()) { + log.info("[WikiEmbedding] Embedding input version: {} (from builder)", inputBuilder.currentVersion()); + return; + } + String builderVersion = inputBuilder.currentVersion(); + int cmp = compareInputVersions(configured.trim(), builderVersion); + if (cmp == 0) { + log.info("[WikiEmbedding] Embedding input version: {} (matches builder)", configured); + } else if (cmp < 0) { + log.warn("[WikiEmbedding] Configured embedding input version {} is older than builder {}; " + + "new embeddings will still be stamped with the configured value. " + + "Clear mate.wiki.embedding-text-version-current to use the builder default.", + configured, builderVersion); + } else { + throw new IllegalStateException( + "Configured embedding input version " + configured + " is newer than builder version " + + builderVersion + ". The builder code is older than the deployment config; " + + "upgrade the application or clear mate.wiki.embedding-text-version-current."); + } + } + + /** + * Compare version tags of the form {@code v\d+} numerically (so v2 > v10 + * does not happen). Falls back to case-insensitive string compare when + * either side does not match the expected pattern. + */ + static int compareInputVersions(String a, String b) { + Integer ai = parseNumericVersion(a); + Integer bi = parseNumericVersion(b); + if (ai != null && bi != null) { + return Integer.compare(ai, bi); + } + return a.compareToIgnoreCase(b); + } + + private static Integer parseNumericVersion(String tag) { + if (tag == null || tag.length() < 2) return null; + if (tag.charAt(0) != 'v' && tag.charAt(0) != 'V') return null; + try { + return Integer.parseInt(tag.substring(1)); + } catch (NumberFormatException e) { + return null; + } + } + /** * 判断全局是否有可用的 embedding 能力（任何 enabled 的 embedding 模型配置） */ @@ -116,8 +190,10 @@ public class WikiEmbeddingService { /** * 批量嵌入指定 KB 中缺失 embedding 的 chunk。 *

- * 只嵌入 embedding 为 NULL 或 embeddingModel 与当前解析出的模型不一致的 chunk。 - * 模型切换时自动触发全量重嵌（通过 embedding_model 字段比对）。 + * Pending criteria: embedding is NULL, the stored embedding_model differs + * from the currently-resolved model, or the stored embedding_text_version + * differs from the active builder version. Switching the embedding model + * or bumping the input format both trigger a full re-embed pass. */ public int embedMissingChunks(Long kbId) { Resolved r = resolveForKb(kbId); @@ -127,17 +203,21 @@ public class WikiEmbeddingService { } String modelName = r.modelName(); + String inputVersion = currentInputVersion(); List pending = chunkMapper.selectList( new LambdaQueryWrapper() .eq(WikiChunkEntity::getKbId, kbId) .and(w -> w.isNull(WikiChunkEntity::getEmbedding) - .or().ne(WikiChunkEntity::getEmbeddingModel, modelName))); + .or().ne(WikiChunkEntity::getEmbeddingModel, modelName) + .or().isNull(WikiChunkEntity::getEmbeddingTextVersion) + .or().ne(WikiChunkEntity::getEmbeddingTextVersion, inputVersion))); if (pending.isEmpty()) { log.debug("[WikiEmbedding] No chunks need embedding for kbId={}", kbId); return 0; } + RawTitleLookup titleLookup = preloadTitlesFor(pending); int batchSize = Math.max(1, properties.getEmbeddingBatchSize()); int maxChars = Math.max(500, properties.getEmbeddingMaxChars()); int threshold = Math.max(1, properties.getEmbeddingConsecutiveFailureThreshold()); @@ -165,7 +245,7 @@ public class WikiEmbeddingService { // Short chunks: existing batch path if (!shortBatch.isEmpty()) { - int embedded = embedShortBatch(shortBatch, r.model(), modelName, kbId); + int embedded = embedShortBatch(shortBatch, r.model(), modelName, kbId, inputVersion, titleLookup); total += embedded; if (embedded == 0) { consecutiveFailures++; @@ -180,7 +260,7 @@ public class WikiEmbeddingService { // Long chunks: each goes through sub-segment split + mean pool for (WikiChunkEntity longChunk : longChunks) { - if (embedLongChunk(longChunk, r.model(), modelName, maxChars)) { + if (embedLongChunk(longChunk, r.model(), modelName, maxChars, inputVersion, titleLookup)) { total++; consecutiveFailures = 0; } else { @@ -218,15 +298,19 @@ public class WikiEmbeddingService { * Returns the number of chunks that were successfully embedded and persisted. */ private int embedShortBatch(List batch, EmbeddingModel model, - String modelName, Long kbId) { + String modelName, Long kbId, + String inputVersion, RawTitleLookup titleLookup) { try { - List inputs = batch.stream().map(WikiChunkEntity::getContent).toList(); + List inputs = batch.stream() + .map(c -> inputBuilder.build(c, titleLookup)) + .toList(); EmbeddingResponse resp = model.call(new EmbeddingRequest(inputs, null)); for (int i = 0; i < batch.size(); i++) { float[] vec = resp.getResults().get(i).getOutput(); WikiChunkEntity chunk = batch.get(i); chunk.setEmbedding(floatsToBytes(vec)); chunk.setEmbeddingModel(modelName); + chunk.setEmbeddingTextVersion(inputVersion); chunkMapper.updateById(chunk); } return batch.size(); @@ -248,12 +332,24 @@ public class WikiEmbeddingService { * Returns true if at least one sub-segment succeeded and the chunk was persisted. */ private boolean embedLongChunk(WikiChunkEntity chunk, EmbeddingModel model, - String modelName, int maxChars) { - List segments = splitForEmbedding(chunk.getContent(), maxChars); - if (segments.isEmpty()) { + String modelName, int maxChars, + String inputVersion, RawTitleLookup titleLookup) { + // Prepend the metadata prefix to every sub-segment so the per-segment + // embeddings carry the same context before mean-pooling. The split + // budget is reduced by the prefix length to keep each enriched segment + // under the provider's per-input cap; the floor of 500 keeps the + // splitter from collapsing to single-char windows when a pathological + // metadata prefix appears. + String prefix = inputBuilder.buildPrefix(chunk, titleLookup); + int segmentBudget = Math.max(500, maxChars - prefix.length()); + List rawSegments = splitForEmbedding(chunk.getContent(), segmentBudget); + if (rawSegments.isEmpty()) { log.warn("[WikiEmbedding] Chunk {} produced no embeddable segments after split", chunk.getId()); return false; } + List segments = prefix.isEmpty() + ? rawSegments + : rawSegments.stream().map(s -> prefix + s).toList(); log.info("[WikiEmbedding] Chunk {} ({} chars) split into {} sub-segments", chunk.getId(), chunk.getContent().length(), segments.size()); @@ -286,6 +382,7 @@ public class WikiEmbeddingService { float[] pooled = averageAndNormalize(vectors); chunk.setEmbedding(floatsToBytes(pooled)); chunk.setEmbeddingModel(modelName); + chunk.setEmbeddingTextVersion(inputVersion); chunkMapper.updateById(chunk); return true; } @@ -374,6 +471,51 @@ public class WikiEmbeddingService { } } + /** + * Snapshot of how many chunks in a KB still need to be re-embedded + * against the current model + input version. Powers the admin "embedding + * drift" indicator without exposing internal pending logic. + */ + public EmbeddingDrift describeDrift(Long kbId) { + String inputVersion = currentInputVersion(); + Resolved r = resolveForKb(kbId); + String modelName = r == null ? null : r.modelName(); + + long totalEmbedded = chunkMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiChunkEntity::getKbId, kbId) + .isNotNull(WikiChunkEntity::getEmbedding)); + + LambdaQueryWrapper pendingQ = new LambdaQueryWrapper() + .eq(WikiChunkEntity::getKbId, kbId) + .and(w -> { + w.isNull(WikiChunkEntity::getEmbedding) + .or().isNull(WikiChunkEntity::getEmbeddingTextVersion) + .or().ne(WikiChunkEntity::getEmbeddingTextVersion, inputVersion); + if (modelName != null) { + w.or().ne(WikiChunkEntity::getEmbeddingModel, modelName); + } + }); + List pending = chunkMapper.selectList(pendingQ); + + long pendingChars = 0; + for (WikiChunkEntity c : pending) { + if (c.getContent() != null) pendingChars += c.getContent().length(); + } + // Provider-agnostic token approximation; ~4 chars per token covers + // English and is conservative for Chinese (which is denser per token). + long pendingTokens = pendingChars / 4; + + return new EmbeddingDrift(inputVersion, pending.size(), totalEmbedded, pendingTokens); + } + + /** Result of {@link #describeDrift(Long)}; serialized into KB stats. */ + public record EmbeddingDrift( + String currentEmbeddingTextVersion, + int pendingReembedChunks, + long totalEmbeddedChunks, + long pendingReembedEstimatedTokens) {} + /** * 清空指定 KB 的所有 embedding（模型切换时调用） */ @@ -381,10 +523,22 @@ public class WikiEmbeddingService { chunkMapper.update(null, new LambdaUpdateWrapper() .eq(WikiChunkEntity::getKbId, kbId) .set(WikiChunkEntity::getEmbedding, null) - .set(WikiChunkEntity::getEmbeddingModel, null)); + .set(WikiChunkEntity::getEmbeddingModel, null) + .set(WikiChunkEntity::getEmbeddingTextVersion, null)); log.info("[WikiEmbedding] Cleared all embeddings for kbId={}", kbId); } + private RawTitleLookup preloadTitlesFor(List chunks) { + if (chunks == null || chunks.isEmpty()) { + return RawTitleLookups.empty(); + } + Set rawIds = new HashSet<>(); + for (WikiChunkEntity c : chunks) { + if (c.getRawId() != null) rawIds.add(c.getRawId()); + } + return RawTitleLookups.preload(rawMaterialMapper, rawIds); + } + // ==================== 私有 helper ==================== private ModelConfigEntity safeGetModel(Long id) { diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql b/mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql new file mode 100644 index 00000000..2b399048 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql @@ -0,0 +1,6 @@ +-- V104: track which input format a chunk's stored embedding was generated against. +-- The embedding input builder concatenates raw title / header breadcrumb / page +-- number alongside chunk content; bumping the builder's CURRENT_INPUT_VERSION +-- forces a re-embed pass without changing the model. NULL is treated as the +-- legacy content-only format and re-embedded lazily on the next pass. +ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS embedding_text_version VARCHAR(32) NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql new file mode 100644 index 00000000..6c50d76b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql @@ -0,0 +1,9 @@ +-- V104: track which input format a chunk's stored embedding was generated against. +-- The embedding input builder concatenates raw title / header breadcrumb / page +-- number alongside chunk content; bumping the builder's CURRENT_INPUT_VERSION +-- forces a re-embed pass without changing the model. NULL is treated as the +-- legacy content-only format and re-embedded lazily on the next pass. +-- MySQL lacks `ADD COLUMN IF NOT EXISTS`; use INFORMATION_SCHEMA guard instead. +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'embedding_text_version'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN embedding_text_version VARCHAR(32) NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;