diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/EnrichmentPlan.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/EnrichmentPlan.java new file mode 100644 index 00000000..1baed3b7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/EnrichmentPlan.java @@ -0,0 +1,22 @@ +package vip.mate.wiki.dto; + +import java.util.List; + +/** + * RFC-051 PR-5b: replacement-plan output of {@code WikiLinkEnrichmentService}. + *

+ * The LLM is asked to return a list of surgical wrap operations rather than + * a full rewritten page body. Each {@link EnrichmentReplacement} describes + * "wrap the Nth occurrence of {@code original} with {@code replacement}". + * Java validates and applies them, guaranteeing no non-link prose changes. + */ +public record EnrichmentPlan(List replacements) { + + public EnrichmentPlan { + if (replacements == null) replacements = List.of(); + } + + public boolean isEmpty() { + return replacements.isEmpty(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/EnrichmentReplacement.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/EnrichmentReplacement.java new file mode 100644 index 00000000..e1300cc3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/EnrichmentReplacement.java @@ -0,0 +1,23 @@ +package vip.mate.wiki.dto; + +/** + * RFC-051 PR-5b: one entry of {@link EnrichmentPlan}. + *

+ * Semantics: + *

+ * Default occurrence is 1 when the LLM omits the field. + */ +public record EnrichmentReplacement(String original, String replacement, int occurrence) { + + public EnrichmentReplacement { + if (occurrence <= 0) occurrence = 1; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEnrichmentApplier.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEnrichmentApplier.java new file mode 100644 index 00000000..c0060166 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEnrichmentApplier.java @@ -0,0 +1,198 @@ +package vip.mate.wiki.service; + +import vip.mate.wiki.dto.EnrichmentPlan; +import vip.mate.wiki.dto.EnrichmentReplacement; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * RFC-051 PR-5b: validate and apply a replacement-plan enrichment without + * letting the LLM touch non-link prose. + *

+ * The applier is intentionally pure: no Spring beans, no I/O. That keeps the + * critical path testable as a plain JUnit class and lets the round-trip + * invariant ("stripped text equals original") get pinned down in one place. + * + *

Invariants

+ *
    + *
  1. Each {@link EnrichmentReplacement#replacement()} must be a wikilink + * in {@code [[slug]]} or {@code [[slug|label]]} form.
    2. + *
  2. The visible text of the replacement (slug for the bare form, label + * for the alias form) must equal {@link EnrichmentReplacement#original()} + * byte-for-byte.
    3. + *
  3. After applying every replacement, stripping all wikilinks back to + * their visible text must yield exactly the input page content.
    4. + *
+ * Failures abort the apply and return {@link Result#rejected(String)}; the + * caller is expected to leave the page untouched on rejection. + */ +public final class WikiEnrichmentApplier { + + /** Default cap so a runaway LLM can't propose 1000 wraps per page. */ + public static final int DEFAULT_MAX_REPLACEMENTS = 50; + + /** Matches {@code [[anything]]} (greedy through to the next ]] but not nested). */ + private static final Pattern WIKILINK = Pattern.compile("\\[\\[([^\\[\\]]+)]]"); + + /** Matches {@code [[slug]]} or {@code [[slug|label]]} on the replacement string. */ + private static final Pattern REPLACEMENT_SHAPE = + Pattern.compile("^\\[\\[([^\\[\\]|]+)(?:\\|([^\\[\\]]+))?]]$"); + + private WikiEnrichmentApplier() {} + + public static Result apply(String originalContent, EnrichmentPlan plan) { + return apply(originalContent, plan, DEFAULT_MAX_REPLACEMENTS); + } + + public static Result apply(String originalContent, EnrichmentPlan plan, int maxReplacements) { + if (originalContent == null) return Result.rejected("content is null"); + if (plan == null || plan.isEmpty()) { + return Result.unchanged(originalContent); + } + if (plan.replacements().size() > maxReplacements) { + return Result.rejected("too many replacements: " + + plan.replacements().size() + " > " + maxReplacements); + } + + // 1) Per-original index of all candidate positions in the original text, + // skipping positions that fall inside an existing wikilink. + java.util.Map> positionsByOriginal = new java.util.HashMap<>(); + boolean[] insideWikilink = computeWikilinkMask(originalContent); + + // 2) Plan splices: for each replacement pick positions[occurrence-1]. + List splices = new ArrayList<>(); // [start, end, replacementIndex] + List replacementTexts = new ArrayList<>(); + for (EnrichmentReplacement r : plan.replacements()) { + String original = r.original(); + String replacement = r.replacement(); + if (original == null || original.isEmpty()) { + return Result.rejected("empty original in replacement"); + } + Matcher shape = REPLACEMENT_SHAPE.matcher(replacement == null ? "" : replacement); + if (!shape.matches()) { + return Result.rejected("replacement is not a wikilink form: " + replacement); + } + String slug = shape.group(1).trim(); + String label = shape.group(2); + String visible = (label == null) ? slug : label; + if (!visible.equals(original)) { + return Result.rejected("visible text mismatch: replacement='" + + replacement + "' must render '" + original + "'"); + } + + List positions = positionsByOriginal.computeIfAbsent(original, + o -> findPositions(originalContent, o, insideWikilink)); + int idx = r.occurrence() - 1; + if (idx < 0 || idx >= positions.size()) { + // Skip silently — the page may have been re-edited since the LLM saw it. + continue; + } + int start = positions.get(idx); + splices.add(new int[]{start, start + original.length(), replacementTexts.size()}); + replacementTexts.add(replacement); + } + + if (splices.isEmpty()) { + return Result.unchanged(originalContent); + } + + // 3) Apply in reverse offset order so earlier indices don't shift. + splices.sort((a, b) -> Integer.compare(b[0], a[0])); + StringBuilder sb = new StringBuilder(originalContent); + java.util.Set claimed = new java.util.HashSet<>(); + int applied = 0; + for (int[] sp : splices) { + int start = sp[0]; + int end = sp[1]; + // Reject overlapping splices defensively. + for (int i = start; i < end; i++) { + if (claimed.contains(i)) { + return Result.rejected("overlapping splice at " + start); + } + } + sb.replace(start, end, replacementTexts.get(sp[2])); + for (int i = start; i < end; i++) claimed.add(i); + applied++; + } + + // 4) Round-trip: stripping wikilinks from the result must reproduce input. + String enriched = sb.toString(); + if (!stripWikilinks(enriched).equals(stripWikilinks(originalContent))) { + return Result.rejected("round-trip invariant violated"); + } + return Result.applied(enriched, applied); + } + + /** + * Strip every {@code [[...]]} down to its visible text: + *
    + *
  • {@code [[slug]]} → {@code slug}
    • + *
  • {@code [[slug|label]]} → {@code label}
    • + *
+ * Used both for round-trip validation and for diff testing. + */ + public static String stripWikilinks(String content) { + if (content == null) return ""; + Matcher m = WIKILINK.matcher(content); + StringBuilder out = new StringBuilder(content.length()); + int last = 0; + while (m.find()) { + out.append(content, last, m.start()); + String inner = m.group(1); + int pipe = inner.indexOf('|'); + String visible = pipe >= 0 ? inner.substring(pipe + 1) : inner; + out.append(visible); + last = m.end(); + } + out.append(content, last, content.length()); + return out.toString(); + } + + private static boolean[] computeWikilinkMask(String content) { + boolean[] mask = new boolean[content.length()]; + Matcher m = WIKILINK.matcher(content); + while (m.find()) { + for (int i = m.start(); i < m.end(); i++) mask[i] = true; + } + return mask; + } + + private static List findPositions(String content, String needle, boolean[] insideWikilink) { + List out = new ArrayList<>(); + if (needle.isEmpty()) return out; + int from = 0; + while (from <= content.length() - needle.length()) { + int p = content.indexOf(needle, from); + if (p < 0) break; + // Skip if any byte of the match falls inside an existing wikilink. + boolean overlap = false; + for (int i = p; i < p + needle.length(); i++) { + if (insideWikilink[i]) { overlap = true; break; } + } + if (!overlap) out.add(p); + from = p + 1; + } + return out; + } + + /** + * Outcome of {@link #apply(String, EnrichmentPlan)}. + */ + public sealed interface Result permits Result.Applied, Result.Unchanged, Result.Rejected { + + String content(); + + static Result applied(String content, int count) { return new Applied(content, count); } + static Result unchanged(String content) { return new Unchanged(content); } + static Result rejected(String reason) { return new Rejected(reason); } + + record Applied(String content, int replacementCount) implements Result {} + record Unchanged(String content) implements Result {} + record Rejected(String reason) implements Result { + @Override public String content() { return null; } + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java index 1cf12c6b..dcad0643 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java @@ -1,5 +1,7 @@ package vip.mate.wiki.service; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.messages.SystemMessage; @@ -9,9 +11,12 @@ import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.stereotype.Service; import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.EnrichmentPlan; +import vip.mate.wiki.dto.EnrichmentReplacement; import vip.mate.wiki.job.WikiModelRoutingService; import vip.mate.wiki.model.WikiPageEntity; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -19,9 +24,14 @@ import java.util.concurrent.Semaphore; import java.util.stream.Collectors; /** - * RFC-031: Lightweight wikilink enrichment service. - * Adds [[slug]] cross-references to page content without modifying - * the actual text. Corresponds to llm_wiki's enrich-wikilinks.ts. + * RFC-031 + RFC-051 PR-5b: lightweight wikilink enrichment. + *

+ * Earlier versions asked the LLM for a fully rewritten page and trusted the + * response if it was "long enough" — which let weak models silently drop + * paragraphs, translate prose, or rephrase claims while pretending to only + * add brackets. This rewrite switches to a replacement plan: the LLM + * proposes wraps, Java applies them surgically, and a round-trip check + * guarantees the non-link prose is unchanged byte-for-byte. */ @Slf4j @Service @@ -31,12 +41,13 @@ public class WikiLinkEnrichmentService { private final WikiPageService pageService; private final WikiModelRoutingService routingService; private final WikiProperties wikiProperties; + private final ObjectMapper objectMapper; private static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor(); /** - * Enrich a single page with [[wikilinks]]. + * Enrich a single page with [[wikilinks]] using a replacement plan. */ public void enrichPage(Long pageId, Long modelId) { WikiPageEntity page = pageService.getById(pageId); @@ -44,18 +55,11 @@ public class WikiLinkEnrichmentService { String index = buildIndexPrompt(page.getKbId()); ChatModel chatModel = routingService.buildChatModel(modelId); - String enriched = callEnrichLlm(chatModel, page.getContent(), index); - - if (enriched != null - && enriched.length() >= page.getContent().length() * wikiProperties.getWikilinkMinContentRatio()) { - page.setContent(enriched); - page.setOutgoingLinks(pageService.extractLinksAsJson(enriched)); - pageService.updateById(page); - } + applyEnrichment(page, chatModel, index); } /** - * Batch-enrich all pages in a KB (e.g. after initial ingest). + * Batch-enrich all pages in a KB. */ public void enrichAllPages(Long kbId, Long modelId) { List pages = pageService.listByKbIdWithContent(kbId); @@ -75,45 +79,121 @@ public class WikiLinkEnrichmentService { } private void enrichPageWithIndex(WikiPageEntity page, Long modelId, String index) { + if (page == null || page.getContent() == null) return; ChatModel chatModel = routingService.buildChatModel(modelId); - String enriched = callEnrichLlm(chatModel, page.getContent(), index); - if (enriched != null - && enriched.length() >= page.getContent().length() * wikiProperties.getWikilinkMinContentRatio()) { - page.setContent(enriched); - page.setOutgoingLinks(pageService.extractLinksAsJson(enriched)); + applyEnrichment(page, chatModel, index); + } + + private void applyEnrichment(WikiPageEntity page, ChatModel chatModel, String index) { + EnrichmentPlan plan = requestPlan(chatModel, page.getContent(), index, page.getSlug()); + if (plan == null || plan.isEmpty()) { + return; // LLM had nothing to add or call failed; leave page alone. + } + WikiEnrichmentApplier.Result result = WikiEnrichmentApplier.apply(page.getContent(), plan); + if (result instanceof WikiEnrichmentApplier.Result.Rejected rejected) { + log.warn("[WikiEnrich] Plan rejected for slug={}: {}", page.getSlug(), rejected.reason()); + return; + } + if (result instanceof WikiEnrichmentApplier.Result.Unchanged) { + return; + } + if (result instanceof WikiEnrichmentApplier.Result.Applied applied) { + page.setContent(applied.content()); + page.setOutgoingLinks(pageService.extractLinksAsJson(applied.content())); pageService.updateById(page); + log.info("[WikiEnrich] Applied {} replacements on slug={}", applied.replacementCount(), page.getSlug()); } } - private String callEnrichLlm(ChatModel model, String content, String index) { + private EnrichmentPlan requestPlan(ChatModel chatModel, String content, String index, String slug) { String systemPrompt = """ You are a wiki cross-referencing assistant. - Your ONLY job: add [[slug]] markers around entity and concept names that appear in the wiki index. - Rules: - - Do NOT change any content, rewrite sentences, or add new text. - - Do NOT modify YAML frontmatter. - - Only wrap existing words/phrases with [[ and ]]. - - Use the exact slug from the wiki index (not the title). - - Return the COMPLETE page text with [[wikilinks]] added. + Your ONLY job: emit a JSON replacement plan that wraps existing words/phrases + with [[wikilinks]] from the supplied wiki index. + + Strict output contract — return ONLY this JSON object, nothing else: + { + "replacements": [ + {"original": "", + "replacement": "[[]]" or "[[|