From 48024e4a831afa7f98890d252b519e03ea5b2fcc Mon Sep 17 00:00:00 2001 From: matevip Date: Fri, 12 Jun 2026 15:17:32 +0800 Subject: [PATCH] fix(wiki): dedup pages by title and bound route prompt growth (#321) --- .../java/vip/mate/wiki/WikiProperties.java | 23 ++ .../wiki/controller/WikiAdminController.java | 19 ++ .../mate/wiki/service/WikiPageService.java | 234 ++++++++++++++++++ .../wiki/service/WikiProcessingService.java | 106 +++++++- .../WikiMergeDuplicateTitlesE2ETest.java | 150 +++++++++++ .../wiki/service/WikiPageServiceTest.java | 73 ++++++ 6 files changed, 598 insertions(+), 7 deletions(-) create mode 100644 mateclaw-server/src/test/java/vip/mate/wiki/service/WikiMergeDuplicateTitlesE2ETest.java diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java index 4267f2b5..e4a64e24 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java @@ -56,6 +56,29 @@ public class WikiProperties { /** 注入 agent prompt 的最大字符数 */ private int maxContextChars = 10000; + /** + * Hard cap on the existing-pages index injected into the route / batch-create + * prompts. The index lists every non-archived page in the KB so the router can + * decide create-vs-update and emit cross-links. Without a cap it grows linearly + * with the KB and eventually overflows the model context window + * ("Prompt exceeds max length"). When the rendered index exceeds this many + * characters the listing stops and a trailing marker records how many pages + * were omitted. Title-based dedup at persist time (findByCanonicalTitle) keeps + * truncation safe: a page the router can no longer see is converted from + * create to update on save rather than duplicated. Set to 0 to disable the + * char cap. + */ + private int existingPagesIndexMaxChars = 12000; + + /** + * Hard cap on the number of pages listed in the existing-pages index, applied + * together with {@link #existingPagesIndexMaxChars} — whichever limit is hit + * first stops the listing. Manually-edited pages are listed first so + * user-curated entries are never the ones dropped. Set to 0 to disable the + * page-count cap. + */ + private int existingPagesIndexMaxPages = 200; + /** 单个原始材料最多生成的 Wiki 页面数 */ private int maxPagesPerRaw = 15; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiAdminController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiAdminController.java index d9c43d50..f836025e 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiAdminController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiAdminController.java @@ -9,9 +9,11 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import vip.mate.wiki.job.WikiChunkTokenBackfillJob; import vip.mate.wiki.service.WikiOverviewService; +import vip.mate.wiki.service.WikiPageService; import vip.mate.wiki.service.WikiScaffoldService; import java.util.HashMap; @@ -34,6 +36,7 @@ import vip.mate.workspace.core.annotation.RequireWorkspaceRole; public class WikiAdminController { private final WikiScaffoldService scaffoldService; + private final WikiPageService pageService; /** Optional so the controller can boot in environments where the rebuilder isn't wired (e.g. minimal tests). */ @Autowired(required = false) @@ -81,4 +84,20 @@ public class WikiAdminController { body.put("filledThisBatch", Math.max(0, beforePending - afterPending)); return ResponseEntity.ok(body); } + + @Operation(summary = "Merge duplicate pages that share a canonical title", + description = "Heals duplicate rows produced before title-based dedup existed (one concept " + + "stored under several LLM-minted slugs). Defaults to a dry run that only reports " + + "what would change. Set dryRun=false to apply. concatenate=true (default) appends each " + + "loser's body to the winner so no content is lost; concatenate=false keeps only the " + + "winner's body. Protected (system/locked) pages always win and are never deleted.") + @PostMapping("/kb/{kbId}/merge-duplicate-titles") + @RequireWorkspaceRole("admin") + public ResponseEntity> mergeDuplicateTitles( + @PathVariable Long kbId, + @RequestParam(defaultValue = "true") boolean dryRun, + @RequestParam(defaultValue = "true") boolean concatenate) { + Map report = pageService.mergeDuplicateTitles(kbId, dryRun, concatenate); + return ResponseEntity.ok(report); + } } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java index 7f085eac..66156b16 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java @@ -16,9 +16,12 @@ import vip.mate.wiki.repository.WikiRelationMapper; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; @@ -297,6 +300,51 @@ public class WikiPageService { return null; } + /** + * Normalize a title into its canonical identity form, mirroring how a + * wikilink resolver matches note names: lowercase, then drop every + * whitespace / hyphen / underscore (including the full-width space) so + * {@code "二味拔毒散"}, {@code "二味拔毒散 "} and {@code "二味-拔毒散"} all + * collapse to the same key. + *

+ * Title is the stable, human-meaningful identity of a concept. The slug, by + * contrast, is LLM-generated and drifts across runs and romanizations + * ({@code erwei-badu-san} / {@code er-wei-badu-san} / an English translation), + * which is why slug-only matching leaks duplicate rows for one concept. Title + * matching is the primary dedup key; {@link #canonicalSlug(String)} stays as a + * secondary cross-spelling fallback. + */ + public static String canonicalTitle(String title) { + if (title == null) return ""; + String lowered = title.trim().toLowerCase(); + StringBuilder sb = new StringBuilder(lowered.length()); + for (int i = 0; i < lowered.length(); i++) { + char c = lowered.charAt(i); + if (c == '-' || c == '_' || c == '　' || Character.isWhitespace(c)) { + continue; + } + sb.append(c); + } + return sb.toString(); + } + + /** + * Find an existing page in the KB whose title canonically matches the given + * title. Reuses the {@link #listSummaries(Long)} cache (which carries title), + * then loads the full entity for the match. Returns the first canonical-title + * match, or {@code null} when none exists. + */ + public WikiPageEntity findByCanonicalTitle(Long kbId, String title) { + String canonical = canonicalTitle(title); + if (canonical.isEmpty()) return null; + for (WikiPageEntity p : listSummaries(kbId)) { + if (canonicalTitle(p.getTitle()).equals(canonical)) { + return getBySlug(kbId, p.getSlug()); + } + } + return null; + } + public WikiPageEntity getById(Long id) { return pageMapper.selectById(id); } @@ -917,6 +965,192 @@ public class WikiPageService { return affected; } + /** + * Winner selection within a duplicate-title group: keep the page that + * carries the most information. Prefer the longest content, then the + * highest version (most merged), then the smallest id (earliest-created, + * for a stable deterministic result). + */ + private static final Comparator MERGE_WINNER_ORDER = + Comparator.comparingInt((WikiPageEntity p) -> p.getContent() == null ? 0 : p.getContent().length()) + .thenComparingInt(p -> p.getVersion() == null ? 0 : p.getVersion()) + .thenComparing(WikiPageEntity::getId, Comparator.reverseOrder()); + + /** + * One-time maintenance: collapse pages that share a canonical title (see + * {@link #canonicalTitle(String)}) into a single page, healing the duplicate + * rows produced before title-based dedup existed (an LLM-minted slug drifts + * across runs, so one concept landed as many rows under different slugs). + *

+ * For each group of duplicates a winner is chosen ({@link #MERGE_WINNER_ORDER}); + * every loser's inbound {@code [[loserSlug]]} reference is redirected to the + * winner, the losers' source lineage is folded into the winner, their bodies + * are optionally appended (so no content is lost), and the loser rows are + * deleted. A protected page (system / locked) always wins and is never + * deleted; a group with more than one protected page is skipped for manual + * resolution. + * + * @param kbId knowledge base to clean + * @param dryRun when {@code true}, only report what would change — no writes + * @param concatenateContent when {@code true}, append each loser's body to the + * winner under a separator; when {@code false}, keep + * only the winner's body (loser bodies are discarded) + * @return a structured report (counts + per-group winner/loser slugs) + */ + @Transactional + public Map mergeDuplicateTitles(Long kbId, boolean dryRun, boolean concatenateContent) { + List all = listByKbIdWithContent(kbId); + + // Group by canonical title, preserving first-seen order for a stable report. + Map> groups = new LinkedHashMap<>(); + for (WikiPageEntity p : all) { + String ct = canonicalTitle(p.getTitle()); + if (ct.isEmpty()) continue; + groups.computeIfAbsent(ct, k -> new ArrayList<>()).add(p); + } + + List> groupReports = new ArrayList<>(); + int duplicateGroups = 0; + int pagesRemoved = 0; + + for (Map.Entry> entry : groups.entrySet()) { + List grp = entry.getValue(); + if (grp.size() < 2) continue; + + List protectedPages = grp.stream().filter(WikiPageService::isProtected).toList(); + if (protectedPages.size() > 1) { + Map skip = new LinkedHashMap<>(); + skip.put("canonicalTitle", entry.getKey()); + skip.put("title", grp.get(0).getTitle()); + skip.put("skipped", "multiple protected pages; resolve manually"); + skip.put("slugs", grp.stream().map(WikiPageEntity::getSlug).toList()); + groupReports.add(skip); + continue; + } + + WikiPageEntity winner = protectedPages.size() == 1 + ? protectedPages.get(0) + : grp.stream().max(MERGE_WINNER_ORDER).orElseThrow(); + List losers = grp.stream() + .filter(p -> !p.getId().equals(winner.getId())) + .filter(p -> !isProtected(p)) + .toList(); + if (losers.isEmpty()) continue; + + duplicateGroups++; + pagesRemoved += losers.size(); + + Map gr = new LinkedHashMap<>(); + gr.put("canonicalTitle", entry.getKey()); + gr.put("title", winner.getTitle()); + gr.put("winnerSlug", winner.getSlug()); + gr.put("winnerVersion", winner.getVersion()); + gr.put("loserSlugs", losers.stream().map(WikiPageEntity::getSlug).toList()); + groupReports.add(gr); + + if (!dryRun) { + mergeGroupInto(kbId, winner, losers, concatenateContent && !isProtected(winner)); + } + } + + if (!dryRun && duplicateGroups > 0) { + evictSummaryCache(kbId); + if (auditEventService != null) { + try { + String detail = objectMapper.writeValueAsString(Map.of( + "kbId", kbId, + "duplicateGroups", duplicateGroups, + "pagesRemoved", pagesRemoved, + "concatenateContent", concatenateContent)); + auditEventService.record("wiki.page.merge-duplicates", "wiki_kb", + String.valueOf(kbId), "merge duplicate titles", detail); + } catch (Exception e) { + log.debug("[Wiki] Audit emit failed for merge-duplicates kbId={}: {}", kbId, e.toString()); + } + } + } + + Map report = new LinkedHashMap<>(); + report.put("kbId", kbId); + report.put("dryRun", dryRun); + report.put("concatenateContent", concatenateContent); + report.put("totalPages", all.size()); + report.put("duplicateGroups", duplicateGroups); + report.put("pagesRemoved", dryRun ? 0 : pagesRemoved); + report.put("pagesWouldRemove", pagesRemoved); + report.put("groups", groupReports); + return report; + } + + /** + * Fold {@code losers} into {@code winner}: redirect inbound links, merge + * source lineage, optionally append bodies, then delete the loser rows. + */ + private void mergeGroupInto(Long kbId, WikiPageEntity winner, + List losers, boolean concatenate) { + String winnerSlug = winner.getSlug(); + + for (WikiPageEntity loser : losers) { + // Redirect every [[loserSlug]] reference (in any page, including the + // winner) to the winner before the loser row goes away, so no link + // is demoted to plain text. + try { + cascadeRenameReferrers(kbId, loser.getId(), loser.getSlug(), winnerSlug); + } catch (RuntimeException ex) { + log.warn("[Wiki] merge: redirect referrers {}→{} failed (continuing): {}", + loser.getSlug(), winnerSlug, ex.toString()); + } + // Fold the loser's source provenance into the winner. + for (SourceEntry se : parseSourceEntries(loser.getSourceEntries())) { + mergeSourceLineage(winner.getId(), se.rawId(), se.rawTitle()); + } + for (Long rid : parseSourceRawIds(loser.getSourceRawIds())) { + mergeSourceLineage(winner.getId(), rid, ""); + } + } + + if (concatenate) { + // Re-load to pick up the lineage updates just written. + WikiPageEntity fresh = pageMapper.selectById(winner.getId()); + if (fresh != null) { + StringBuilder merged = new StringBuilder(fresh.getContent() != null ? fresh.getContent() : ""); + for (WikiPageEntity loser : losers) { + String body = loser.getContent(); + if (body == null || body.isBlank()) continue; + // Repoint the loser's own self-links so the appended text + // targets the winner rather than the soon-deleted slug. + body = linkService.renameLink(body, loser.getSlug(), winnerSlug); + merged.append("\n\n---\n\n") + .append("> Merged from duplicate page `").append(loser.getSlug()).append("`"); + if (loser.getTitle() != null && !loser.getTitle().isBlank()) { + merged.append(" (").append(loser.getTitle()).append(")"); + } + merged.append("\n\n").append(body); + } + fresh.setContent(merged.toString()); + fresh.setVersion((fresh.getVersion() == null ? 1 : fresh.getVersion()) + 1); + fresh.setUpdateTime(LocalDateTime.now()); + applyLinkAnalysis(fresh); + pageMapper.updateById(fresh); + } + } + + for (WikiPageEntity loser : losers) { + if (relationMapper != null) { + try { + relationMapper.delete(new LambdaQueryWrapper() + .eq(WikiRelationEntity::getKbId, kbId) + .and(w -> w.eq(WikiRelationEntity::getPageAId, loser.getId()) + .or().eq(WikiRelationEntity::getPageBId, loser.getId()))); + } catch (RuntimeException ignore) { + // relation table is a reserved cache; cleanup is best-effort + } + } + pageMapper.deleteById(loser.getId()); + } + evictSummaryCache(kbId); + } + /** * RFC-051 PR-7: flip the {@code archived} flag. *

diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index c6d783e9..70656254 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -168,6 +168,17 @@ public class WikiProcessingService { * slug 注册为 winner，后到的 chunk 看到 winner 后会把内容写入 winner 对应的 page。 */ final ConcurrentHashMap slugClaims = new ConcurrentHashMap<>(); + /** + * Per-run title claim table: canonical title → the actual slug of the first + * page that claimed that concept this run. + *

+ * Title is the stable concept identity (the slug is LLM-generated and drifts), + * so this closes the parallel-create race that {@link #slugClaims} cannot: + * two phase-B pages with the same title but different slugs would both miss + * the DB lookup and insert two rows. The first to {@link ConcurrentHashMap#computeIfAbsent} + * wins; later creates redirect their content into the winner page. + */ + final ConcurrentHashMap titleClaims = new ConcurrentHashMap<>(); /** * Per-run merge dedup set: slugs that have already been successfully merged during * this raw material processing run. Prevents the same page from being merged N times @@ -1540,6 +1551,22 @@ public class WikiProcessingService { // point at a tombstoned row. if (isAborted(rawId, "savePageContent slug=" + slug)) return false; + // Fallback 0a: canonical-title match — title is the stable concept identity. + // The LLM-generated slug drifts across runs and romanizations, so the same + // concept otherwise lands as many rows under different slugs (the duplicate + // explosion this guards against). If a page with the same canonical title + // already exists under a different slug, merge into it instead of creating. + WikiPageEntity existingByTitle = pageService.findByCanonicalTitle(kbId, title); + if (existingByTitle != null && !existingByTitle.getSlug().equals(slug)) { + String actualSlug = existingByTitle.getSlug(); + pageService.updatePageByAi(kbId, actualSlug, content, pageSummary, rawId); + pageService.mergeSourceLineage(existingByTitle.getId(), rawId, raw.getTitle()); + afterPagePersisted(existingByTitle.getId(), kbId, pageType, metadataNode, dependsOnNode, true); + log.info("[Wiki] Phase B create slug='{}' title='{}' canonical-title-matches existing '{}', updated", + slug, title, actualSlug); + return false; + } + // Fallback 0: cross-spelling canonical match (DB has same concept under different slug) WikiPageEntity existingByCanonical = pageService.findByCanonicalSlug(kbId, slug); if (existingByCanonical != null && !existingByCanonical.getSlug().equals(slug)) { @@ -1552,8 +1579,34 @@ public class WikiProcessingService { return false; } - // Fallback 0.5: in-flight slug-claim arbitration across parallel chunks + // Fallback 0.25: in-flight title-claim arbitration across parallel pages. + // Closes the race that findByCanonicalTitle cannot: two parallel phase-B + // creates with the same title but different slugs can both miss the DB + // lookup (the row isn't committed yet) and insert two rows — title has no + // DB unique constraint, so DuplicateKey won't catch it. The first to claim + // the canonical title wins; losers redirect their content into the winner. ProgressCounter pcLocal = progressCounters.get(rawId); + String canonicalTitle = WikiPageService.canonicalTitle(title); + if (pcLocal != null && !canonicalTitle.isEmpty()) { + final String claimingSlug = slug; + String winnerSlug = pcLocal.titleClaims.computeIfAbsent(canonicalTitle, k -> claimingSlug); + if (!winnerSlug.equals(slug)) { + WikiPageEntity winner = pageService.getBySlug(kbId, winnerSlug); + if (winner != null) { + pageService.updatePageByAi(kbId, winnerSlug, content, pageSummary, rawId); + pageService.mergeSourceLineage(winner.getId(), rawId, raw.getTitle()); + afterPagePersisted(winner.getId(), kbId, pageType, metadataNode, dependsOnNode, true); + log.info("[Wiki] Phase B create slug='{}' title='{}' lost title-claim race to '{}', updated", + slug, title, winnerSlug); + return false; + } + log.info("[Wiki] Phase B create slug='{}' redirects to in-flight title winner '{}'", + slug, winnerSlug); + slug = winnerSlug; + } + } + + // Fallback 0.5: in-flight slug-claim arbitration across parallel chunks String canonical = WikiPageService.canonicalSlug(slug); if (pcLocal != null && !canonical.isEmpty()) { final String routedSlug = slug; @@ -2048,20 +2101,59 @@ public class WikiProcessingService { return "（暂无已有页面）"; } + // The index is injected verbatim into the route / batch-create prompts, so + // it must stay bounded — otherwise it grows linearly with the KB and + // overflows the model context window. Cap by both page count and chars; + // truncation is safe because savePageContent dedups by canonical title at + // persist time, so an omitted page is merged-on-save rather than duplicated. + int maxChars = Math.max(0, properties.getExistingPagesIndexMaxChars()); + int maxPages = Math.max(0, properties.getExistingPagesIndexMaxPages()); + + // List manually-edited pages first so user-curated entries are never the + // ones dropped when a cap is hit. Title order within each group is + // preserved (listSummaries already sorts by title). + List ordered = new ArrayList<>(summaries.size()); + for (WikiPageEntity p : summaries) { + if ("manual".equals(p.getLastUpdatedBy())) ordered.add(p); + } + for (WikiPageEntity p : summaries) { + if (!"manual".equals(p.getLastUpdatedBy())) ordered.add(p); + } + StringBuilder sb = new StringBuilder(); - for (WikiPageEntity page : summaries) { - sb.append("- [[").append(page.getSlug()).append("]]"); + int listed = 0; + for (WikiPageEntity page : ordered) { + StringBuilder row = new StringBuilder(); + row.append("- [[").append(page.getSlug()).append("]]"); if (page.getTitle() != null && !page.getTitle().isBlank()) { - sb.append(" — ").append(page.getTitle()); + row.append(" — ").append(page.getTitle()); } if ("manual".equals(page.getLastUpdatedBy())) { - sb.append(" (手动编辑)"); + row.append(" (手动编辑)"); } String summary = page.getSummary(); if (summary != null && !summary.isBlank()) { - sb.append(" — ").append(summary); + row.append(" — ").append(summary); } - sb.append("\n"); + row.append("\n"); + + // Stop before exceeding either cap, but always emit at least one row. + boolean overPageCap = maxPages > 0 && listed >= maxPages; + boolean overCharCap = maxChars > 0 && listed > 0 && sb.length() + row.length() > maxChars; + if (overPageCap || overCharCap) { + break; + } + sb.append(row); + listed++; + } + + int omitted = ordered.size() - listed; + if (omitted > 0) { + sb.append("- …（已省略 ").append(omitted) + .append(" 个页面：已有页面过多，索引已截断。若材料涉及未列出的概念，按新建处理即可，") + .append("系统会在落库时按标题自动归并到既有页面）\n"); + log.info("[Wiki] existing-pages index truncated for kbId={}: listed={} omitted={} chars={}", + kbId, listed, omitted, sb.length()); } return sb.toString().trim(); } diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiMergeDuplicateTitlesE2ETest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiMergeDuplicateTitlesE2ETest.java new file mode 100644 index 00000000..8e62fe8c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiMergeDuplicateTitlesE2ETest.java @@ -0,0 +1,150 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.DirtiesContext; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiKnowledgeBaseMapper; +import vip.mate.wiki.repository.WikiPageMapper; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * E2E coverage for {@link WikiPageService#mergeDuplicateTitles} — the one-time + * maintenance op that collapses pages sharing a canonical title (the duplicate + * rows produced before title-based dedup existed, when one concept landed under + * several LLM-minted slugs). + * + *

Boots the full Spring + H2 + Flyway context so MyBatis-Plus's lambda cache + * and the real cascade/link machinery are exercised end to end. + */ +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.flyway.enabled=true", + "spring.flyway.locations=classpath:db/migration/h2", + "mateclaw.feature-flag.refresh-ms=999999" + } +) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class WikiMergeDuplicateTitlesE2ETest { + + @Autowired private WikiPageService pageService; + @Autowired private WikiKnowledgeBaseService kbService; + @Autowired private WikiPageMapper pageMapper; + @Autowired private WikiKnowledgeBaseMapper kbMapper; + + private Long kbId; + + @AfterEach + void cleanup() { + if (kbId != null) { + pageMapper.delete(new LambdaQueryWrapper().eq(WikiPageEntity::getKbId, kbId)); + kbMapper.deleteById(kbId); + pageService.evictSummaryCache(kbId); + kbId = null; + } + } + + private void seedKb() { + WikiKnowledgeBaseEntity kb = kbService.create("merge-dup-" + System.nanoTime(), "merge test", null); + kbId = kb.getId(); + pageMapper.delete(new LambdaQueryWrapper().eq(WikiPageEntity::getKbId, kbId)); + pageService.evictSummaryCache(kbId); + } + + /** + * Three pages share the canonical title "医宗金鉴" under different slugs (the + * exact failure mode from the bug report), plus a referrer linking to a loser. + */ + private void seedDuplicates() { + // Winner: longest content. + pageService.createPage(kbId, "yizong-jinjian", "医宗金鉴", + "## 医宗金鉴\n\nThis is the most complete body with the richest detail aaa bbb ccc ddd.", + "complete summary", "[]"); + // Losers: same canonical title, shorter content, different slugs. + pageService.createPage(kbId, "yizong-jinjian-quanshu", "医宗金鉴", + "Body B shorter.", "b summary", "[]"); + // Trailing-space title still canonicalizes equal. + pageService.createPage(kbId, "yzjj", "医宗金鉴 ", + "Body C tiny.", "c summary", "[]"); + // A referrer pointing at one of the losers. + pageService.createPage(kbId, "ref", "Ref", + "See [[yizong-jinjian-quanshu]] for the canonical text.", + "referrer summary", "[]"); + } + + @Test + @DisplayName("dry run reports duplicates without mutating anything") + void dryRunReportsButDoesNotMutate() { + seedKb(); + seedDuplicates(); + + Map report = pageService.mergeDuplicateTitles(kbId, true, true); + + assertThat(report.get("dryRun")).isEqualTo(true); + assertThat(report.get("duplicateGroups")).isEqualTo(1); + assertThat(report.get("pagesWouldRemove")).isEqualTo(2); + assertThat(report.get("pagesRemoved")).isEqualTo(0); + + // Nothing deleted: all four pages still present. + assertThat(pageService.listByKbIdWithContent(kbId)).hasSize(4); + assertThat(pageService.getBySlug(kbId, "yizong-jinjian-quanshu")).isNotNull(); + assertThat(pageService.getBySlug(kbId, "yzjj")).isNotNull(); + } + + @Test + @DisplayName("apply with concatenate merges losers into winner, redirects refs, deletes losers") + void applyConcatenateCollapsesGroup() { + seedKb(); + seedDuplicates(); + + Map report = pageService.mergeDuplicateTitles(kbId, false, true); + assertThat(report.get("duplicateGroups")).isEqualTo(1); + assertThat(report.get("pagesRemoved")).isEqualTo(2); + + // Only the winner + the referrer survive. + assertThat(pageService.listByKbIdWithContent(kbId)).hasSize(2); + assertThat(pageService.getBySlug(kbId, "yizong-jinjian-quanshu")).isNull(); + assertThat(pageService.getBySlug(kbId, "yzjj")).isNull(); + + // Winner keeps its body and gains the losers' bodies (no content lost). + WikiPageEntity winner = pageService.getBySlug(kbId, "yizong-jinjian"); + assertThat(winner).isNotNull(); + assertThat(winner.getContent()) + .contains("most complete body") + .contains("Body B shorter.") + .contains("Body C tiny."); + assertThat(winner.getVersion()).isGreaterThan(1); + + // Referrer's [[loserSlug]] is redirected to the winner, not demoted. + WikiPageEntity ref = pageService.getBySlug(kbId, "ref"); + assertThat(ref.getContent()) + .doesNotContain("[[yizong-jinjian-quanshu]]") + .contains("[[yizong-jinjian]]"); + } + + @Test + @DisplayName("apply without concatenate keeps only the winner's body") + void applyWithoutConcatenateDiscardsLoserBodies() { + seedKb(); + seedDuplicates(); + + pageService.mergeDuplicateTitles(kbId, false, false); + + assertThat(pageService.listByKbIdWithContent(kbId)).hasSize(2); + WikiPageEntity winner = pageService.getBySlug(kbId, "yizong-jinjian"); + assertThat(winner).isNotNull(); + assertThat(winner.getContent()) + .contains("most complete body") + .doesNotContain("Body B shorter.") + .doesNotContain("Body C tiny."); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java index 4288363f..88e70e77 100644 --- a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java @@ -1,12 +1,19 @@ package vip.mate.wiki.service; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; import org.junit.jupiter.api.Test; import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.repository.WikiPageMapper; import java.time.LocalDateTime; +import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -15,6 +22,15 @@ import static org.mockito.Mockito.when; class WikiPageServiceTest { + static { + // LambdaQueryWrapper resolves column metadata from MyBatis-Plus's TableInfo + // cache, which Spring normally populates at startup. In a plain unit test we + // seed it once so getBySlug / listSummaries can build their lambda queries. + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new MybatisConfiguration(), ""), + WikiPageEntity.class); + } + @Test void manualUpdateRefreshesUpdateTimeBeforePersisting() { WikiPageMapper mapper = mock(WikiPageMapper.class); @@ -39,4 +55,61 @@ class WikiPageServiceTest { assertTrue(page.getUpdateTime().isAfter(oldUpdateTime)); verify(mapper).updateById(page); } + + @Test + void canonicalTitleFoldsCaseAndSeparators() { + // Case + ASCII separators fold away so spelling variants collapse to one key. + assertEquals("erweibadusan", WikiPageService.canonicalTitle("Erwei-Badu_San")); + assertEquals("erweibadusan", WikiPageService.canonicalTitle("er wei badu san")); + // Chinese title with surrounding/full-width whitespace and an inserted hyphen. + assertEquals("二味拔毒散", WikiPageService.canonicalTitle(" 二味-拔毒散　")); + assertEquals(WikiPageService.canonicalTitle("二味拔毒散"), + WikiPageService.canonicalTitle("二味拔毒散 ")); + // Null / blank degrade to empty so callers can short-circuit. + assertEquals("", WikiPageService.canonicalTitle(null)); + assertEquals("", WikiPageService.canonicalTitle(" ")); + } + + @Test + void findByCanonicalTitleMatchesAcrossDifferentSlugs() { + // Same concept already stored under an LLM-chosen slug; a later run arrives + // with the same title but would have minted a different slug. Title match + // must find the existing row regardless of the slug spelling. + WikiPageMapper mapper = mock(WikiPageMapper.class); + WikiPageEntity summary = new WikiPageEntity(); + summary.setKbId(7L); + summary.setSlug("erwei-badu-san"); + summary.setTitle("二味拔毒散"); + WikiPageEntity full = new WikiPageEntity(); + full.setId(42L); + full.setKbId(7L); + full.setSlug("erwei-badu-san"); + full.setTitle("二味拔毒散"); + // listSummaries() -> selectList ; getBySlug() -> selectOne + when(mapper.selectList(any())).thenReturn(List.of(summary)); + when(mapper.selectOne(any())).thenReturn(full); + + ObjectMapper om = new ObjectMapper(); + WikiPageService service = new WikiPageService(mapper, om, new WikiLinkService(om)); + + WikiPageEntity hit = service.findByCanonicalTitle(7L, "二味拔毒散"); + assertNotNull(hit); + assertEquals(42L, hit.getId()); + assertEquals("erwei-badu-san", hit.getSlug()); + } + + @Test + void findByCanonicalTitleReturnsNullWhenNoConceptMatches() { + WikiPageMapper mapper = mock(WikiPageMapper.class); + WikiPageEntity summary = new WikiPageEntity(); + summary.setKbId(7L); + summary.setSlug("shennong-bencao"); + summary.setTitle("神农本草经"); + when(mapper.selectList(any())).thenReturn(List.of(summary)); + + ObjectMapper om = new ObjectMapper(); + WikiPageService service = new WikiPageService(mapper, om, new WikiLinkService(om)); + + assertNull(service.findByCanonicalTitle(7L, "二味拔毒散")); + } }