mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 11:58:34 +08:00
fix(wiki): dedup pages by title and bound route prompt growth (#321)
This commit is contained in:
matevip
parent
c1691e466c
commit
48024e4a83
@ -56,6 +56,29 @@ public class WikiProperties {
|
|||||||
/** 注入 agent prompt 的最大字符数 */
|
/** 注入 agent prompt 的最大字符数 */
|
||||||
private int maxContextChars = 10000;
|
private int maxContextChars = 10000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hard cap on the existing-pages index injected into the route / batch-create
|
||||||
|
* prompts. The index lists every non-archived page in the KB so the router can
|
||||||
|
* decide create-vs-update and emit cross-links. Without a cap it grows linearly
|
||||||
|
* with the KB and eventually overflows the model context window
|
||||||
|
* ("Prompt exceeds max length"). When the rendered index exceeds this many
|
||||||
|
* characters the listing stops and a trailing marker records how many pages
|
||||||
|
* were omitted. Title-based dedup at persist time (findByCanonicalTitle) keeps
|
||||||
|
* truncation safe: a page the router can no longer see is converted from
|
||||||
|
* create to update on save rather than duplicated. Set to 0 to disable the
|
||||||
|
* char cap.
|
||||||
|
*/
|
||||||
|
private int existingPagesIndexMaxChars = 12000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hard cap on the number of pages listed in the existing-pages index, applied
|
||||||
|
* together with {@link #existingPagesIndexMaxChars} — whichever limit is hit
|
||||||
|
* first stops the listing. Manually-edited pages are listed first so
|
||||||
|
* user-curated entries are never the ones dropped. Set to 0 to disable the
|
||||||
|
* page-count cap.
|
||||||
|
*/
|
||||||
|
private int existingPagesIndexMaxPages = 200;
|
||||||
|
|
||||||
/** 单个原始材料最多生成的 Wiki 页面数 */
|
/** 单个原始材料最多生成的 Wiki 页面数 */
|
||||||
private int maxPagesPerRaw = 15;
|
private int maxPagesPerRaw = 15;
|
||||||
|
|
||||||
|
|||||||
@ -9,9 +9,11 @@ import org.springframework.http.ResponseEntity;
|
|||||||
import org.springframework.web.bind.annotation.PathVariable;
|
import org.springframework.web.bind.annotation.PathVariable;
|
||||||
import org.springframework.web.bind.annotation.PostMapping;
|
import org.springframework.web.bind.annotation.PostMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
import org.springframework.web.bind.annotation.RestController;
|
import org.springframework.web.bind.annotation.RestController;
|
||||||
import vip.mate.wiki.job.WikiChunkTokenBackfillJob;
|
import vip.mate.wiki.job.WikiChunkTokenBackfillJob;
|
||||||
import vip.mate.wiki.service.WikiOverviewService;
|
import vip.mate.wiki.service.WikiOverviewService;
|
||||||
|
import vip.mate.wiki.service.WikiPageService;
|
||||||
import vip.mate.wiki.service.WikiScaffoldService;
|
import vip.mate.wiki.service.WikiScaffoldService;
|
||||||
|
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@ -34,6 +36,7 @@ import vip.mate.workspace.core.annotation.RequireWorkspaceRole;
|
|||||||
public class WikiAdminController {
|
public class WikiAdminController {
|
||||||
|
|
||||||
private final WikiScaffoldService scaffoldService;
|
private final WikiScaffoldService scaffoldService;
|
||||||
|
private final WikiPageService pageService;
|
||||||
|
|
||||||
/** Optional so the controller can boot in environments where the rebuilder isn't wired (e.g. minimal tests). */
|
/** Optional so the controller can boot in environments where the rebuilder isn't wired (e.g. minimal tests). */
|
||||||
@Autowired(required = false)
|
@Autowired(required = false)
|
||||||
@ -81,4 +84,20 @@ public class WikiAdminController {
|
|||||||
body.put("filledThisBatch", Math.max(0, beforePending - afterPending));
|
body.put("filledThisBatch", Math.max(0, beforePending - afterPending));
|
||||||
return ResponseEntity.ok(body);
|
return ResponseEntity.ok(body);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Operation(summary = "Merge duplicate pages that share a canonical title",
|
||||||
|
description = "Heals duplicate rows produced before title-based dedup existed (one concept "
|
||||||
|
+ "stored under several LLM-minted slugs). Defaults to a dry run that only reports "
|
||||||
|
+ "what would change. Set dryRun=false to apply. concatenate=true (default) appends each "
|
||||||
|
+ "loser's body to the winner so no content is lost; concatenate=false keeps only the "
|
||||||
|
+ "winner's body. Protected (system/locked) pages always win and are never deleted.")
|
||||||
|
@PostMapping("/kb/{kbId}/merge-duplicate-titles")
|
||||||
|
@RequireWorkspaceRole("admin")
|
||||||
|
public ResponseEntity<Map<String, Object>> mergeDuplicateTitles(
|
||||||
|
@PathVariable Long kbId,
|
||||||
|
@RequestParam(defaultValue = "true") boolean dryRun,
|
||||||
|
@RequestParam(defaultValue = "true") boolean concatenate) {
|
||||||
|
Map<String, Object> report = pageService.mergeDuplicateTitles(kbId, dryRun, concatenate);
|
||||||
|
return ResponseEntity.ok(report);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,9 +16,12 @@ import vip.mate.wiki.repository.WikiRelationMapper;
|
|||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
import java.util.Locale;
|
||||||
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
@ -297,6 +300,51 @@ public class WikiPageService {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize a title into its canonical identity form, mirroring how a
|
||||||
|
* wikilink resolver matches note names: lowercase, then drop every
|
||||||
|
* whitespace / hyphen / underscore (including the full-width space) so
|
||||||
|
* {@code "二味拔毒散"}, {@code "二味拔毒散 "} and {@code "二味-拔毒散"} all
|
||||||
|
* collapse to the same key.
|
||||||
|
* <p>
|
||||||
|
* Title is the stable, human-meaningful identity of a concept. The slug, by
|
||||||
|
* contrast, is LLM-generated and drifts across runs and romanizations
|
||||||
|
* ({@code erwei-badu-san} / {@code er-wei-badu-san} / an English translation),
|
||||||
|
* which is why slug-only matching leaks duplicate rows for one concept. Title
|
||||||
|
* matching is the primary dedup key; {@link #canonicalSlug(String)} stays as a
|
||||||
|
* secondary cross-spelling fallback.
|
||||||
|
*/
|
||||||
|
public static String canonicalTitle(String title) {
|
||||||
|
if (title == null) return "";
|
||||||
|
String lowered = title.trim().toLowerCase();
|
||||||
|
StringBuilder sb = new StringBuilder(lowered.length());
|
||||||
|
for (int i = 0; i < lowered.length(); i++) {
|
||||||
|
char c = lowered.charAt(i);
|
||||||
|
if (c == '-' || c == '_' || c == ' ' || Character.isWhitespace(c)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
sb.append(c);
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find an existing page in the KB whose title canonically matches the given
|
||||||
|
* title. Reuses the {@link #listSummaries(Long)} cache (which carries title),
|
||||||
|
* then loads the full entity for the match. Returns the first canonical-title
|
||||||
|
* match, or {@code null} when none exists.
|
||||||
|
*/
|
||||||
|
public WikiPageEntity findByCanonicalTitle(Long kbId, String title) {
|
||||||
|
String canonical = canonicalTitle(title);
|
||||||
|
if (canonical.isEmpty()) return null;
|
||||||
|
for (WikiPageEntity p : listSummaries(kbId)) {
|
||||||
|
if (canonicalTitle(p.getTitle()).equals(canonical)) {
|
||||||
|
return getBySlug(kbId, p.getSlug());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
public WikiPageEntity getById(Long id) {
|
public WikiPageEntity getById(Long id) {
|
||||||
return pageMapper.selectById(id);
|
return pageMapper.selectById(id);
|
||||||
}
|
}
|
||||||
@ -917,6 +965,192 @@ public class WikiPageService {
|
|||||||
return affected;
|
return affected;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Winner selection within a duplicate-title group: keep the page that
|
||||||
|
* carries the most information. Prefer the longest content, then the
|
||||||
|
* highest version (most merged), then the smallest id (earliest-created,
|
||||||
|
* for a stable deterministic result).
|
||||||
|
*/
|
||||||
|
private static final Comparator<WikiPageEntity> MERGE_WINNER_ORDER =
|
||||||
|
Comparator.comparingInt((WikiPageEntity p) -> p.getContent() == null ? 0 : p.getContent().length())
|
||||||
|
.thenComparingInt(p -> p.getVersion() == null ? 0 : p.getVersion())
|
||||||
|
.thenComparing(WikiPageEntity::getId, Comparator.reverseOrder());
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One-time maintenance: collapse pages that share a canonical title (see
|
||||||
|
* {@link #canonicalTitle(String)}) into a single page, healing the duplicate
|
||||||
|
* rows produced before title-based dedup existed (an LLM-minted slug drifts
|
||||||
|
* across runs, so one concept landed as many rows under different slugs).
|
||||||
|
* <p>
|
||||||
|
* For each group of duplicates a winner is chosen ({@link #MERGE_WINNER_ORDER});
|
||||||
|
* every loser's inbound {@code [[loserSlug]]} reference is redirected to the
|
||||||
|
* winner, the losers' source lineage is folded into the winner, their bodies
|
||||||
|
* are optionally appended (so no content is lost), and the loser rows are
|
||||||
|
* deleted. A protected page (system / locked) always wins and is never
|
||||||
|
* deleted; a group with more than one protected page is skipped for manual
|
||||||
|
* resolution.
|
||||||
|
*
|
||||||
|
* @param kbId knowledge base to clean
|
||||||
|
* @param dryRun when {@code true}, only report what would change — no writes
|
||||||
|
* @param concatenateContent when {@code true}, append each loser's body to the
|
||||||
|
* winner under a separator; when {@code false}, keep
|
||||||
|
* only the winner's body (loser bodies are discarded)
|
||||||
|
* @return a structured report (counts + per-group winner/loser slugs)
|
||||||
|
*/
|
||||||
|
@Transactional
|
||||||
|
public Map<String, Object> mergeDuplicateTitles(Long kbId, boolean dryRun, boolean concatenateContent) {
|
||||||
|
List<WikiPageEntity> all = listByKbIdWithContent(kbId);
|
||||||
|
|
||||||
|
// Group by canonical title, preserving first-seen order for a stable report.
|
||||||
|
Map<String, List<WikiPageEntity>> groups = new LinkedHashMap<>();
|
||||||
|
for (WikiPageEntity p : all) {
|
||||||
|
String ct = canonicalTitle(p.getTitle());
|
||||||
|
if (ct.isEmpty()) continue;
|
||||||
|
groups.computeIfAbsent(ct, k -> new ArrayList<>()).add(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Map<String, Object>> groupReports = new ArrayList<>();
|
||||||
|
int duplicateGroups = 0;
|
||||||
|
int pagesRemoved = 0;
|
||||||
|
|
||||||
|
for (Map.Entry<String, List<WikiPageEntity>> entry : groups.entrySet()) {
|
||||||
|
List<WikiPageEntity> grp = entry.getValue();
|
||||||
|
if (grp.size() < 2) continue;
|
||||||
|
|
||||||
|
List<WikiPageEntity> protectedPages = grp.stream().filter(WikiPageService::isProtected).toList();
|
||||||
|
if (protectedPages.size() > 1) {
|
||||||
|
Map<String, Object> skip = new LinkedHashMap<>();
|
||||||
|
skip.put("canonicalTitle", entry.getKey());
|
||||||
|
skip.put("title", grp.get(0).getTitle());
|
||||||
|
skip.put("skipped", "multiple protected pages; resolve manually");
|
||||||
|
skip.put("slugs", grp.stream().map(WikiPageEntity::getSlug).toList());
|
||||||
|
groupReports.add(skip);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
WikiPageEntity winner = protectedPages.size() == 1
|
||||||
|
? protectedPages.get(0)
|
||||||
|
: grp.stream().max(MERGE_WINNER_ORDER).orElseThrow();
|
||||||
|
List<WikiPageEntity> losers = grp.stream()
|
||||||
|
.filter(p -> !p.getId().equals(winner.getId()))
|
||||||
|
.filter(p -> !isProtected(p))
|
||||||
|
.toList();
|
||||||
|
if (losers.isEmpty()) continue;
|
||||||
|
|
||||||
|
duplicateGroups++;
|
||||||
|
pagesRemoved += losers.size();
|
||||||
|
|
||||||
|
Map<String, Object> gr = new LinkedHashMap<>();
|
||||||
|
gr.put("canonicalTitle", entry.getKey());
|
||||||
|
gr.put("title", winner.getTitle());
|
||||||
|
gr.put("winnerSlug", winner.getSlug());
|
||||||
|
gr.put("winnerVersion", winner.getVersion());
|
||||||
|
gr.put("loserSlugs", losers.stream().map(WikiPageEntity::getSlug).toList());
|
||||||
|
groupReports.add(gr);
|
||||||
|
|
||||||
|
if (!dryRun) {
|
||||||
|
mergeGroupInto(kbId, winner, losers, concatenateContent && !isProtected(winner));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!dryRun && duplicateGroups > 0) {
|
||||||
|
evictSummaryCache(kbId);
|
||||||
|
if (auditEventService != null) {
|
||||||
|
try {
|
||||||
|
String detail = objectMapper.writeValueAsString(Map.of(
|
||||||
|
"kbId", kbId,
|
||||||
|
"duplicateGroups", duplicateGroups,
|
||||||
|
"pagesRemoved", pagesRemoved,
|
||||||
|
"concatenateContent", concatenateContent));
|
||||||
|
auditEventService.record("wiki.page.merge-duplicates", "wiki_kb",
|
||||||
|
String.valueOf(kbId), "merge duplicate titles", detail);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("[Wiki] Audit emit failed for merge-duplicates kbId={}: {}", kbId, e.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, Object> report = new LinkedHashMap<>();
|
||||||
|
report.put("kbId", kbId);
|
||||||
|
report.put("dryRun", dryRun);
|
||||||
|
report.put("concatenateContent", concatenateContent);
|
||||||
|
report.put("totalPages", all.size());
|
||||||
|
report.put("duplicateGroups", duplicateGroups);
|
||||||
|
report.put("pagesRemoved", dryRun ? 0 : pagesRemoved);
|
||||||
|
report.put("pagesWouldRemove", pagesRemoved);
|
||||||
|
report.put("groups", groupReports);
|
||||||
|
return report;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fold {@code losers} into {@code winner}: redirect inbound links, merge
|
||||||
|
* source lineage, optionally append bodies, then delete the loser rows.
|
||||||
|
*/
|
||||||
|
private void mergeGroupInto(Long kbId, WikiPageEntity winner,
|
||||||
|
List<WikiPageEntity> losers, boolean concatenate) {
|
||||||
|
String winnerSlug = winner.getSlug();
|
||||||
|
|
||||||
|
for (WikiPageEntity loser : losers) {
|
||||||
|
// Redirect every [[loserSlug]] reference (in any page, including the
|
||||||
|
// winner) to the winner before the loser row goes away, so no link
|
||||||
|
// is demoted to plain text.
|
||||||
|
try {
|
||||||
|
cascadeRenameReferrers(kbId, loser.getId(), loser.getSlug(), winnerSlug);
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
log.warn("[Wiki] merge: redirect referrers {}→{} failed (continuing): {}",
|
||||||
|
loser.getSlug(), winnerSlug, ex.toString());
|
||||||
|
}
|
||||||
|
// Fold the loser's source provenance into the winner.
|
||||||
|
for (SourceEntry se : parseSourceEntries(loser.getSourceEntries())) {
|
||||||
|
mergeSourceLineage(winner.getId(), se.rawId(), se.rawTitle());
|
||||||
|
}
|
||||||
|
for (Long rid : parseSourceRawIds(loser.getSourceRawIds())) {
|
||||||
|
mergeSourceLineage(winner.getId(), rid, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (concatenate) {
|
||||||
|
// Re-load to pick up the lineage updates just written.
|
||||||
|
WikiPageEntity fresh = pageMapper.selectById(winner.getId());
|
||||||
|
if (fresh != null) {
|
||||||
|
StringBuilder merged = new StringBuilder(fresh.getContent() != null ? fresh.getContent() : "");
|
||||||
|
for (WikiPageEntity loser : losers) {
|
||||||
|
String body = loser.getContent();
|
||||||
|
if (body == null || body.isBlank()) continue;
|
||||||
|
// Repoint the loser's own self-links so the appended text
|
||||||
|
// targets the winner rather than the soon-deleted slug.
|
||||||
|
body = linkService.renameLink(body, loser.getSlug(), winnerSlug);
|
||||||
|
merged.append("\n\n---\n\n")
|
||||||
|
.append("> Merged from duplicate page `").append(loser.getSlug()).append("`");
|
||||||
|
if (loser.getTitle() != null && !loser.getTitle().isBlank()) {
|
||||||
|
merged.append(" (").append(loser.getTitle()).append(")");
|
||||||
|
}
|
||||||
|
merged.append("\n\n").append(body);
|
||||||
|
}
|
||||||
|
fresh.setContent(merged.toString());
|
||||||
|
fresh.setVersion((fresh.getVersion() == null ? 1 : fresh.getVersion()) + 1);
|
||||||
|
fresh.setUpdateTime(LocalDateTime.now());
|
||||||
|
applyLinkAnalysis(fresh);
|
||||||
|
pageMapper.updateById(fresh);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (WikiPageEntity loser : losers) {
|
||||||
|
if (relationMapper != null) {
|
||||||
|
try {
|
||||||
|
relationMapper.delete(new LambdaQueryWrapper<WikiRelationEntity>()
|
||||||
|
.eq(WikiRelationEntity::getKbId, kbId)
|
||||||
|
.and(w -> w.eq(WikiRelationEntity::getPageAId, loser.getId())
|
||||||
|
.or().eq(WikiRelationEntity::getPageBId, loser.getId())));
|
||||||
|
} catch (RuntimeException ignore) {
|
||||||
|
// relation table is a reserved cache; cleanup is best-effort
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pageMapper.deleteById(loser.getId());
|
||||||
|
}
|
||||||
|
evictSummaryCache(kbId);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RFC-051 PR-7: flip the {@code archived} flag.
|
* RFC-051 PR-7: flip the {@code archived} flag.
|
||||||
* <p>
|
* <p>
|
||||||
|
|||||||
@ -168,6 +168,17 @@ public class WikiProcessingService {
|
|||||||
* slug 注册为 winner，后到的 chunk 看到 winner 后会把内容写入 winner 对应的 page。
|
* slug 注册为 winner，后到的 chunk 看到 winner 后会把内容写入 winner 对应的 page。
|
||||||
*/
|
*/
|
||||||
final ConcurrentHashMap<String, String> slugClaims = new ConcurrentHashMap<>();
|
final ConcurrentHashMap<String, String> slugClaims = new ConcurrentHashMap<>();
|
||||||
|
/**
|
||||||
|
* Per-run title claim table: canonical title → the actual slug of the first
|
||||||
|
* page that claimed that concept this run.
|
||||||
|
* <p>
|
||||||
|
* Title is the stable concept identity (the slug is LLM-generated and drifts),
|
||||||
|
* so this closes the parallel-create race that {@link #slugClaims} cannot:
|
||||||
|
* two phase-B pages with the same title but different slugs would both miss
|
||||||
|
* the DB lookup and insert two rows. The first to {@link ConcurrentHashMap#computeIfAbsent}
|
||||||
|
* wins; later creates redirect their content into the winner page.
|
||||||
|
*/
|
||||||
|
final ConcurrentHashMap<String, String> titleClaims = new ConcurrentHashMap<>();
|
||||||
/**
|
/**
|
||||||
* Per-run merge dedup set: slugs that have already been successfully merged during
|
* Per-run merge dedup set: slugs that have already been successfully merged during
|
||||||
* this raw material processing run. Prevents the same page from being merged N times
|
* this raw material processing run. Prevents the same page from being merged N times
|
||||||
@ -1540,6 +1551,22 @@ public class WikiProcessingService {
|
|||||||
// point at a tombstoned row.
|
// point at a tombstoned row.
|
||||||
if (isAborted(rawId, "savePageContent slug=" + slug)) return false;
|
if (isAborted(rawId, "savePageContent slug=" + slug)) return false;
|
||||||
|
|
||||||
|
// Fallback 0a: canonical-title match — title is the stable concept identity.
|
||||||
|
// The LLM-generated slug drifts across runs and romanizations, so the same
|
||||||
|
// concept otherwise lands as many rows under different slugs (the duplicate
|
||||||
|
// explosion this guards against). If a page with the same canonical title
|
||||||
|
// already exists under a different slug, merge into it instead of creating.
|
||||||
|
WikiPageEntity existingByTitle = pageService.findByCanonicalTitle(kbId, title);
|
||||||
|
if (existingByTitle != null && !existingByTitle.getSlug().equals(slug)) {
|
||||||
|
String actualSlug = existingByTitle.getSlug();
|
||||||
|
pageService.updatePageByAi(kbId, actualSlug, content, pageSummary, rawId);
|
||||||
|
pageService.mergeSourceLineage(existingByTitle.getId(), rawId, raw.getTitle());
|
||||||
|
afterPagePersisted(existingByTitle.getId(), kbId, pageType, metadataNode, dependsOnNode, true);
|
||||||
|
log.info("[Wiki] Phase B create slug='{}' title='{}' canonical-title-matches existing '{}', updated",
|
||||||
|
slug, title, actualSlug);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
// Fallback 0: cross-spelling canonical match (DB has same concept under different slug)
|
// Fallback 0: cross-spelling canonical match (DB has same concept under different slug)
|
||||||
WikiPageEntity existingByCanonical = pageService.findByCanonicalSlug(kbId, slug);
|
WikiPageEntity existingByCanonical = pageService.findByCanonicalSlug(kbId, slug);
|
||||||
if (existingByCanonical != null && !existingByCanonical.getSlug().equals(slug)) {
|
if (existingByCanonical != null && !existingByCanonical.getSlug().equals(slug)) {
|
||||||
@ -1552,8 +1579,34 @@ public class WikiProcessingService {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback 0.5: in-flight slug-claim arbitration across parallel chunks
|
// Fallback 0.25: in-flight title-claim arbitration across parallel pages.
|
||||||
|
// Closes the race that findByCanonicalTitle cannot: two parallel phase-B
|
||||||
|
// creates with the same title but different slugs can both miss the DB
|
||||||
|
// lookup (the row isn't committed yet) and insert two rows — title has no
|
||||||
|
// DB unique constraint, so DuplicateKey won't catch it. The first to claim
|
||||||
|
// the canonical title wins; losers redirect their content into the winner.
|
||||||
ProgressCounter pcLocal = progressCounters.get(rawId);
|
ProgressCounter pcLocal = progressCounters.get(rawId);
|
||||||
|
String canonicalTitle = WikiPageService.canonicalTitle(title);
|
||||||
|
if (pcLocal != null && !canonicalTitle.isEmpty()) {
|
||||||
|
final String claimingSlug = slug;
|
||||||
|
String winnerSlug = pcLocal.titleClaims.computeIfAbsent(canonicalTitle, k -> claimingSlug);
|
||||||
|
if (!winnerSlug.equals(slug)) {
|
||||||
|
WikiPageEntity winner = pageService.getBySlug(kbId, winnerSlug);
|
||||||
|
if (winner != null) {
|
||||||
|
pageService.updatePageByAi(kbId, winnerSlug, content, pageSummary, rawId);
|
||||||
|
pageService.mergeSourceLineage(winner.getId(), rawId, raw.getTitle());
|
||||||
|
afterPagePersisted(winner.getId(), kbId, pageType, metadataNode, dependsOnNode, true);
|
||||||
|
log.info("[Wiki] Phase B create slug='{}' title='{}' lost title-claim race to '{}', updated",
|
||||||
|
slug, title, winnerSlug);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
log.info("[Wiki] Phase B create slug='{}' redirects to in-flight title winner '{}'",
|
||||||
|
slug, winnerSlug);
|
||||||
|
slug = winnerSlug;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback 0.5: in-flight slug-claim arbitration across parallel chunks
|
||||||
String canonical = WikiPageService.canonicalSlug(slug);
|
String canonical = WikiPageService.canonicalSlug(slug);
|
||||||
if (pcLocal != null && !canonical.isEmpty()) {
|
if (pcLocal != null && !canonical.isEmpty()) {
|
||||||
final String routedSlug = slug;
|
final String routedSlug = slug;
|
||||||
@ -2048,20 +2101,59 @@ public class WikiProcessingService {
|
|||||||
return "（暂无已有页面）";
|
return "（暂无已有页面）";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The index is injected verbatim into the route / batch-create prompts, so
|
||||||
|
// it must stay bounded — otherwise it grows linearly with the KB and
|
||||||
|
// overflows the model context window. Cap by both page count and chars;
|
||||||
|
// truncation is safe because savePageContent dedups by canonical title at
|
||||||
|
// persist time, so an omitted page is merged-on-save rather than duplicated.
|
||||||
|
int maxChars = Math.max(0, properties.getExistingPagesIndexMaxChars());
|
||||||
|
int maxPages = Math.max(0, properties.getExistingPagesIndexMaxPages());
|
||||||
|
|
||||||
|
// List manually-edited pages first so user-curated entries are never the
|
||||||
|
// ones dropped when a cap is hit. Title order within each group is
|
||||||
|
// preserved (listSummaries already sorts by title).
|
||||||
|
List<WikiPageEntity> ordered = new ArrayList<>(summaries.size());
|
||||||
|
for (WikiPageEntity p : summaries) {
|
||||||
|
if ("manual".equals(p.getLastUpdatedBy())) ordered.add(p);
|
||||||
|
}
|
||||||
|
for (WikiPageEntity p : summaries) {
|
||||||
|
if (!"manual".equals(p.getLastUpdatedBy())) ordered.add(p);
|
||||||
|
}
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
for (WikiPageEntity page : summaries) {
|
int listed = 0;
|
||||||
sb.append("- [[").append(page.getSlug()).append("]]");
|
for (WikiPageEntity page : ordered) {
|
||||||
|
StringBuilder row = new StringBuilder();
|
||||||
|
row.append("- [[").append(page.getSlug()).append("]]");
|
||||||
if (page.getTitle() != null && !page.getTitle().isBlank()) {
|
if (page.getTitle() != null && !page.getTitle().isBlank()) {
|
||||||
sb.append(" — ").append(page.getTitle());
|
row.append(" — ").append(page.getTitle());
|
||||||
}
|
}
|
||||||
if ("manual".equals(page.getLastUpdatedBy())) {
|
if ("manual".equals(page.getLastUpdatedBy())) {
|
||||||
sb.append(" (手动编辑)");
|
row.append(" (手动编辑)");
|
||||||
}
|
}
|
||||||
String summary = page.getSummary();
|
String summary = page.getSummary();
|
||||||
if (summary != null && !summary.isBlank()) {
|
if (summary != null && !summary.isBlank()) {
|
||||||
sb.append(" — ").append(summary);
|
row.append(" — ").append(summary);
|
||||||
}
|
}
|
||||||
sb.append("\n");
|
row.append("\n");
|
||||||
|
|
||||||
|
// Stop before exceeding either cap, but always emit at least one row.
|
||||||
|
boolean overPageCap = maxPages > 0 && listed >= maxPages;
|
||||||
|
boolean overCharCap = maxChars > 0 && listed > 0 && sb.length() + row.length() > maxChars;
|
||||||
|
if (overPageCap || overCharCap) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
sb.append(row);
|
||||||
|
listed++;
|
||||||
|
}
|
||||||
|
|
||||||
|
int omitted = ordered.size() - listed;
|
||||||
|
if (omitted > 0) {
|
||||||
|
sb.append("- …（已省略 ").append(omitted)
|
||||||
|
.append(" 个页面：已有页面过多，索引已截断。若材料涉及未列出的概念，按新建处理即可，")
|
||||||
|
.append("系统会在落库时按标题自动归并到既有页面）\n");
|
||||||
|
log.info("[Wiki] existing-pages index truncated for kbId={}: listed={} omitted={} chars={}",
|
||||||
|
kbId, listed, omitted, sb.length());
|
||||||
}
|
}
|
||||||
return sb.toString().trim();
|
return sb.toString().trim();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,150 @@
|
|||||||
|
package vip.mate.wiki.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||||
|
import org.junit.jupiter.api.AfterEach;
|
||||||
|
import org.junit.jupiter.api.DisplayName;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.boot.test.context.SpringBootTest;
|
||||||
|
import org.springframework.test.annotation.DirtiesContext;
|
||||||
|
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
|
||||||
|
import vip.mate.wiki.model.WikiPageEntity;
|
||||||
|
import vip.mate.wiki.repository.WikiKnowledgeBaseMapper;
|
||||||
|
import vip.mate.wiki.repository.WikiPageMapper;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* E2E coverage for {@link WikiPageService#mergeDuplicateTitles} — the one-time
|
||||||
|
* maintenance op that collapses pages sharing a canonical title (the duplicate
|
||||||
|
* rows produced before title-based dedup existed, when one concept landed under
|
||||||
|
* several LLM-minted slugs).
|
||||||
|
*
|
||||||
|
* <p>Boots the full Spring + H2 + Flyway context so MyBatis-Plus's lambda cache
|
||||||
|
* and the real cascade/link machinery are exercised end to end.
|
||||||
|
*/
|
||||||
|
@SpringBootTest(
|
||||||
|
webEnvironment = SpringBootTest.WebEnvironment.NONE,
|
||||||
|
properties = {
|
||||||
|
"spring.flyway.enabled=true",
|
||||||
|
"spring.flyway.locations=classpath:db/migration/h2",
|
||||||
|
"mateclaw.feature-flag.refresh-ms=999999"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
|
||||||
|
class WikiMergeDuplicateTitlesE2ETest {
|
||||||
|
|
||||||
|
@Autowired private WikiPageService pageService;
|
||||||
|
@Autowired private WikiKnowledgeBaseService kbService;
|
||||||
|
@Autowired private WikiPageMapper pageMapper;
|
||||||
|
@Autowired private WikiKnowledgeBaseMapper kbMapper;
|
||||||
|
|
||||||
|
private Long kbId;
|
||||||
|
|
||||||
|
@AfterEach
|
||||||
|
void cleanup() {
|
||||||
|
if (kbId != null) {
|
||||||
|
pageMapper.delete(new LambdaQueryWrapper<WikiPageEntity>().eq(WikiPageEntity::getKbId, kbId));
|
||||||
|
kbMapper.deleteById(kbId);
|
||||||
|
pageService.evictSummaryCache(kbId);
|
||||||
|
kbId = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void seedKb() {
|
||||||
|
WikiKnowledgeBaseEntity kb = kbService.create("merge-dup-" + System.nanoTime(), "merge test", null);
|
||||||
|
kbId = kb.getId();
|
||||||
|
pageMapper.delete(new LambdaQueryWrapper<WikiPageEntity>().eq(WikiPageEntity::getKbId, kbId));
|
||||||
|
pageService.evictSummaryCache(kbId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Three pages share the canonical title "医宗金鉴" under different slugs (the
|
||||||
|
* exact failure mode from the bug report), plus a referrer linking to a loser.
|
||||||
|
*/
|
||||||
|
private void seedDuplicates() {
|
||||||
|
// Winner: longest content.
|
||||||
|
pageService.createPage(kbId, "yizong-jinjian", "医宗金鉴",
|
||||||
|
"## 医宗金鉴\n\nThis is the most complete body with the richest detail aaa bbb ccc ddd.",
|
||||||
|
"complete summary", "[]");
|
||||||
|
// Losers: same canonical title, shorter content, different slugs.
|
||||||
|
pageService.createPage(kbId, "yizong-jinjian-quanshu", "医宗金鉴",
|
||||||
|
"Body B shorter.", "b summary", "[]");
|
||||||
|
// Trailing-space title still canonicalizes equal.
|
||||||
|
pageService.createPage(kbId, "yzjj", "医宗金鉴 ",
|
||||||
|
"Body C tiny.", "c summary", "[]");
|
||||||
|
// A referrer pointing at one of the losers.
|
||||||
|
pageService.createPage(kbId, "ref", "Ref",
|
||||||
|
"See [[yizong-jinjian-quanshu]] for the canonical text.",
|
||||||
|
"referrer summary", "[]");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("dry run reports duplicates without mutating anything")
|
||||||
|
void dryRunReportsButDoesNotMutate() {
|
||||||
|
seedKb();
|
||||||
|
seedDuplicates();
|
||||||
|
|
||||||
|
Map<String, Object> report = pageService.mergeDuplicateTitles(kbId, true, true);
|
||||||
|
|
||||||
|
assertThat(report.get("dryRun")).isEqualTo(true);
|
||||||
|
assertThat(report.get("duplicateGroups")).isEqualTo(1);
|
||||||
|
assertThat(report.get("pagesWouldRemove")).isEqualTo(2);
|
||||||
|
assertThat(report.get("pagesRemoved")).isEqualTo(0);
|
||||||
|
|
||||||
|
// Nothing deleted: all four pages still present.
|
||||||
|
assertThat(pageService.listByKbIdWithContent(kbId)).hasSize(4);
|
||||||
|
assertThat(pageService.getBySlug(kbId, "yizong-jinjian-quanshu")).isNotNull();
|
||||||
|
assertThat(pageService.getBySlug(kbId, "yzjj")).isNotNull();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("apply with concatenate merges losers into winner, redirects refs, deletes losers")
|
||||||
|
void applyConcatenateCollapsesGroup() {
|
||||||
|
seedKb();
|
||||||
|
seedDuplicates();
|
||||||
|
|
||||||
|
Map<String, Object> report = pageService.mergeDuplicateTitles(kbId, false, true);
|
||||||
|
assertThat(report.get("duplicateGroups")).isEqualTo(1);
|
||||||
|
assertThat(report.get("pagesRemoved")).isEqualTo(2);
|
||||||
|
|
||||||
|
// Only the winner + the referrer survive.
|
||||||
|
assertThat(pageService.listByKbIdWithContent(kbId)).hasSize(2);
|
||||||
|
assertThat(pageService.getBySlug(kbId, "yizong-jinjian-quanshu")).isNull();
|
||||||
|
assertThat(pageService.getBySlug(kbId, "yzjj")).isNull();
|
||||||
|
|
||||||
|
// Winner keeps its body and gains the losers' bodies (no content lost).
|
||||||
|
WikiPageEntity winner = pageService.getBySlug(kbId, "yizong-jinjian");
|
||||||
|
assertThat(winner).isNotNull();
|
||||||
|
assertThat(winner.getContent())
|
||||||
|
.contains("most complete body")
|
||||||
|
.contains("Body B shorter.")
|
||||||
|
.contains("Body C tiny.");
|
||||||
|
assertThat(winner.getVersion()).isGreaterThan(1);
|
||||||
|
|
||||||
|
// Referrer's [[loserSlug]] is redirected to the winner, not demoted.
|
||||||
|
WikiPageEntity ref = pageService.getBySlug(kbId, "ref");
|
||||||
|
assertThat(ref.getContent())
|
||||||
|
.doesNotContain("[[yizong-jinjian-quanshu]]")
|
||||||
|
.contains("[[yizong-jinjian]]");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("apply without concatenate keeps only the winner's body")
|
||||||
|
void applyWithoutConcatenateDiscardsLoserBodies() {
|
||||||
|
seedKb();
|
||||||
|
seedDuplicates();
|
||||||
|
|
||||||
|
pageService.mergeDuplicateTitles(kbId, false, false);
|
||||||
|
|
||||||
|
assertThat(pageService.listByKbIdWithContent(kbId)).hasSize(2);
|
||||||
|
WikiPageEntity winner = pageService.getBySlug(kbId, "yizong-jinjian");
|
||||||
|
assertThat(winner).isNotNull();
|
||||||
|
assertThat(winner.getContent())
|
||||||
|
.contains("most complete body")
|
||||||
|
.doesNotContain("Body B shorter.")
|
||||||
|
.doesNotContain("Body C tiny.");
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,12 +1,19 @@
|
|||||||
package vip.mate.wiki.service;
|
package vip.mate.wiki.service;
|
||||||
|
|
||||||
|
import com.baomidou.mybatisplus.core.MybatisConfiguration;
|
||||||
|
import com.baomidou.mybatisplus.core.metadata.TableInfoHelper;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import org.apache.ibatis.builder.MapperBuilderAssistant;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
import vip.mate.wiki.model.WikiPageEntity;
|
import vip.mate.wiki.model.WikiPageEntity;
|
||||||
import vip.mate.wiki.repository.WikiPageMapper;
|
import vip.mate.wiki.repository.WikiPageMapper;
|
||||||
|
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
@ -15,6 +22,15 @@ import static org.mockito.Mockito.when;
|
|||||||
|
|
||||||
class WikiPageServiceTest {
|
class WikiPageServiceTest {
|
||||||
|
|
||||||
|
static {
|
||||||
|
// LambdaQueryWrapper resolves column metadata from MyBatis-Plus's TableInfo
|
||||||
|
// cache, which Spring normally populates at startup. In a plain unit test we
|
||||||
|
// seed it once so getBySlug / listSummaries can build their lambda queries.
|
||||||
|
TableInfoHelper.initTableInfo(
|
||||||
|
new MapperBuilderAssistant(new MybatisConfiguration(), ""),
|
||||||
|
WikiPageEntity.class);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void manualUpdateRefreshesUpdateTimeBeforePersisting() {
|
void manualUpdateRefreshesUpdateTimeBeforePersisting() {
|
||||||
WikiPageMapper mapper = mock(WikiPageMapper.class);
|
WikiPageMapper mapper = mock(WikiPageMapper.class);
|
||||||
@ -39,4 +55,61 @@ class WikiPageServiceTest {
|
|||||||
assertTrue(page.getUpdateTime().isAfter(oldUpdateTime));
|
assertTrue(page.getUpdateTime().isAfter(oldUpdateTime));
|
||||||
verify(mapper).updateById(page);
|
verify(mapper).updateById(page);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void canonicalTitleFoldsCaseAndSeparators() {
|
||||||
|
// Case + ASCII separators fold away so spelling variants collapse to one key.
|
||||||
|
assertEquals("erweibadusan", WikiPageService.canonicalTitle("Erwei-Badu_San"));
|
||||||
|
assertEquals("erweibadusan", WikiPageService.canonicalTitle("er wei badu san"));
|
||||||
|
// Chinese title with surrounding/full-width whitespace and an inserted hyphen.
|
||||||
|
assertEquals("二味拔毒散", WikiPageService.canonicalTitle(" 二味-拔毒散 "));
|
||||||
|
assertEquals(WikiPageService.canonicalTitle("二味拔毒散"),
|
||||||
|
WikiPageService.canonicalTitle("二味拔毒散 "));
|
||||||
|
// Null / blank degrade to empty so callers can short-circuit.
|
||||||
|
assertEquals("", WikiPageService.canonicalTitle(null));
|
||||||
|
assertEquals("", WikiPageService.canonicalTitle(" "));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void findByCanonicalTitleMatchesAcrossDifferentSlugs() {
|
||||||
|
// Same concept already stored under an LLM-chosen slug; a later run arrives
|
||||||
|
// with the same title but would have minted a different slug. Title match
|
||||||
|
// must find the existing row regardless of the slug spelling.
|
||||||
|
WikiPageMapper mapper = mock(WikiPageMapper.class);
|
||||||
|
WikiPageEntity summary = new WikiPageEntity();
|
||||||
|
summary.setKbId(7L);
|
||||||
|
summary.setSlug("erwei-badu-san");
|
||||||
|
summary.setTitle("二味拔毒散");
|
||||||
|
WikiPageEntity full = new WikiPageEntity();
|
||||||
|
full.setId(42L);
|
||||||
|
full.setKbId(7L);
|
||||||
|
full.setSlug("erwei-badu-san");
|
||||||
|
full.setTitle("二味拔毒散");
|
||||||
|
// listSummaries() -> selectList ; getBySlug() -> selectOne
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of(summary));
|
||||||
|
when(mapper.selectOne(any())).thenReturn(full);
|
||||||
|
|
||||||
|
ObjectMapper om = new ObjectMapper();
|
||||||
|
WikiPageService service = new WikiPageService(mapper, om, new WikiLinkService(om));
|
||||||
|
|
||||||
|
WikiPageEntity hit = service.findByCanonicalTitle(7L, "二味拔毒散");
|
||||||
|
assertNotNull(hit);
|
||||||
|
assertEquals(42L, hit.getId());
|
||||||
|
assertEquals("erwei-badu-san", hit.getSlug());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void findByCanonicalTitleReturnsNullWhenNoConceptMatches() {
|
||||||
|
WikiPageMapper mapper = mock(WikiPageMapper.class);
|
||||||
|
WikiPageEntity summary = new WikiPageEntity();
|
||||||
|
summary.setKbId(7L);
|
||||||
|
summary.setSlug("shennong-bencao");
|
||||||
|
summary.setTitle("神农本草经");
|
||||||
|
when(mapper.selectList(any())).thenReturn(List.of(summary));
|
||||||
|
|
||||||
|
ObjectMapper om = new ObjectMapper();
|
||||||
|
WikiPageService service = new WikiPageService(mapper, om, new WikiLinkService(om));
|
||||||
|
|
||||||
|
assertNull(service.findByCanonicalTitle(7L, "二味拔毒散"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user