mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(wiki): broken-link rescan precision, slug/title resolution, and dangling-link reconcile (#333)
- rescan: keep the KB id as a string end to end so the 19-digit snowflake id isn't truncated past Number.MAX_SAFE_INTEGER (rescan no longer 404s) - lint: resolve [[...]] targets against page slugs AND titles like the viewer, so a title reference to an existing page is no longer reported broken - ingest: derive the slug deterministically from the title (no inconsistent romanization), auto-recompute broken links once a KB finishes importing, and reconcile dangling [[concept]] links — redirect to the covering page via declared aliases, or demote to plain text when uncovered - add the page aliases column migration for h2 / mysql / kingbase
This commit is contained in:
matevip
parent
6a13cc2f50
commit
d9a9d07704
@ -38,6 +38,16 @@ public class WikiPageEntity {
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private String outgoingLinks;
|
||||
|
||||
/**
|
||||
* Alternate concept names this page also covers (JSON array, e.g.
|
||||
* ["叶绿体","线粒体"]). Set for discrimination / composite pages that absorb
|
||||
* several fine-grained concepts which never became standalone pages. The
|
||||
* post-ingestion link reconciler uses these so a [[叶绿体]] reference from
|
||||
* another page resolves to this page instead of dangling.
|
||||
*/
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private String aliases;
|
||||
|
||||
/** 来源原始材料 ID（JSON 数组） */
|
||||
@TableField(updateStrategy = FieldStrategy.ALWAYS)
|
||||
private String sourceRawIds;
|
||||
|
||||
@ -9,6 +9,7 @@ import vip.mate.wiki.model.WikiPageEntity;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
@ -114,19 +115,27 @@ public class WikiLinkService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the broken subset of {@code outlinks} given the KB's active
|
||||
* page slug set. {@code activeSlugs} is expected to be already lowercased
|
||||
* — callers compute it once per scan and reuse across pages.
|
||||
* Compute the broken subset of {@code outlinks} given the KB's set of
|
||||
* resolvable link-target keys. {@code resolvableKeysLower} is expected to
|
||||
* be already lowercased and to contain BOTH page slugs and page titles
|
||||
* (see {@link #resolvableTargetKeys}) — callers compute it once per scan
|
||||
* and reuse it across pages.
|
||||
* <p>
|
||||
* A target counts as broken only when it matches neither a slug nor a
|
||||
* title, mirroring the page viewer's {@code resolveWikilink}. Matching on
|
||||
* slugs alone would report every {@code [[Page Title]]} reference to an
|
||||
* existing page as broken even though the viewer renders it as a working
|
||||
* link.
|
||||
*
|
||||
* @return targets that have no matching page slug, in the same insertion
|
||||
* @return targets that resolve to no existing page, in the same insertion
|
||||
* order as {@code outlinks}
|
||||
*/
|
||||
public List<String> computeBrokenLinks(Set<String> outlinks, Set<String> activeSlugsLower) {
|
||||
public List<String> computeBrokenLinks(Set<String> outlinks, Set<String> resolvableKeysLower) {
|
||||
if (outlinks == null || outlinks.isEmpty()) return Collections.emptyList();
|
||||
if (activeSlugsLower == null) activeSlugsLower = Collections.emptySet();
|
||||
if (resolvableKeysLower == null) resolvableKeysLower = Collections.emptySet();
|
||||
List<String> broken = new ArrayList<>();
|
||||
for (String t : outlinks) {
|
||||
if (!activeSlugsLower.contains(t)) broken.add(t);
|
||||
if (!resolvableKeysLower.contains(t)) broken.add(t);
|
||||
}
|
||||
return broken;
|
||||
}
|
||||
@ -134,11 +143,12 @@ public class WikiLinkService {
|
||||
/**
|
||||
* Convenience: extract + compute in one call. Used from
|
||||
* {@code WikiPageService.save/update} where both fields are written in
|
||||
* the same transaction.
|
||||
* the same transaction. {@code resolvableKeysLower} should carry slugs
|
||||
* and titles — see {@link #resolvableTargetKeys}.
|
||||
*/
|
||||
public LinkAnalysis analyze(String content, Set<String> activeSlugsLower) {
|
||||
public LinkAnalysis analyze(String content, Set<String> resolvableKeysLower) {
|
||||
Set<String> outlinks = extractOutlinks(content);
|
||||
List<String> broken = computeBrokenLinks(outlinks, activeSlugsLower);
|
||||
List<String> broken = computeBrokenLinks(outlinks, resolvableKeysLower);
|
||||
return new LinkAnalysis(new ArrayList<>(outlinks), broken);
|
||||
}
|
||||
|
||||
@ -181,6 +191,45 @@ public class WikiLinkService {
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the set of resolvable wikilink-target keys for a KB from a
|
||||
* pre-loaded page list — the union of each page's lowercased slug AND its
|
||||
* trimmed-lowercased title.
|
||||
* <p>
|
||||
* This is the broken-link counterpart to {@link #lowercaseSlugSet} and is
|
||||
* what {@link #computeBrokenLinks} should be fed: it mirrors the page
|
||||
* viewer's {@code resolveWikilink}, which resolves a {@code [[target]]}
|
||||
* against an exact slug OR an exact title before declaring it broken.
|
||||
* Using slugs alone flags every {@code [[Page Title]]} reference to an
|
||||
* existing page as broken even though the viewer renders it as a working
|
||||
* link — a false positive that is pervasive when slugs are transliterated
|
||||
* (e.g. a CJK title {@code 光合作用} stored under the pinyin slug
|
||||
* {@code guanghe-zuoyong}).
|
||||
* <p>
|
||||
* Titles are trimmed before lowercasing to match {@code extractOutlinks},
|
||||
* which trims a {@code [[ Page Title ]]} target before recording it.
|
||||
*
|
||||
* @param pages active pages (callers filter out archived); both slug and
|
||||
* title columns must be loaded
|
||||
* @return mutable set of lowercased slug + title keys; empty for null/empty
|
||||
*/
|
||||
public Set<String> resolvableTargetKeys(List<WikiPageEntity> pages) {
|
||||
if (pages == null || pages.isEmpty()) return new HashSet<>();
|
||||
Set<String> keys = new HashSet<>(pages.size() * 2);
|
||||
for (WikiPageEntity p : pages) {
|
||||
if (p == null) continue;
|
||||
String slug = p.getSlug();
|
||||
if (slug != null && !slug.isBlank()) {
|
||||
keys.add(slug.toLowerCase(Locale.ROOT));
|
||||
}
|
||||
String title = p.getTitle();
|
||||
if (title != null && !title.isBlank()) {
|
||||
keys.add(title.trim().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Cascade rewrite — used by page delete + rename to update referrers
|
||||
// ============================================================
|
||||
@ -236,6 +285,48 @@ public class WikiLinkService {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides what to do with one wikilink during reconciliation. Receives the
|
||||
* original-case target (before any {@code |alias}) and the explicit alias
|
||||
* (or {@code null}); returns {@code null} to leave the link untouched, or
|
||||
* the literal replacement text — plain text to demote a dangling link, or a
|
||||
* re-formed {@code [[coverSlug|display]]} to redirect it to a covering page.
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface LinkReconciler {
|
||||
String reconcile(String target, String alias);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk {@code content} and hand every wikilink to {@code reconciler}, used
|
||||
* by the post-ingestion pass that redirects or demotes links the model
|
||||
* wrote to concepts that never became their own page. Mirrors
|
||||
* {@link #rewriteWikilinks} (code spans preserved verbatim) but passes the
|
||||
* original-case target so the reconciler can build readable display text.
|
||||
*/
|
||||
public String reconcileLinks(String content, LinkReconciler reconciler) {
|
||||
if (content == null || content.isEmpty()) return content;
|
||||
List<Region> regions = splitByCode(content);
|
||||
StringBuilder out = new StringBuilder(content.length() + 16);
|
||||
for (Region r : regions) {
|
||||
if (r.isCode) { out.append(r.text); continue; }
|
||||
Matcher m = WIKILINK.matcher(r.text);
|
||||
int last = 0;
|
||||
while (m.find()) {
|
||||
out.append(r.text, last, m.start());
|
||||
String raw = m.group(1).trim();
|
||||
int pipe = raw.indexOf('|');
|
||||
String target = (pipe >= 0 ? raw.substring(0, pipe) : raw).trim();
|
||||
String alias = pipe >= 0 ? raw.substring(pipe + 1).trim() : null;
|
||||
String replacement = target.isEmpty() ? null : reconciler.reconcile(target, alias);
|
||||
out.append(replacement == null ? m.group() : replacement);
|
||||
last = m.end();
|
||||
}
|
||||
out.append(r.text, last, r.text.length());
|
||||
}
|
||||
return out.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk {@code content} replacing wikilinks via {@code rewriter}. Code
|
||||
* spans are detected and restored verbatim — replacement only happens in
|
||||
|
||||
@ -216,11 +216,14 @@ public class WikiLintJobService {
|
||||
* the whole scan.
|
||||
*/
|
||||
private ScanCounts scan(Long kbId) {
|
||||
// listSummaries gives us the active (non-archived) page slug set —
|
||||
// archived pages are NOT considered as valid targets, matching the
|
||||
// resolver's behaviour.
|
||||
// listSummaries gives us the active (non-archived) pages — archived
|
||||
// pages are NOT considered as valid targets, matching the resolver's
|
||||
// behaviour. The key set carries both slugs AND titles so a
|
||||
// `[[Page Title]]` reference to an existing page resolves the same way
|
||||
// the viewer renders it, instead of being reported as a false-positive
|
||||
// broken link.
|
||||
List<WikiPageEntity> summaries = pageService.listSummaries(kbId);
|
||||
Set<String> activeSlugs = linkService.lowercaseSlugSet(summaries);
|
||||
Set<String> activeSlugs = linkService.resolvableTargetKeys(summaries);
|
||||
|
||||
// Now fetch the same pages WITH content so we can re-extract outlinks.
|
||||
// We must not use listSummaries here because it omits the content
|
||||
|
||||
@ -767,25 +767,23 @@ public class WikiPageService {
|
||||
kbId, deletedPageId, likePattern);
|
||||
if (candidates.isEmpty()) return List.of();
|
||||
|
||||
// Pre-compute the active slug set ONCE for the recompute pass — every
|
||||
// referrer's broken_links recompute would otherwise re-trigger the
|
||||
// summary query.
|
||||
// Pre-compute the resolvable target keys (slugs + titles) ONCE for the
|
||||
// recompute pass — every referrer's broken_links recompute would
|
||||
// otherwise re-trigger the summary query.
|
||||
Set<String> activeSlugs;
|
||||
try {
|
||||
activeSlugs = linkService.lowercaseSlugSet(listSummaries(kbId));
|
||||
activeSlugs = linkService.resolvableTargetKeys(listSummaries(kbId));
|
||||
} catch (RuntimeException e) {
|
||||
activeSlugs = java.util.Collections.emptySet();
|
||||
activeSlugs = new HashSet<>();
|
||||
}
|
||||
// The deleted page is, by construction, no longer "active" — remove
|
||||
// its slug from the set so any referrers' broken_links recompute
|
||||
// doesn't accidentally still resolve `[[deletedSlug]]` in their
|
||||
// (now-rewritten) content.
|
||||
if (!activeSlugs.contains(slugLower)) {
|
||||
// already missing — common case
|
||||
} else {
|
||||
Set<String> trimmed = new HashSet<>(activeSlugs);
|
||||
trimmed.remove(slugLower);
|
||||
activeSlugs = trimmed;
|
||||
// its slug AND title from the set so any referrers' broken_links
|
||||
// recompute doesn't accidentally still resolve `[[deletedSlug]]` or
|
||||
// `[[Deleted Title]]` in their (now-rewritten) content if listSummaries
|
||||
// returned a stale cache that still included the deleted page.
|
||||
activeSlugs.remove(slugLower);
|
||||
if (snapshotTitle != null && !snapshotTitle.isBlank()) {
|
||||
activeSlugs.remove(snapshotTitle.trim().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
|
||||
List<Long> affected = new ArrayList<>(candidates.size());
|
||||
@ -922,18 +920,17 @@ public class WikiPageService {
|
||||
|
||||
Set<String> activeSlugs;
|
||||
try {
|
||||
activeSlugs = linkService.lowercaseSlugSet(listSummaries(kbId));
|
||||
activeSlugs = linkService.resolvableTargetKeys(listSummaries(kbId));
|
||||
} catch (RuntimeException e) {
|
||||
activeSlugs = java.util.Collections.emptySet();
|
||||
activeSlugs = new HashSet<>();
|
||||
}
|
||||
// The renamed page is now under newSlug; oldSlug is gone, newSlug
|
||||
// should resolve. listSummaries has been evicted above so this picks
|
||||
// up the new row when re-queried, but be defensive in case the cache
|
||||
// hasn't repopulated yet.
|
||||
Set<String> activeBase = new HashSet<>(activeSlugs);
|
||||
activeBase.remove(slugLower);
|
||||
activeBase.add(newSlug.toLowerCase(Locale.ROOT));
|
||||
activeSlugs = activeBase;
|
||||
// hasn't repopulated yet. The title is unchanged by a rename, so it
|
||||
// stays resolvable via the title key carried in the set.
|
||||
activeSlugs.remove(slugLower);
|
||||
activeSlugs.add(newSlug.toLowerCase(Locale.ROOT));
|
||||
|
||||
List<Long> affected = new ArrayList<>(candidates.size());
|
||||
for (WikiPageEntity referrer : candidates) {
|
||||
@ -1287,28 +1284,125 @@ public class WikiPageService {
|
||||
// update path) and every extracted target is recorded as broken —
|
||||
// which is harmless because tests don't assert on broken_links
|
||||
// values, and production code paths never hit this branch.
|
||||
Set<String> activeSlugs;
|
||||
Set<String> resolvableKeys;
|
||||
try {
|
||||
activeSlugs = linkService.lowercaseSlugSet(listSummaries(entity.getKbId()));
|
||||
resolvableKeys = linkService.resolvableTargetKeys(listSummaries(entity.getKbId()));
|
||||
} catch (RuntimeException e) {
|
||||
log.warn("[Wiki] applyLinkAnalysis: failed to load slug set for kbId={}, treating as empty: {}",
|
||||
log.warn("[Wiki] applyLinkAnalysis: failed to load target keys for kbId={}, treating as empty: {}",
|
||||
entity.getKbId(), e.toString());
|
||||
activeSlugs = java.util.Collections.emptySet();
|
||||
resolvableKeys = new HashSet<>();
|
||||
}
|
||||
// Include self-slug so [[my-own-slug]] doesn't appear as broken on the
|
||||
// very save that creates the page (listSummaries may not see it yet
|
||||
// depending on cache state).
|
||||
// Include self slug + title so [[my-own-slug]] / [[My Own Title]] don't
|
||||
// appear as broken on the very save that creates the page (listSummaries
|
||||
// may not see it yet depending on cache state).
|
||||
if (entity.getSlug() != null && !entity.getSlug().isBlank()) {
|
||||
Set<String> withSelf = new HashSet<>(activeSlugs);
|
||||
withSelf.add(entity.getSlug().toLowerCase(Locale.ROOT));
|
||||
activeSlugs = withSelf;
|
||||
resolvableKeys.add(entity.getSlug().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
WikiLinkService.LinkAnalysis a = linkService.analyze(entity.getContent(), activeSlugs);
|
||||
if (entity.getTitle() != null && !entity.getTitle().isBlank()) {
|
||||
resolvableKeys.add(entity.getTitle().trim().toLowerCase(Locale.ROOT));
|
||||
}
|
||||
WikiLinkService.LinkAnalysis a = linkService.analyze(entity.getContent(), resolvableKeys);
|
||||
entity.setOutgoingLinks(linkService.toJsonArray(a.outgoingLinks()));
|
||||
entity.setBrokenLinks(linkService.toJsonArray(a.brokenLinks()));
|
||||
entity.setBrokenLinksScannedAt(LocalDateTime.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge {@code newAliases} into the page identified by {@code title} (the
|
||||
* canonical concept identity used for dedup). Aliases are the alternate
|
||||
* concept names a discrimination / composite page also covers; they let the
|
||||
* post-ingestion reconciler redirect {@code [[concept]]} references that
|
||||
* never became their own page. The page's own title is never stored as an
|
||||
* alias of itself. No-op when the page or alias list is empty.
|
||||
*/
|
||||
public void mergeAliasesByTitle(Long kbId, String title, List<String> newAliases) {
|
||||
if (kbId == null || title == null || newAliases == null || newAliases.isEmpty()) return;
|
||||
WikiPageEntity page = findByCanonicalTitle(kbId, title);
|
||||
if (page == null) page = getBySlug(kbId, toSlug(title));
|
||||
if (page == null) return;
|
||||
Set<String> merged = new java.util.LinkedHashSet<>(linkService.fromJsonArray(page.getAliases()));
|
||||
String ownTitle = page.getTitle() == null ? "" : page.getTitle().trim();
|
||||
boolean added = false;
|
||||
for (String a : newAliases) {
|
||||
String t = a == null ? "" : a.trim();
|
||||
if (t.isEmpty() || t.equalsIgnoreCase(ownTitle)) continue;
|
||||
if (merged.add(t)) added = true;
|
||||
}
|
||||
if (!added) return;
|
||||
pageMapper.update(null, new com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper<WikiPageEntity>()
|
||||
.eq(WikiPageEntity::getId, page.getId())
|
||||
.set(WikiPageEntity::getAliases, linkService.toJsonArray(new ArrayList<>(merged))));
|
||||
evictSummaryCache(kbId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-ingestion link reconciliation for a KB. Each {@code [[target]]} the
|
||||
* model wrote is handled by where its concept ended up:
|
||||
* <ul>
|
||||
* <li>resolves to a real page (slug or title) → left untouched;</li>
|
||||
* <li>matches another page's declared alias → rewritten to
|
||||
* {@code [[coverSlug|target]]} so it links to the covering page;</li>
|
||||
* <li>otherwise → demoted to plain text (the concept name), removing the
|
||||
* dangling link entirely.</li>
|
||||
* </ul>
|
||||
* Only pages whose content actually changes are re-persisted, so re-running
|
||||
* after a settled KB is a cheap no-op. Touches content + outgoing_links
|
||||
* only — the caller recomputes broken_links via the lint scan afterwards.
|
||||
*
|
||||
* @return the number of pages rewritten
|
||||
*/
|
||||
public int reconcileKbLinks(Long kbId) {
|
||||
if (kbId == null) return 0;
|
||||
List<WikiPageEntity> pages = pageMapper.selectList(
|
||||
new LambdaQueryWrapper<WikiPageEntity>()
|
||||
.select(WikiPageEntity::getId, WikiPageEntity::getSlug, WikiPageEntity::getTitle,
|
||||
WikiPageEntity::getContent, WikiPageEntity::getAliases)
|
||||
.eq(WikiPageEntity::getKbId, kbId)
|
||||
.ne(WikiPageEntity::getArchived, 1));
|
||||
if (pages.isEmpty()) return 0;
|
||||
Set<String> resolvable = linkService.resolvableTargetKeys(pages);
|
||||
// alias (lowercased) → covering page slug; first declarer wins, and a
|
||||
// name owned by a real page is never treated as an alias.
|
||||
Map<String, String> aliasToSlug = new LinkedHashMap<>();
|
||||
for (WikiPageEntity p : pages) {
|
||||
if (p.getSlug() == null || p.getSlug().isBlank()) continue;
|
||||
for (String a : linkService.fromJsonArray(p.getAliases())) {
|
||||
String key = a == null ? "" : a.trim().toLowerCase(Locale.ROOT);
|
||||
if (key.isEmpty() || resolvable.contains(key)) continue;
|
||||
aliasToSlug.putIfAbsent(key, p.getSlug());
|
||||
}
|
||||
}
|
||||
int changed = 0;
|
||||
for (WikiPageEntity p : pages) {
|
||||
String content = p.getContent();
|
||||
if (content == null || !content.contains("[[")) continue;
|
||||
String selfSlug = p.getSlug() == null ? "" : p.getSlug().toLowerCase(Locale.ROOT);
|
||||
String reconciled = linkService.reconcileLinks(content, (target, alias) -> {
|
||||
String key = target.trim().toLowerCase(Locale.ROOT);
|
||||
if (resolvable.contains(key)) return null; // real page — keep
|
||||
String display = (alias != null && !alias.isBlank()) ? alias : target;
|
||||
String coverSlug = aliasToSlug.get(key);
|
||||
if (coverSlug != null && !coverSlug.toLowerCase(Locale.ROOT).equals(selfSlug)) {
|
||||
return "[[" + coverSlug + "|" + display + "]]"; // redirect to covering page
|
||||
}
|
||||
return display; // demote dangling link to plain text
|
||||
});
|
||||
if (!reconciled.equals(content)) {
|
||||
List<String> outlinks = new ArrayList<>(linkService.extractOutlinks(reconciled));
|
||||
pageMapper.update(null, new com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper<WikiPageEntity>()
|
||||
.eq(WikiPageEntity::getId, p.getId())
|
||||
.set(WikiPageEntity::getContent, reconciled)
|
||||
.set(WikiPageEntity::getOutgoingLinks, linkService.toJsonArray(outlinks)));
|
||||
changed++;
|
||||
}
|
||||
}
|
||||
if (changed > 0) {
|
||||
evictSummaryCache(kbId);
|
||||
log.info("[Wiki] Link reconciliation rewrote {} page(s) for kbId={}", changed, kbId);
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将标题转换为 slug（URL 安全标识符）
|
||||
*/
|
||||
|
||||
@ -118,6 +118,15 @@ public class WikiProcessingService {
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private WikiScaffoldService scaffoldService;
|
||||
|
||||
/**
|
||||
* Recomputes broken links once a raw material finishes processing. Optional
|
||||
* (field-injected) so unit tests that construct this service directly
|
||||
* without Spring don't need to supply it — when absent, the post-ingestion
|
||||
* auto-scan is simply skipped.
|
||||
*/
|
||||
@org.springframework.beans.factory.annotation.Autowired(required = false)
|
||||
private WikiLintJobService lintJobService;
|
||||
|
||||
/**
|
||||
* RFC-051 PR-3: optional model routing service. When wired, route /
|
||||
* create_page / merge_page LLM calls inside the eager pipeline ask the
|
||||
@ -537,6 +546,41 @@ public class WikiProcessingService {
|
||||
ProgressCounter pc = progressCounters.remove(rawId);
|
||||
if (pc != null) {
|
||||
rawService.updateProgress(rawId, "done", pc.done.get(), pc.total.get());
|
||||
// Once every material in the KB has settled, reconcile links and
|
||||
// recompute broken_links. Gating on "no unsettled raws" avoids
|
||||
// two hazards of doing this per-material mid-batch: (1) demoting a
|
||||
// [[concept]] link before a later material creates that page, and
|
||||
// (2) recording a link to a later-created page as broken. The last
|
||||
// material to finish runs both; earlier completions skip. Both
|
||||
// steps are idempotent, so a rare concurrent double-run is benign.
|
||||
boolean kbSettled;
|
||||
try {
|
||||
kbSettled = rawService.listByKbId(kb.getId()).stream()
|
||||
.noneMatch(r -> "pending".equals(r.getProcessingStatus())
|
||||
|| "processing".equals(r.getProcessingStatus()));
|
||||
} catch (RuntimeException e) {
|
||||
kbSettled = true; // best-effort: prefer reconciling over skipping
|
||||
}
|
||||
if (kbSettled) {
|
||||
try {
|
||||
// Redirect alias-covered links to their covering page and
|
||||
// demote the genuinely uncovered ones to plain text before
|
||||
// the scan, so freshly imported content doesn't surface
|
||||
// dangling links to concepts that were merged away.
|
||||
pageService.reconcileKbLinks(kb.getId());
|
||||
} catch (RuntimeException reconErr) {
|
||||
log.warn("[Wiki] post-ingestion link reconciliation failed for kbId={}: {}",
|
||||
kb.getId(), reconErr.toString());
|
||||
}
|
||||
if (lintJobService != null) {
|
||||
try {
|
||||
lintJobService.startOrGetRunning(kb.getId());
|
||||
} catch (RuntimeException scanErr) {
|
||||
log.warn("[Wiki] post-ingestion broken-link scan trigger failed for kbId={}: {}",
|
||||
kb.getId(), scanErr.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1303,6 +1347,17 @@ public class WikiProcessingService {
|
||||
}
|
||||
JsonNode metadataNode = pageJson.path("metadata");
|
||||
JsonNode dependsOnNode = pageJson.path("depends_on");
|
||||
// Alternate concept names this page covers (composite / 辨析
|
||||
// pages list the fine-grained concepts they absorbed) — used by
|
||||
// the post-ingestion reconciler to redirect [[concept]] links.
|
||||
List<String> pageAliases = new ArrayList<>();
|
||||
JsonNode aliasesNode = pageJson.path("aliases");
|
||||
if (aliasesNode.isArray()) {
|
||||
for (JsonNode a : aliasesNode) {
|
||||
String s = a.asText("").trim();
|
||||
if (!s.isEmpty()) pageAliases.add(s);
|
||||
}
|
||||
}
|
||||
if (content.isBlank()) {
|
||||
log.info("[Wiki] BatchCreate: blank content for slug='{}', retrying individually", slug);
|
||||
final String blankSlug = slug;
|
||||
@ -1332,6 +1387,13 @@ public class WikiProcessingService {
|
||||
boolean ok = false;
|
||||
try {
|
||||
wasCreated = savePageContent(kb, raw, slug, title, content, pageSummary, pageType, metadataNode, dependsOnNode);
|
||||
if (!pageAliases.isEmpty()) {
|
||||
try {
|
||||
pageService.mergeAliasesByTitle(kbId, title, pageAliases);
|
||||
} catch (RuntimeException aliasErr) {
|
||||
log.warn("[Wiki] Failed to persist aliases for title='{}': {}", title, aliasErr.toString());
|
||||
}
|
||||
}
|
||||
if (wasCreated) {
|
||||
created.incrementAndGet();
|
||||
totalCreated++;
|
||||
@ -1546,6 +1608,22 @@ public class WikiProcessingService {
|
||||
Long kbId = kb.getId();
|
||||
Long rawId = raw.getId();
|
||||
|
||||
// Derive the slug deterministically from the title rather than trusting
|
||||
// the model-supplied one. A model-minted slug romanizes inconsistently
|
||||
// across runs (the same concept lands under different spellings) and
|
||||
// forces every [[...]] reference to guess a transliteration that often
|
||||
// misses — surfacing as broken links on a freshly imported KB. Title is
|
||||
// already the canonical concept identity used for dedup below, so keying
|
||||
// the slug off it keeps the stored slug and the human-meaningful title
|
||||
// from ever drifting apart, and makes [[Title]] references resolve. Falls
|
||||
// back to the supplied slug only when the title yields no usable slug.
|
||||
if (title != null && !title.isBlank()) {
|
||||
String derivedSlug = WikiPageService.toSlug(title);
|
||||
if (derivedSlug != null && !derivedSlug.isBlank()) {
|
||||
slug = derivedSlug;
|
||||
}
|
||||
}
|
||||
|
||||
// Refuse to materialize a page, or merge into an existing one, for a
|
||||
// raw the user just deleted. This prevents pages whose source_raw_ids
|
||||
// point at a tombstoned row.
|
||||
|
||||
@ -0,0 +1,12 @@
|
||||
-- V147: Page aliases — alternate concept names a page also covers.
|
||||
--
|
||||
-- aliases JSON array of alternate names (concepts) that this page covers but
|
||||
-- that did not become standalone pages — e.g. a "细胞器术语辨析"
|
||||
-- discrimination page covers 叶绿体 / 线粒体 / 高尔基体. The
|
||||
-- post-ingestion link reconciler consults this so a [[叶绿体]]
|
||||
-- reference written by another page is rewritten to point at the
|
||||
-- covering page ([[细胞器术语辨析|叶绿体]]) instead of dangling as a
|
||||
-- broken link. NULL or empty array means the page declares no extra
|
||||
-- names. Distinct from the title (the page's primary identity) and
|
||||
-- outgoing_links (targets this page links out to).
|
||||
ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS aliases TEXT DEFAULT NULL;
|
||||
@ -0,0 +1,5 @@
|
||||
-- V147: Page aliases — KingbaseES dialect.
|
||||
--
|
||||
-- See h2/V147__wiki_page_aliases.sql for column semantics.
|
||||
-- KingbaseES (PostgreSQL) supports ADD COLUMN IF NOT EXISTS natively.
|
||||
ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS aliases TEXT DEFAULT NULL;
|
||||
@ -0,0 +1,15 @@
|
||||
-- V147: Page aliases — MySQL dialect.
|
||||
--
|
||||
-- See h2/V147__wiki_page_aliases.sql for column semantics. MySQL needs an
|
||||
-- INFORMATION_SCHEMA guard because ADD COLUMN IF NOT EXISTS is unavailable on
|
||||
-- the older 8.0.x versions the deploy targets.
|
||||
SET @col_exists := (
|
||||
SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'mate_wiki_page'
|
||||
AND COLUMN_NAME = 'aliases'
|
||||
);
|
||||
SET @stmt := IF(@col_exists = 0,
|
||||
'ALTER TABLE mate_wiki_page ADD COLUMN aliases JSON DEFAULT NULL COMMENT ''Alternate concept names this page also covers''',
|
||||
'SELECT 1');
|
||||
PREPARE s FROM @stmt; EXECUTE s; DEALLOCATE PREPARE s;
|
||||
@ -32,10 +32,10 @@
|
||||
|
||||
## slug 规范（仅用于 `key_concepts` 中的建议 slug）
|
||||
|
||||
- 多音节中文词按整词分组拼音，不要按字一隔
|
||||
- ✅ `zhongyao-qiqing-peiwu`（中药 / 七情 / 配伍）
|
||||
- ❌ `zhong-yao-qi-qing-pei-wu`
|
||||
- 小写字母 + 连字符，无空格
|
||||
- **直接用概念名本身**：中文保留中文（**不要转拼音**），英文小写、空格转连字符
|
||||
- ✅ `光合作用`、`energy-metabolism`
|
||||
- ❌ `guanghe-zuoyong`（不要转拼音）
|
||||
- slug 与概念名用词保持一致，无空格
|
||||
|
||||
## 关键纪律
|
||||
|
||||
|
||||
@ -24,12 +24,12 @@
|
||||
## 链接（**单一契约，必须严格遵守**）
|
||||
|
||||
- 只允许两种形态：
|
||||
- `[[slug]]` —— 显示文本默认为目标页标题
|
||||
- `[[slug|显示文本]]` —— 显示文本自定义
|
||||
- slug **必须**来自以下两类来源之一：
|
||||
- **已有 Wiki 页面索引**（user prompt 中列出）—— 这些链接是**强保证**，slug 100% 可用
|
||||
- **本批次同时创建的页面**（即 `pages_to_create` 数组中的 slug）—— 这类链接**不保证成功**：本批次中的页面可能因去重 / 合并 / 失败而最终未落库，导致链接转为死链；这是预期行为，系统会在写入后由 lint 标记并由人工修复
|
||||
- 禁止发明上述两类来源之外的 slug；禁止写 `[[页面标题]]` 形态 —— 系统按 slug 严格匹配，写标题会被识别为死链
|
||||
- `[[目标]]` —— 显示文本默认为目标页标题
|
||||
- `[[目标|显示文本]]` —— 显示文本自定义
|
||||
- `目标`可以写页面的 **slug 或 title**（系统两者都能解析），且**必须**来自以下两类来源之一：
|
||||
- **已有 Wiki 页面索引**（user prompt 中列出）—— 这些链接是**强保证**，目标 100% 可用
|
||||
- **本批次同时创建的页面**（即 `pages_to_create` 数组中的 slug / title）—— 这类链接**不保证成功**：本批次中的页面可能因去重 / 合并 / 失败而最终未落库，导致链接转为死链；这是预期行为，系统会在写入后由 lint 标记并由人工修复
|
||||
- 禁止链接到上述两类来源之外的目标——不要发明不存在的页面
|
||||
|
||||
## 输出格式（严格遵守）
|
||||
|
||||
@ -53,6 +53,7 @@
|
||||
- `page_type`：页面类型，从下面"允许的页面类型"列表中选一个；都不合适时选 concept。
|
||||
- `metadata`（可选）：与所选 page_type 对应的结构化字段对象，只输出该类型声明的字段（带"required metadata"标注的字段应尽量补全）。
|
||||
- `depends_on`（可选，仅经验层页面需要）：本页所依赖的**事实层页面 slug 数组**。经验层(如 analysis/pattern/regime)必须列出其结论所基于的事实页 slug;事实层页面留空或不输出。
|
||||
- `aliases`（可选，强烈建议用于"辨析/综合"类页面）：本页**虽然讲到、但没有单独成页**的细粒度概念名数组。例如一个「细胞器术语辨析」页同时讲了叶绿体、线粒体、高尔基体，却不会为它们各建一页，就写 `"aliases":["叶绿体","线粒体","高尔基体"]`。这样别处写的 `[[叶绿体]]` 会被系统自动指向本页，而不是变成死链。只列**本页确实充分讲解**的概念；不要列本页标题本身，也不要列已经独立成页的概念。
|
||||
|
||||
允许的页面类型：
|
||||
{allowed_page_types}
|
||||
|
||||
@ -30,13 +30,11 @@
|
||||
## metadata 格式（仅 `create` 数组使用）
|
||||
|
||||
每条 metadata 包含三个字段：
|
||||
- `slug`：URL 安全的标识符（小写字母 + 连字符）。
|
||||
- **多音节中文词的拼音必须按整词分组、不要按字隔开**。
|
||||
- ✅ 正确：`shennong-bencao-jing`（神农 / 本草 / 经 三个词）
|
||||
- ❌ 错误：`shen-nong-ben-cao-jing`（按字一隔，会被识别为另一概念）
|
||||
- ✅ 正确：`zhongyao-qiqing-peiwu`（中药 / 七情 / 配伍）
|
||||
- ❌ 错误：`zhong-yao-qi-qing-pei-wu`
|
||||
- **同一概念在不同段落必须用同一 slug**：选定一个 slug 就坚持用，不要换写法。
|
||||
- `slug`：页面标识符，**直接用概念名本身**——中文保留中文（**不要转拼音 / 罗马字**），英文小写、空格转连字符、去掉标点符号。
|
||||
- ✅ 正确：`光合作用`、`神农本草经`、`energy-metabolism`
|
||||
- ❌ 错误：`guanghe-zuoyong`、`shennong-bencao-jing`（不要转拼音）
|
||||
- slug 应与 `title` 用词一致；系统会按标题自动规范化 slug，所以**保持 slug 与 title 一致**即可。
|
||||
- **同一概念在不同段落必须用同一写法**：选定后就坚持用，不要换写法。
|
||||
- `title`：人类可读的页面标题
|
||||
- `summary`：一段话简短摘要（一两句话即可，让"单页生成助手"知道这一页要写什么）
|
||||
|
||||
|
||||
@ -0,0 +1,188 @@
|
||||
package vip.mate.wiki.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import vip.mate.wiki.model.WikiPageEntity;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Coverage for broken-link computation, focused on the slug-or-title
|
||||
* resolution contract that {@link WikiLinkService#resolvableTargetKeys} and
|
||||
* {@link WikiLinkService#computeBrokenLinks} jointly enforce.
|
||||
*
|
||||
* <p>The page viewer's {@code resolveWikilink} treats a {@code [[target]]} as
|
||||
* a hit when it matches either an existing page slug OR an existing page title.
|
||||
* The lint must agree, otherwise a {@code [[Page Title]]} reference to a real
|
||||
* page is rendered as a working link yet reported as a broken link — the
|
||||
* false-positive class this test pins down. The mismatch is most visible when
|
||||
* slugs are transliterated (a CJK title stored under a pinyin slug).
|
||||
*/
|
||||
class WikiLinkServiceBrokenLinkTest {
|
||||
|
||||
private final WikiLinkService svc = new WikiLinkService(new ObjectMapper());
|
||||
|
||||
private static WikiPageEntity page(String slug, String title) {
|
||||
WikiPageEntity p = new WikiPageEntity();
|
||||
p.setSlug(slug);
|
||||
p.setTitle(title);
|
||||
return p;
|
||||
}
|
||||
|
||||
/** Mirrors the real KB in issue #333: Chinese titles, pinyin slugs. */
|
||||
private List<WikiPageEntity> cjkKb() {
|
||||
return List.of(
|
||||
page("guanghe-zuoyong", "光合作用"),
|
||||
page("xianliti", "线粒体"),
|
||||
page("yelvti", "叶绿体"),
|
||||
page("nengliang-daixie", "能量代谢"),
|
||||
page("energy-metabolism", "Energy Metabolism"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvableKeysCarryBothSlugAndTitle() {
|
||||
Set<String> keys = svc.resolvableTargetKeys(cjkKb());
|
||||
assertTrue(keys.contains("guanghe-zuoyong"), "slug must be a key");
|
||||
assertTrue(keys.contains("光合作用"), "title must be a key");
|
||||
assertTrue(keys.contains("energy-metabolism"));
|
||||
assertTrue(keys.contains("energy metabolism"), "title lowercased, spaces kept");
|
||||
}
|
||||
|
||||
@Test
|
||||
void titleFormLinkToExistingCjkPageIsNotBroken() {
|
||||
// The model naturally writes the readable title, not the pinyin slug.
|
||||
String content = "参见 [[光合作用]] 与 [[线粒体]]。";
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertTrue(a.brokenLinks().isEmpty(),
|
||||
"title-form links to existing pages must resolve, got: " + a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void slugFormLinkIsNotBroken() {
|
||||
String content = "See [[guanghe-zuoyong]] and [[energy-metabolism]].";
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertTrue(a.brokenLinks().isEmpty(), "slug-form links must resolve, got: " + a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void englishTitleWithSpacesResolvesAgainstTitleNotSlug() {
|
||||
// slug is "energy-metabolism" (dashed); the title has a space. Only the
|
||||
// title key can match the [[Energy Metabolism]] target.
|
||||
String content = "Read [[Energy Metabolism]] first.";
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertTrue(a.brokenLinks().isEmpty(), "title-with-space must resolve, got: " + a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aliasFormResolvesOnTitleTarget() {
|
||||
String content = "更多见 [[线粒体|线粒体别名]]。";
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertTrue(a.brokenLinks().isEmpty(), "aliased title link must resolve, got: " + a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void genuinelyMissingTargetIsStillBroken() {
|
||||
String content = "悬挂引用 [[不存在的概念XYZ]] 应当被标记。";
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertEquals(List.of("不存在的概念xyz"), a.brokenLinks(),
|
||||
"a target matching no slug and no title must be broken");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixedContentReportsOnlyTheGenuineBreak() {
|
||||
// Reproduces the issue scenario across several markdown shapes: only the
|
||||
// hallucinated target is broken; the four title-form links resolve.
|
||||
String content = String.join("\n",
|
||||
"# 标题 [[能量代谢]]",
|
||||
"段落：[[光合作用]] 与 [[guanghe-zuoyong]] 指向同一页。",
|
||||
"- 列表：[[线粒体|别名]]",
|
||||
"> 引用：[[叶绿体]]",
|
||||
"悬挂：[[未知页面ABC]]");
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertEquals(List.of("未知页面abc"), a.brokenLinks(),
|
||||
"only the hallucinated target is broken, got: " + a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void linksInsideCodeAreNeitherOutgoingNorBroken() {
|
||||
String content = String.join("\n",
|
||||
"行内 `[[光合作用]]` 不计入。",
|
||||
"```",
|
||||
"围栏 [[不存在XYZ]] 不计入",
|
||||
"```");
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(cjkKb()));
|
||||
assertTrue(a.outgoingLinks().isEmpty(), "code-block links must be ignored, got: " + a.outgoingLinks());
|
||||
assertTrue(a.brokenLinks().isEmpty(), "code-block links must not be broken, got: " + a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void emptyKbMakesEveryLinkBroken() {
|
||||
String content = "[[anything]]";
|
||||
WikiLinkService.LinkAnalysis a = svc.analyze(content, svc.resolvableTargetKeys(List.of()));
|
||||
assertEquals(List.of("anything"), a.brokenLinks());
|
||||
}
|
||||
|
||||
@Test
|
||||
void computeBrokenLinksMatchesAgainstTitleKeys() {
|
||||
Set<String> keys = svc.resolvableTargetKeys(cjkKb());
|
||||
// direct unit on the predicate: "光合作用" is a title key → resolvable
|
||||
assertFalse(svc.computeBrokenLinks(Set.of("光合作用"), keys).contains("光合作用"));
|
||||
assertTrue(svc.computeBrokenLinks(Set.of("missing"), keys).contains("missing"));
|
||||
}
|
||||
|
||||
// ── reconcileLinks: post-ingestion redirect / demote of dangling links ──
|
||||
|
||||
/** Reconciler mirroring the production rule: keep resolvable, redirect via
|
||||
* alias to a covering page, else demote to plain text. */
|
||||
private WikiLinkService.LinkReconciler reconciler(Set<String> resolvable,
|
||||
java.util.Map<String, String> aliasToSlug) {
|
||||
return (target, alias) -> {
|
||||
String key = target.trim().toLowerCase();
|
||||
if (resolvable.contains(key)) return null;
|
||||
String display = (alias != null && !alias.isBlank()) ? alias : target;
|
||||
String cover = aliasToSlug.get(key);
|
||||
if (cover != null) return "[[" + cover + "|" + display + "]]";
|
||||
return display;
|
||||
};
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcileKeepsResolvableLinks() {
|
||||
Set<String> resolvable = Set.of("光合作用", "guanghe-zuoyong");
|
||||
String in = "见 [[光合作用]] 与 [[guanghe-zuoyong]]。";
|
||||
String out = svc.reconcileLinks(in, reconciler(resolvable, java.util.Map.of()));
|
||||
assertEquals(in, out, "resolvable links must be left untouched");
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcileRedirectsAliasToCoveringPage() {
|
||||
Set<String> resolvable = Set.of("细胞器术语辨析");
|
||||
var aliasMap = java.util.Map.of("叶绿体", "细胞器术语辨析");
|
||||
String out = svc.reconcileLinks("叶绿体见 [[叶绿体]]。", reconciler(resolvable, aliasMap));
|
||||
assertEquals("叶绿体见 [[细胞器术语辨析|叶绿体]]。", out,
|
||||
"an alias-covered link must redirect to the covering page, keeping a readable label");
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcileDemotesUncoveredDanglingToPlainText() {
|
||||
Set<String> resolvable = Set.of("细胞器术语辨析");
|
||||
String out = svc.reconcileLinks("讲到 [[不存在的概念]] 和 [[未知|别名显示]]。",
|
||||
reconciler(resolvable, java.util.Map.of()));
|
||||
assertEquals("讲到 不存在的概念 和 别名显示。", out,
|
||||
"uncovered links demote to plain text, honouring the alias display when present");
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcileLeavesCodeBlocksUntouched() {
|
||||
String in = "正文 [[不存在]] 降级。\n\n```\n代码里的 [[不存在]] 保留\n```\n行内 `[[不存在]]` 保留。";
|
||||
String out = svc.reconcileLinks(in, reconciler(Set.of(), java.util.Map.of()));
|
||||
assertTrue(out.startsWith("正文 不存在 降级。"), "narrative link demoted, got: " + out);
|
||||
assertTrue(out.contains("代码里的 [[不存在]] 保留"), "fenced code must be preserved");
|
||||
assertTrue(out.contains("`[[不存在]]`"), "inline code must be preserved");
|
||||
}
|
||||
}
|
||||
@ -93,7 +93,10 @@ const formattedTimestamp = computed(() => {
|
||||
async function onScan() {
|
||||
if (!store.currentKB) return
|
||||
try {
|
||||
await store.startBrokenLinksScan(Number(store.currentKB.id))
|
||||
// Pass the raw Snowflake id through untouched — Number()/parseInt() would
|
||||
// truncate the 19-digit id past Number.MAX_SAFE_INTEGER and scan a KB that
|
||||
// doesn't exist (the request 404s, so "rescan" silently does nothing).
|
||||
await store.startBrokenLinksScan(store.currentKB.id)
|
||||
} catch (e: any) {
|
||||
console.error('[Wiki] scan failed', e)
|
||||
}
|
||||
|
||||
@ -65,7 +65,9 @@ const report = computed(() => store.brokenLinksReport)
|
||||
|
||||
async function onOpenPage(slug: string) {
|
||||
if (!store.currentKB) return
|
||||
await store.loadPage(Number(store.currentKB.id), slug)
|
||||
// Keep the Snowflake id as-is; Number() would truncate it and open a page
|
||||
// lookup against a non-existent KB (per the project ID precision convention).
|
||||
await store.loadPage(store.currentKB.id, slug)
|
||||
emit('close')
|
||||
}
|
||||
</script>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user