diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCompileService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCompileService.java index 787545f7..09adc5a4 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCompileService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCompileService.java @@ -51,6 +51,13 @@ public class WikiCompileService { @Autowired(required = false) private WikiModelRoutingService modelRoutingService; + /** RFC-051 PR-2b/2c: optional overview rebuilder + log appender. */ + @Autowired(required = false) + private WikiOverviewService overviewService; + + @Autowired(required = false) + private WikiLogService logService; + public record CompileResult(Long pageId, String slug, String title, int evidenceChunkCount, boolean created) {} @@ -157,6 +164,15 @@ public class WikiCompileService { log.info("[WikiCompile] {} page slug={} title='{}' from {} evidence chunks (kbId={})", created ? "Created" : "Updated", resolvedSlug, title, evidenceChunkIds.size(), kbId); + + // RFC-051 PR-2c: log every compile attempt; PR-2b: refresh overview. + if (logService != null) { + logService.append(kbId, WikiLogService.EventType.COMPILE, + (created ? "compiled new page " : "recompiled page ") + resolvedSlug + + " · topic='" + topic + "' · " + evidenceChunkIds.size() + " evidence chunks"); + } + if (overviewService != null) overviewService.rebuild(kbId); + return new CompileResult(persisted.getId(), resolvedSlug, title, evidenceChunkIds.size(), created); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLogService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLogService.java new file mode 100644 index 00000000..39db8f40 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLogService.java @@ -0,0 +1,114 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageMapper; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * RFC-051 PR-2c: append-friendly activity log for the {@code log} system page. + *

+ * The log page is intentionally just a Markdown document the system writes + * into. Each entry is a single bullet under a {@code ## YYYY-MM-DD } + * section so the page stays human-readable. We don't ship a separate + * activity table yet — when the volume justifies it (RFC §7.3 calls it + * {@code mate_wiki_activity_log}), this service is the only place that + * needs to swap storage backends. + * + *

Trim policy

+ * Because the log page is a single Markdown blob, we cap it at 10 000 + * characters. Once exceeded, the oldest sections are dropped from the top + * (right after the {@code # Log} heading) until the page is back under the + * cap. This keeps the page readable and bounded. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiLogService { + + private static final int MAX_LOG_CHARS = 10_000; + private static final DateTimeFormatter DAY = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter TIME = DateTimeFormatter.ofPattern("HH:mm"); + + private final WikiPageService pageService; + private final WikiPageMapper pageMapper; + + /** Common entry types so callers don't have to invent strings. */ + public enum EventType { + INGEST, COMPILE, EDIT, ARCHIVE, REINDEX + } + + /** + * Append a single bullet under today's section header. Idempotent in the + * sense that two simultaneous calls produce two bullets — never crashes, + * never overwrites prior content. + * + * @param kbId knowledge base id + * @param event high-level category + * @param body human-readable detail; rendered as Markdown after a hyphen + */ + public void append(Long kbId, EventType event, String body) { + if (kbId == null || event == null || body == null || body.isBlank()) return; + WikiPageEntity log = pageService.getBySlug(kbId, WikiScaffoldService.LOG_SLUG); + if (log == null) { + // Caller is expected to ensure scaffold first; bail quietly. + WikiLogService.log.debug("[WikiLog] No log page for kbId={}, skipping append", kbId); + return; + } + try { + String existing = log.getContent() == null ? "# Log\n" : log.getContent(); + String today = LocalDate.now().format(DAY); + String time = LocalDateTime.now().format(TIME); + String bullet = "- " + time + " — " + body.replace("\n", " ").trim(); + String updated = appendBullet(existing, today, event.name().toLowerCase(), bullet); + if (updated.length() > MAX_LOG_CHARS) { + updated = trimOldest(updated, MAX_LOG_CHARS); + } + if (updated.equals(existing)) return; + log.setContent(updated); + pageMapper.updateById(log); + } catch (Exception e) { + WikiLogService.log.warn("[WikiLog] Append failed for kbId={}: {}", kbId, e.getMessage()); + } + } + + String appendBullet(String content, String today, String eventTag, String bullet) { + String header = "## " + today + " " + eventTag; + int idx = content.indexOf("\n" + header + "\n"); + if (idx >= 0) { + // Section exists — insert bullet at end of that section (right before the next "## "). + int sectionStart = idx + 1; + int sectionContentStart = content.indexOf('\n', sectionStart); + int nextSection = content.indexOf("\n## ", sectionContentStart); + int insertAt = nextSection < 0 ? content.length() : nextSection; + String before = content.substring(0, insertAt); + String after = content.substring(insertAt); + String prefix = before.endsWith("\n") ? before : before + "\n"; + return prefix + bullet + "\n" + (after.startsWith("\n") ? after : "\n" + after); + } + // New section. Insert at the top, right after the "# Log" heading. + int firstSection = content.indexOf("\n## "); + String section = "\n" + header + "\n\n" + bullet + "\n"; + if (firstSection < 0) { + String tail = content.endsWith("\n") ? content : content + "\n"; + return tail + section; + } + return content.substring(0, firstSection) + section + content.substring(firstSection); + } + + String trimOldest(String content, int cap) { + // Drop the oldest "## ..." section (which is at the bottom under our prepend + // strategy) until we're under the cap. + while (content.length() > cap) { + int lastHeading = content.lastIndexOf("\n## "); + if (lastHeading < 0) break; + content = content.substring(0, lastHeading) + "\n"; + } + return content; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiOverviewService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiOverviewService.java new file mode 100644 index 00000000..73a25bbe --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiOverviewService.java @@ -0,0 +1,148 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.model.WikiChunkEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.repository.WikiChunkMapper; +import vip.mate.wiki.repository.WikiPageMapper; +import vip.mate.wiki.repository.WikiRawMaterialMapper; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +/** + * RFC-051 PR-2b: deterministic overview rebuilder. + *

+ * The {@code overview} system page wraps an auto-generated stats block + * inside marker comments: + * + * 

+ * <!-- mate:overview:v1:start -->
+ *   ... rebuilt block ...
+ * <!-- mate:overview:v1:end -->
+ *
+ * + * Anything outside the markers is user-authored prose and is preserved + * verbatim. Inside the markers, this service rewrites a small set of stats + * derived directly from the database — no LLM, no judgement calls. + *

+ * Hook points: {@code WikiProcessingService.processRawMaterial} on success + * and {@code WikiCompileService.compilePage} after a compile result. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiOverviewService { + + public static final String MARKER_START = ""; + public static final String MARKER_END = ""; + + private final WikiPageService pageService; + private final WikiPageMapper pageMapper; + private final WikiRawMaterialMapper rawMapper; + private final WikiChunkMapper chunkMapper; + + private static final DateTimeFormatter ISO = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"); + + /** + * Rebuild the marker region of the overview page for {@code kbId}. + * No-op when the overview page is missing — call + * {@code WikiScaffoldService.ensureScaffold} first. + */ + public void rebuild(Long kbId) { + if (kbId == null) return; + WikiPageEntity overview = pageService.getBySlug(kbId, WikiScaffoldService.OVERVIEW_SLUG); + if (overview == null) { + log.debug("[WikiOverview] No overview page for kbId={}, skipping rebuild", kbId); + return; + } + try { + String stats = computeStatsBlock(kbId); + String rewritten = spliceMarkerRegion(overview.getContent(), stats); + if (rewritten.equals(overview.getContent())) return; + overview.setContent(rewritten); + pageMapper.updateById(overview); + log.debug("[WikiOverview] Refreshed overview for kbId={}", kbId); + } catch (Exception e) { + log.warn("[WikiOverview] Rebuild failed for kbId={}: {}", kbId, e.getMessage()); + } + } + + private String computeStatsBlock(Long kbId) { + long rawCount = rawMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiRawMaterialEntity::getKbId, kbId)); + // Page count excludes system pages (overview / log themselves). + long pageCount = pageMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiPageEntity::getKbId, kbId) + .ne(WikiPageEntity::getPageType, WikiScaffoldService.SYSTEM_PAGE_TYPE)); + long chunkCount = chunkMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiChunkEntity::getKbId, kbId)); + long embeddedChunks = chunkMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiChunkEntity::getKbId, kbId) + .isNotNull(WikiChunkEntity::getEmbedding)); + long pagesWithLinks = pageMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiPageEntity::getKbId, kbId) + .ne(WikiPageEntity::getPageType, WikiScaffoldService.SYSTEM_PAGE_TYPE) + .isNotNull(WikiPageEntity::getOutgoingLinks) + .ne(WikiPageEntity::getOutgoingLinks, "[]") + .ne(WikiPageEntity::getOutgoingLinks, "")); + WikiRawMaterialEntity latest = rawMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiRawMaterialEntity::getKbId, kbId) + .isNotNull(WikiRawMaterialEntity::getLastProcessedAt) + .orderByDesc(WikiRawMaterialEntity::getLastProcessedAt) + .last("LIMIT 1")); + String lastIngest = (latest != null && latest.getLastProcessedAt() != null) + ? latest.getLastProcessedAt().format(ISO) : "—"; + + long embedPct = chunkCount == 0 ? 0 : Math.round(100.0 * embeddedChunks / chunkCount); + long linkPct = pageCount == 0 ? 0 : Math.round(100.0 * pagesWithLinks / pageCount); + + return """ + ## Scope + + - Sources: %d + - Wiki pages: %d + - Chunks: %d + - Last ingest: %s + + ## Coverage + + - Embedding coverage: %d / %d (%d%%) + - Pages with wikilinks: %d / %d (%d%%) + """.formatted( + rawCount, pageCount, chunkCount, lastIngest, + embeddedChunks, chunkCount, embedPct, + pagesWithLinks, pageCount, linkPct); + } + + String spliceMarkerRegion(String content, String newBlock) { + if (content == null || content.isEmpty()) { + // No prior overview — synthesize one with both markers. + return "# Overview\n\n" + MARKER_START + "\n" + newBlock.trim() + "\n" + MARKER_END + "\n"; + } + int start = content.indexOf(MARKER_START); + int end = content.indexOf(MARKER_END); + String generated = MARKER_START + "\n" + newBlock.trim() + "\n" + MARKER_END; + if (start < 0 || end < 0 || end < start) { + // Markers missing or scrambled — append the block at the end of the page. + String trimmed = content.endsWith("\n") ? content : content + "\n"; + return trimmed + "\n" + generated + "\n"; + } + // Replace the block (markers included) with the freshly generated one. + return content.substring(0, start) + + generated + + content.substring(end + MARKER_END.length()); + } + + LocalDateTime now() { return LocalDateTime.now(); } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index 3861e3d1..902ecd8c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -87,6 +87,13 @@ public class WikiProcessingService { @org.springframework.beans.factory.annotation.Autowired(required = false) private vip.mate.wiki.job.WikiModelRoutingService modelRoutingService; + /** RFC-051 PR-2b/2c: optional overview rebuilder + log appender. */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private WikiOverviewService overviewService; + + @org.springframework.beans.factory.annotation.Autowired(required = false) + private WikiLogService logService; + /** Parallel chunk / material processing executor (JDK 21 virtual threads) */ public static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor(); @@ -334,6 +341,19 @@ public class WikiProcessingService { } catch (Exception ignored) {} } + // RFC-051 PR-2c: log every non-failed eager ingest. Failures already get a + // RAW_FAILED broadcast and an error message in the raw row. + if (logService != null && !"failed".equals(finalStatus)) { + logService.append(kb.getId(), WikiLogService.EventType.INGEST, + "eager " + finalStatus + " raw=" + rawId + + " · " + totalPages + " pages · " + totalChunks + " chunks"); + } + // RFC-051 PR-2b: refresh overview stats whenever a raw lands in a terminal state + // (completed or partial). Failures don't shift the stats meaningfully. + if (overviewService != null && !"failed".equals(finalStatus)) { + overviewService.rebuild(kb.getId()); + } + log.info("[Wiki] Processing completed for raw={}, kbId={}, generatedPages={}, totalPages={}", rawId, kb.getId(), totalPages, pageCount); @@ -1824,6 +1844,14 @@ public class WikiProcessingService { "kbPageCount", pageCount, "totalChunks", totalChunks)); + // RFC-051 PR-2c: write an activity log entry. + if (logService != null) { + logService.append(kbId, WikiLogService.EventType.INGEST, + "lazy ingest raw=" + rawId + " · " + totalChunks + " chunks"); + } + // RFC-051 PR-2b: refresh overview stats. + if (overviewService != null) overviewService.rebuild(kbId); + log.info("[Wiki] Lazy processing completed for raw={}, kbId={}, chunks={}", rawId, kbId, totalChunks); } catch (Exception e) {