feat(wiki): PR-1c preprocessor + chunk metadata + search exposure

mateclaw-server/pom.xml

@@ -291,6 +291,20 @@
             <version>5.4.1</version>
         </dependency>
 
+        <!-- ===== jsoup (HTML cleanup for Wiki ingest, RFC-051 PR-1c) ===== -->
+        <!--
+            Used by WikiContentNormalizer to strip nav/footer/script/style/aside
+            and ad-class nodes from URL/HTML uploads before chunking. Small
+            (~430KB), no transitive deps, JVM-only — safe for the desktop bundle.
+            Tika is intentionally not pulled in: DocumentExtractTool already
+            covers PDF/Office via pdftotext, pdfplumber, and Java fallbacks.
+        -->
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.18.3</version>
+        </dependency>
+
         <!-- ===== Database Migration (Flyway) ===== -->
         <dependency>
             <groupId>org.flywaydb</groupId>

mateclaw-server/src/main/java/vip/mate/wiki/service/DocumentPreprocessService.java

@@ -0,0 +1,261 @@
+package vip.mate.wiki.service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import vip.mate.wiki.WikiProperties;
+import vip.mate.wiki.dto.WikiChunkDraft;
+import vip.mate.wiki.model.WikiRawMaterialEntity;
+
+import java.util.ArrayList;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * RFC-051 PR-1c: produces {@link WikiChunkDraft}s from a raw material's
+ * extracted text, populating the structural metadata columns added in V39
+ * ({@code page_number}, {@code token_count}, {@code header_breadcrumb},
+ * {@code source_section}).
+ * <p>
+ * Strategy is intentionally lightweight:
+ * <ul>
+ *   <li>Pre-scan the normalized text once to build sparse maps from char
+ *       offset → header breadcrumb and char offset → PDF page number.</li>
+ *   <li>Reuse the existing sentence-boundary chunker (passed in as an
+ *       interface from {@link WikiProcessingService}).</li>
+ *   <li>For each chunk, look up the breadcrumb and page that were active at
+ *       the chunk's {@code startOffset}.</li>
+ *   <li>Estimate token count as {@code ceil(charCount / 4.0)}, matching the
+ *       backfill heuristic so eager-vs-lazy and old-vs-new chunks share one
+ *       scale until a real tokenizer lands in a follow-up.</li>
+ * </ul>
+ *
+ * No Tika integration: existing {@code DocumentExtractTool} already covers
+ * the binary formats and the metadata we need for chunks lives inside the
+ * extracted text, not in document properties.
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class DocumentPreprocessService {
+
+    private final WikiContentNormalizer normalizer;
+    private final WikiProperties properties;
+
+    // Markdown ATX heading: 1-6 leading '#', then a space, then heading text. Setext-style
+    // (=== / ---) headings are not handled — DocumentExtractTool emits ATX for everything.
+    private static final Pattern MARKDOWN_HEADING = Pattern.compile("^(#{1,6})\\s+(.+?)\\s*$");
+
+    // DocumentExtractTool inserts page markers for PDF text extraction. Match a line like
+    // `--- Page 12 ---` (case-insensitive, tolerant of extra whitespace). PPTX uses
+    // `--- Slide 4 ---`; treat both as a page-number-like signal.
+    private static final Pattern PAGE_MARKER = Pattern.compile(
+            "^\\s*-{2,}\\s*(?:Page|Slide)\\s+(\\d+)\\s*-{2,}\\s*$",
+            Pattern.CASE_INSENSITIVE);
+
+    /**
+     * Chunker SPI: pluggable so a unit test (or PR-1c follow-up using a smarter
+     * splitter) can pass a different boundary algorithm without dragging the
+     * whole {@link WikiProcessingService} into preprocessing.
+     */
+    public interface Chunker {
+        List<int[]> split(String text);
+    }
+
+    /**
+     * Build chunk drafts from already-normalized text. Caller is expected to
+     * have routed the raw text through {@link WikiContentNormalizer} first
+     * (see {@link #preprocess(WikiRawMaterialEntity, String, Chunker)}).
+     */
+    public List<WikiChunkDraft> buildDrafts(String text, Chunker chunker) {
+        if (text == null || text.isBlank()) return List.of();
+
+        HeaderIndex headerIndex = HeaderIndex.scan(text);
+        PageIndex pageIndex = PageIndex.scan(text);
+
+        List<int[]> windows = chunker.split(text);
+        List<WikiChunkDraft> drafts = new ArrayList<>(windows.size());
+        for (int[] win : windows) {
+            int start = win[0];
+            int end = win[1];
+            String content = text.substring(start, end);
+            int chars = content.length();
+            int tokens = (int) Math.ceil(chars / 4.0);
+
+            String breadcrumb = headerIndex.breadcrumbAt(start);
+            String section = headerIndex.sectionAt(start); // last segment of breadcrumb
+            Integer page = pageIndex.pageAt(start);
+
+            drafts.add(new WikiChunkDraft(content, start, end, page, tokens, breadcrumb, section));
+        }
+        return drafts;
+    }
+
+    /**
+     * Convenience entry: normalize, then chunk, then attach metadata. Intended
+     * for the lazy ingest branch of {@link WikiProcessingService}.
+     */
+    public List<WikiChunkDraft> preprocess(WikiRawMaterialEntity raw, String extractedText, Chunker chunker) {
+        String normalized = normalizer.normalize(raw == null ? null : raw.getSourceType(), extractedText);
+        return buildDrafts(normalized, chunker);
+    }
+
+    /** Returns the post-normalization text alone, for callers that need to chunk separately. */
+    public String normalize(WikiRawMaterialEntity raw, String extractedText) {
+        return normalizer.normalize(raw == null ? null : raw.getSourceType(), extractedText);
+    }
+
+    @SuppressWarnings("unused") // exposed for future callers; keeps the property accessible.
+    public WikiProperties properties() {
+        return properties;
+    }
+
+    // ────────────────────────────────────────────────────────────────────────
+    // Header index — precomputed list of (lineStartOffset, breadcrumb, deepestHeading)
+    // sorted by offset. Lookup uses binary search.
+    // ────────────────────────────────────────────────────────────────────────
+
+    private static final class HeaderIndex {
+        private final int[] starts;
+        private final String[] breadcrumbs;
+        private final String[] deepest;
+
+        private HeaderIndex(int[] starts, String[] breadcrumbs, String[] deepest) {
+            this.starts = starts;
+            this.breadcrumbs = breadcrumbs;
+            this.deepest = deepest;
+        }
+
+        static HeaderIndex scan(String text) {
+            List<int[]> offsets = new ArrayList<>();
+            List<String> crumbs = new ArrayList<>();
+            List<String> last = new ArrayList<>();
+            Deque<String[]> stack = new ArrayDeque<>(); // each entry: [level, title]
+
+            int n = text.length();
+            int i = 0;
+            while (i < n) {
+                int lineEnd = text.indexOf('\n', i);
+                if (lineEnd < 0) lineEnd = n;
+                String line = text.substring(i, lineEnd);
+                Matcher m = MARKDOWN_HEADING.matcher(line);
+                if (m.matches()) {
+                    int level = m.group(1).length();
+                    String title = m.group(2).trim();
+                    // Pop deeper-or-equal levels.
+                    while (!stack.isEmpty() && Integer.parseInt(stack.peek()[0]) >= level) {
+                        stack.pop();
+                    }
+                    stack.push(new String[]{Integer.toString(level), title});
+                    String breadcrumb = buildBreadcrumb(stack);
+                    offsets.add(new int[]{i});
+                    crumbs.add(breadcrumb);
+                    last.add(title);
+                }
+                i = lineEnd + 1;
+            }
+
+            int[] starts = new int[offsets.size()];
+            String[] breadcrumbs = new String[offsets.size()];
+            String[] deepest = new String[offsets.size()];
+            for (int k = 0; k < offsets.size(); k++) {
+                starts[k] = offsets.get(k)[0];
+                breadcrumbs[k] = crumbs.get(k);
+                deepest[k] = last.get(k);
+            }
+            return new HeaderIndex(starts, breadcrumbs, deepest);
+        }
+
+        private static String buildBreadcrumb(Deque<String[]> stack) {
+            // Stack iterates top-first; build a root-first list for display.
+            List<String[]> ordered = new ArrayList<>(stack);
+            StringBuilder sb = new StringBuilder();
+            for (int k = ordered.size() - 1; k >= 0; k--) {
+                if (sb.length() > 0) sb.append(" / ");
+                sb.append(ordered.get(k)[1]);
+            }
+            // Cap at the column width of the DB column (1024).
+            return sb.length() > 1000 ? sb.substring(0, 1000) : sb.toString();
+        }
+
+        String breadcrumbAt(int offset) {
+            int idx = floorIndex(offset);
+            return idx < 0 ? null : breadcrumbs[idx];
+        }
+
+        String sectionAt(int offset) {
+            int idx = floorIndex(offset);
+            if (idx < 0) return null;
+            String s = deepest[idx];
+            return s != null && s.length() > 500 ? s.substring(0, 500) : s;
+        }
+
+        private int floorIndex(int offset) {
+            // Largest index with starts[idx] <= offset.
+            if (starts.length == 0 || offset < starts[0]) return -1;
+            int lo = 0, hi = starts.length - 1, ans = -1;
+            while (lo <= hi) {
+                int mid = (lo + hi) >>> 1;
+                if (starts[mid] <= offset) { ans = mid; lo = mid + 1; }
+                else hi = mid - 1;
+            }
+            return ans;
+        }
+    }
+
+    // ────────────────────────────────────────────────────────────────────────
+    // Page index — same shape as HeaderIndex but for "--- Page N ---" markers.
+    // ────────────────────────────────────────────────────────────────────────
+
+    private static final class PageIndex {
+        private final int[] starts;
+        private final int[] pages;
+
+        private PageIndex(int[] starts, int[] pages) {
+            this.starts = starts;
+            this.pages = pages;
+        }
+
+        static PageIndex scan(String text) {
+            List<int[]> matches = new ArrayList<>();
+            int n = text.length();
+            int i = 0;
+            while (i < n) {
+                int lineEnd = text.indexOf('\n', i);
+                if (lineEnd < 0) lineEnd = n;
+                String line = text.substring(i, lineEnd);
+                Matcher m = PAGE_MARKER.matcher(line);
+                if (m.matches()) {
+                    try {
+                        int page = Integer.parseInt(m.group(1));
+                        matches.add(new int[]{i, page});
+                    } catch (NumberFormatException ignored) {
+                        // overflow / not a number; ignore — page stays unknown for this region.
+                    }
+                }
+                i = lineEnd + 1;
+            }
+            int[] starts = new int[matches.size()];
+            int[] pages = new int[matches.size()];
+            for (int k = 0; k < matches.size(); k++) {
+                starts[k] = matches.get(k)[0];
+                pages[k] = matches.get(k)[1];
+            }
+            return new PageIndex(starts, pages);
+        }
+
+        Integer pageAt(int offset) {
+            if (starts.length == 0 || offset < starts[0]) return null;
+            int lo = 0, hi = starts.length - 1, ans = -1;
+            while (lo <= hi) {
+                int mid = (lo + hi) >>> 1;
+                if (starts[mid] <= offset) { ans = mid; lo = mid + 1; }
+                else hi = mid - 1;
+            }
+            return ans < 0 ? null : pages[ans];
+        }
+    }
+}

mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java

@@ -59,8 +59,19 @@ public class HybridRetriever {
 
     /**
      * Chunk-level search result (semantic search).
+     * <p>
+     * RFC-051 PR-1c: {@code pageNumber} and {@code headerBreadcrumb} are
+     * populated when the chunk has those columns set (lazy ingest with
+     * preprocessor on, or backfilled chunks). Both are nullable.
      */
-    public record ChunkHit(Long chunkId, Long rawId, String snippet, float score) {}
+    public record ChunkHit(Long chunkId, Long rawId, String snippet, float score,
+                            Integer pageNumber, String headerBreadcrumb) {
+
+        /** Backwards-compatible factory for callers that don't yet pass metadata. */
+        public ChunkHit(Long chunkId, Long rawId, String snippet, float score) {
+            this(chunkId, rawId, snippet, score, null, null);
+        }
+    }
 
     /**
      * RFC-032: Enhanced search returning PageSearchResult with snippet and matchedBy metadata.
@@ -152,7 +163,8 @@ public class HybridRetriever {
                     String snippet = c.getContent().length() > 300
                             ? c.getContent().substring(0, 300) + "..."
                             : c.getContent();
-                    return new ChunkHit(c.getId(), c.getRawId(), snippet, score);
+                    return new ChunkHit(c.getId(), c.getRawId(), snippet, score,
+                            c.getPageNumber(), c.getHeaderBreadcrumb());
                 })
                 .sorted(Comparator.comparingDouble(ChunkHit::score).reversed())
                 .limit(topK)

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java

@@ -0,0 +1,104 @@
+package vip.mate.wiki.service;
+
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.stereotype.Component;
+
+/**
+ * RFC-051 PR-1c: source-type-specific text normalization.
+ * <p>
+ * Goal is structural cleanup before chunking — strip noise from HTML,
+ * normalize line endings, trim duplicate whitespace — without touching the
+ * actual semantic content. Heading and page-marker extraction lives in
+ * {@link DocumentPreprocessService} because it needs the offsets of the
+ * surviving text, not just the text itself.
+ *
+ * <p>Tika is intentionally not pulled in: existing {@code DocumentExtractTool}
+ * already covers PDF/Office via pdftotext / pdfplumber / Java fallbacks.
+ */
+@Slf4j
+@Component
+public class WikiContentNormalizer {
+
+    private static final int MAX_HTML_LEN = 8 * 1024 * 1024; // 8 MB safety cap
+
+    /**
+     * Normalize raw text by source type. Returns the input unchanged when the
+     * source type is unknown so we never lose content silently.
+     *
+     * @param sourceType {@code WikiRawMaterialEntity.sourceType} value
+     *                   (text / pdf / docx / xlsx / pptx / url / paste / markdown)
+     * @param rawText    extracted text content
+     * @return cleaned text, never {@code null}
+     */
+    public String normalize(String sourceType, String rawText) {
+        if (rawText == null) return "";
+        String type = sourceType == null ? "" : sourceType.toLowerCase();
+        return switch (type) {
+            case "url", "html" -> normalizeHtml(rawText);
+            // PDF text from DocumentExtractTool may already contain "--- Page N ---"
+            // markers; we keep them so the preprocessor can map char offsets to pages.
+            case "pdf" -> collapseBlankLines(rawText);
+            case "docx", "pptx", "xlsx" -> collapseBlankLines(rawText);
+            case "markdown", "md", "text", "paste" -> collapseBlankLines(rawText);
+            default -> collapseBlankLines(rawText);
+        };
+    }
+
+    /**
+     * Strip nav/footer/script/style/aside and ad-like classes from HTML, then
+     * return readable text. Falls through to the raw input when the document
+     * is too large to parse safely or jsoup throws.
+     */
+    private String normalizeHtml(String rawHtml) {
+        if (rawHtml.length() > MAX_HTML_LEN) {
+            log.warn("[WikiContentNormalizer] HTML payload exceeds {} bytes, skipping cleanup", MAX_HTML_LEN);
+            return collapseBlankLines(rawHtml);
+        }
+        try {
+            Document doc = Jsoup.parse(rawHtml);
+            // Drop structural noise.
+            doc.select("script, style, noscript, nav, header, footer, aside, form, iframe").remove();
+            // Drop common ad / share / cookie banners by class hint.
+            Elements adNodes = doc.select(
+                    "[class*=ad-], [class*=ads], [class^=ad_], [id*=ads], " +
+                    "[class*=cookie-banner], [class*=share-], [class*=related-posts]");
+            adNodes.remove();
+            // Strip aria-hidden / display:none nodes — these are usually skip links / overlays.
+            for (Element hidden : doc.select("[aria-hidden=true], [hidden]")) hidden.remove();
+
+            // Convert to text. Jsoup .text() collapses whitespace; we want headings on
+            // their own lines so the preprocessor can detect them. Walk children manually
+            // to preserve heading boundaries.
+            StringBuilder sb = new StringBuilder(Math.min(rawHtml.length(), 256 * 1024));
+            for (Element el : doc.body() != null ? doc.body().getAllElements() : doc.getAllElements()) {
+                String tag = el.tagName();
+                String text = el.ownText();
+                if (text.isBlank()) continue;
+                if (tag.matches("h[1-6]")) {
+                    int level = Integer.parseInt(tag.substring(1));
+                    sb.append('\n').append("#".repeat(level)).append(' ').append(text.trim()).append('\n');
+                } else {
+                    sb.append(text.trim()).append('\n');
+                }
+            }
+            String out = sb.toString();
+            return out.isBlank() ? collapseBlankLines(rawHtml) : collapseBlankLines(out);
+        } catch (Exception e) {
+            log.warn("[WikiContentNormalizer] HTML parse failed, falling back to raw text: {}", e.getMessage());
+            return collapseBlankLines(rawHtml);
+        }
+    }
+
+    /**
+     * Collapse 3+ consecutive blank lines down to 2, normalize CRLF to LF.
+     * Cheap, lossless cleanup that helps chunk boundary detection.
+     */
+    private String collapseBlankLines(String text) {
+        String unified = text.replace("\r\n", "\n").replace('\r', '\n');
+        return unified.replaceAll("\n{3,}", "\n\n");
+    }
+}

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java

@@ -16,6 +16,7 @@ import vip.mate.agent.prompt.PromptLoader;
 import vip.mate.llm.model.ModelConfigEntity;
 import vip.mate.llm.service.ModelConfigService;
 import vip.mate.wiki.WikiProperties;
+import vip.mate.wiki.dto.WikiChunkDraft;
 import vip.mate.wiki.job.WikiKbConfig;
 import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
 import vip.mate.wiki.model.WikiPageEntity;
@@ -60,6 +61,15 @@ public class WikiProcessingService {
     @org.springframework.context.annotation.Lazy
     private vip.mate.wiki.job.WikiProcessingJobService wikiJobService;
 
+    /**
+     * RFC-051 PR-1c: optional preprocessor that fills chunk metadata
+     * (page_number / token_count / header_breadcrumb / source_section).
+     * Marked optional so unit tests that construct this service directly
+     * (without Spring) can opt out without exploding.
+     */
+    @org.springframework.beans.factory.annotation.Autowired(required = false)
+    private DocumentPreprocessService preprocessService;
+
     /** Parallel chunk / material processing executor (JDK 21 virtual threads) */
     public static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor();
 
@@ -1650,6 +1660,34 @@ public class WikiProcessingService {
         }
     }
 
+    /**
+     * RFC-051 PR-1c: bridge from {@link DocumentPreprocessService.Chunker} to
+     * the existing sentence-boundary chunker. Returns {@code [start, end]}
+     * pairs over the supplied text.
+     */
+    private List<int[]> splitToOffsetPairs(String text) {
+        List<ChunkWithOffset> windows = splitIntoChunksWithOffsets(text);
+        List<int[]> out = new ArrayList<>(windows.size());
+        for (ChunkWithOffset w : windows) {
+            out.add(new int[]{w.startOffset(), w.endOffset()});
+        }
+        return out;
+    }
+
+    /**
+     * Legacy chunk persistence path used by the lazy branch when the
+     * preprocessor is unavailable or yields nothing usable. Returns the
+     * persisted chunk count.
+     */
+    private int persistLegacyLazy(Long kbId, Long rawId, String textContent) {
+        List<ChunkWithOffset> chunksWithOffset = splitIntoChunksWithOffsets(textContent);
+        List<String> chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList();
+        List<int[]> offsets = chunksWithOffset.stream()
+                .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList();
+        chunkService.persistChunks(kbId, rawId, chunks, offsets);
+        return chunks.size();
+    }
+
     /**
      * RFC-051 PR-1b: lazy ingest — chunk + embed, no page generation.
      * <p>
@@ -1677,12 +1715,21 @@ public class WikiProcessingService {
                 return;
             }
 
-            List<ChunkWithOffset> chunksWithOffset = splitIntoChunksWithOffsets(textContent);
-            List<String> chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList();
-            List<int[]> offsets = chunksWithOffset.stream()
-                    .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList();
-            chunkService.persistChunks(kbId, rawId, chunks, offsets);
-            int totalChunks = chunks.size();
+            int totalChunks;
+            // PR-1c: when the preprocessor is on the classpath, normalize +
+            // attach metadata; otherwise fall back to the legacy chunker.
+            if (preprocessService != null) {
+                List<WikiChunkDraft> drafts = preprocessService.preprocess(raw, textContent, this::splitToOffsetPairs);
+                if (drafts.isEmpty()) {
+                    log.warn("[Wiki] Lazy preprocess produced 0 drafts for raw={}, falling back to legacy split", rawId);
+                    totalChunks = persistLegacyLazy(kbId, rawId, textContent);
+                } else {
+                    chunkService.persistChunks(kbId, rawId, drafts);
+                    totalChunks = drafts.size();
+                }
+            } else {
+                totalChunks = persistLegacyLazy(kbId, rawId, textContent);
+            }
             log.info("[Wiki] Lazy ingest persisted {} chunks for raw={}", totalChunks, rawId);
 
             // Async embedding — mirror the eager path so a slow embedding model

mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java

@@ -260,11 +260,18 @@ public class WikiTool {
 
         JSONArray arr = new JSONArray();
         for (HybridRetriever.ChunkHit hit : hits) {
-            arr.add(JSONUtil.createObj()
+            cn.hutool.json.JSONObject obj = JSONUtil.createObj()
                     .set("chunkId", hit.chunkId())
                     .set("rawTitle", rawTitles.getOrDefault(hit.rawId(), "unknown"))
                     .set("snippet", hit.snippet())
-                    .set("score", String.format("%.4f", hit.score())));
+                    .set("score", String.format("%.4f", hit.score()));
+            // RFC-051 PR-1c: surface chunk metadata when available so the agent
+            // can cite "page 12, section 'Setup / Linux'" rather than an opaque snippet.
+            if (hit.pageNumber() != null) obj.set("pageNumber", hit.pageNumber());
+            if (hit.headerBreadcrumb() != null && !hit.headerBreadcrumb().isBlank()) {
+                obj.set("section", hit.headerBreadcrumb());
+            }
+            arr.add(obj);
         }
 
         return JSONUtil.createObj()