diff --git a/mateclaw-server/pom.xml b/mateclaw-server/pom.xml index e30d79bc..7bcd5cfd 100644 --- a/mateclaw-server/pom.xml +++ b/mateclaw-server/pom.xml @@ -291,6 +291,20 @@ 5.4.1 + + + + org.jsoup + jsoup + 1.18.3 + + org.flywaydb diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/DocumentPreprocessService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/DocumentPreprocessService.java new file mode 100644 index 00000000..82e89008 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/DocumentPreprocessService.java @@ -0,0 +1,261 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.WikiChunkDraft; +import vip.mate.wiki.model.WikiRawMaterialEntity; + +import java.util.ArrayList; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * RFC-051 PR-1c: produces {@link WikiChunkDraft}s from a raw material's + * extracted text, populating the structural metadata columns added in V39 + * ({@code page_number}, {@code token_count}, {@code header_breadcrumb}, + * {@code source_section}). + *

+ * Strategy is intentionally lightweight: + *

+ * + * No Tika integration: existing {@code DocumentExtractTool} already covers + * the binary formats and the metadata we need for chunks lives inside the + * extracted text, not in document properties. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class DocumentPreprocessService { + + private final WikiContentNormalizer normalizer; + private final WikiProperties properties; + + // Markdown ATX heading: 1-6 leading '#', then a space, then heading text. Setext-style + // (=== / ---) headings are not handled — DocumentExtractTool emits ATX for everything. + private static final Pattern MARKDOWN_HEADING = Pattern.compile("^(#{1,6})\\s+(.+?)\\s*$"); + + // DocumentExtractTool inserts page markers for PDF text extraction. Match a line like + // `--- Page 12 ---` (case-insensitive, tolerant of extra whitespace). PPTX uses + // `--- Slide 4 ---`; treat both as a page-number-like signal. + private static final Pattern PAGE_MARKER = Pattern.compile( + "^\\s*-{2,}\\s*(?:Page|Slide)\\s+(\\d+)\\s*-{2,}\\s*$", + Pattern.CASE_INSENSITIVE); + + /** + * Chunker SPI: pluggable so a unit test (or PR-1c follow-up using a smarter + * splitter) can pass a different boundary algorithm without dragging the + * whole {@link WikiProcessingService} into preprocessing. + */ + public interface Chunker { + List split(String text); + } + + /** + * Build chunk drafts from already-normalized text. Caller is expected to + * have routed the raw text through {@link WikiContentNormalizer} first + * (see {@link #preprocess(WikiRawMaterialEntity, String, Chunker)}). + */ + public List buildDrafts(String text, Chunker chunker) { + if (text == null || text.isBlank()) return List.of(); + + HeaderIndex headerIndex = HeaderIndex.scan(text); + PageIndex pageIndex = PageIndex.scan(text); + + List windows = chunker.split(text); + List drafts = new ArrayList<>(windows.size()); + for (int[] win : windows) { + int start = win[0]; + int end = win[1]; + String content = text.substring(start, end); + int chars = content.length(); + int tokens = (int) Math.ceil(chars / 4.0); + + String breadcrumb = headerIndex.breadcrumbAt(start); + String section = headerIndex.sectionAt(start); // last segment of breadcrumb + Integer page = pageIndex.pageAt(start); + + drafts.add(new WikiChunkDraft(content, start, end, page, tokens, breadcrumb, section)); + } + return drafts; + } + + /** + * Convenience entry: normalize, then chunk, then attach metadata. Intended + * for the lazy ingest branch of {@link WikiProcessingService}. + */ + public List preprocess(WikiRawMaterialEntity raw, String extractedText, Chunker chunker) { + String normalized = normalizer.normalize(raw == null ? null : raw.getSourceType(), extractedText); + return buildDrafts(normalized, chunker); + } + + /** Returns the post-normalization text alone, for callers that need to chunk separately. */ + public String normalize(WikiRawMaterialEntity raw, String extractedText) { + return normalizer.normalize(raw == null ? null : raw.getSourceType(), extractedText); + } + + @SuppressWarnings("unused") // exposed for future callers; keeps the property accessible. + public WikiProperties properties() { + return properties; + } + + // ──────────────────────────────────────────────────────────────────────── + // Header index — precomputed list of (lineStartOffset, breadcrumb, deepestHeading) + // sorted by offset. Lookup uses binary search. + // ──────────────────────────────────────────────────────────────────────── + + private static final class HeaderIndex { + private final int[] starts; + private final String[] breadcrumbs; + private final String[] deepest; + + private HeaderIndex(int[] starts, String[] breadcrumbs, String[] deepest) { + this.starts = starts; + this.breadcrumbs = breadcrumbs; + this.deepest = deepest; + } + + static HeaderIndex scan(String text) { + List offsets = new ArrayList<>(); + List crumbs = new ArrayList<>(); + List last = new ArrayList<>(); + Deque stack = new ArrayDeque<>(); // each entry: [level, title] + + int n = text.length(); + int i = 0; + while (i < n) { + int lineEnd = text.indexOf('\n', i); + if (lineEnd < 0) lineEnd = n; + String line = text.substring(i, lineEnd); + Matcher m = MARKDOWN_HEADING.matcher(line); + if (m.matches()) { + int level = m.group(1).length(); + String title = m.group(2).trim(); + // Pop deeper-or-equal levels. + while (!stack.isEmpty() && Integer.parseInt(stack.peek()[0]) >= level) { + stack.pop(); + } + stack.push(new String[]{Integer.toString(level), title}); + String breadcrumb = buildBreadcrumb(stack); + offsets.add(new int[]{i}); + crumbs.add(breadcrumb); + last.add(title); + } + i = lineEnd + 1; + } + + int[] starts = new int[offsets.size()]; + String[] breadcrumbs = new String[offsets.size()]; + String[] deepest = new String[offsets.size()]; + for (int k = 0; k < offsets.size(); k++) { + starts[k] = offsets.get(k)[0]; + breadcrumbs[k] = crumbs.get(k); + deepest[k] = last.get(k); + } + return new HeaderIndex(starts, breadcrumbs, deepest); + } + + private static String buildBreadcrumb(Deque stack) { + // Stack iterates top-first; build a root-first list for display. + List ordered = new ArrayList<>(stack); + StringBuilder sb = new StringBuilder(); + for (int k = ordered.size() - 1; k >= 0; k--) { + if (sb.length() > 0) sb.append(" / "); + sb.append(ordered.get(k)[1]); + } + // Cap at the column width of the DB column (1024). + return sb.length() > 1000 ? sb.substring(0, 1000) : sb.toString(); + } + + String breadcrumbAt(int offset) { + int idx = floorIndex(offset); + return idx < 0 ? null : breadcrumbs[idx]; + } + + String sectionAt(int offset) { + int idx = floorIndex(offset); + if (idx < 0) return null; + String s = deepest[idx]; + return s != null && s.length() > 500 ? s.substring(0, 500) : s; + } + + private int floorIndex(int offset) { + // Largest index with starts[idx] <= offset. + if (starts.length == 0 || offset < starts[0]) return -1; + int lo = 0, hi = starts.length - 1, ans = -1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + if (starts[mid] <= offset) { ans = mid; lo = mid + 1; } + else hi = mid - 1; + } + return ans; + } + } + + // ──────────────────────────────────────────────────────────────────────── + // Page index — same shape as HeaderIndex but for "--- Page N ---" markers. + // ──────────────────────────────────────────────────────────────────────── + + private static final class PageIndex { + private final int[] starts; + private final int[] pages; + + private PageIndex(int[] starts, int[] pages) { + this.starts = starts; + this.pages = pages; + } + + static PageIndex scan(String text) { + List matches = new ArrayList<>(); + int n = text.length(); + int i = 0; + while (i < n) { + int lineEnd = text.indexOf('\n', i); + if (lineEnd < 0) lineEnd = n; + String line = text.substring(i, lineEnd); + Matcher m = PAGE_MARKER.matcher(line); + if (m.matches()) { + try { + int page = Integer.parseInt(m.group(1)); + matches.add(new int[]{i, page}); + } catch (NumberFormatException ignored) { + // overflow / not a number; ignore — page stays unknown for this region. + } + } + i = lineEnd + 1; + } + int[] starts = new int[matches.size()]; + int[] pages = new int[matches.size()]; + for (int k = 0; k < matches.size(); k++) { + starts[k] = matches.get(k)[0]; + pages[k] = matches.get(k)[1]; + } + return new PageIndex(starts, pages); + } + + Integer pageAt(int offset) { + if (starts.length == 0 || offset < starts[0]) return null; + int lo = 0, hi = starts.length - 1, ans = -1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1; + if (starts[mid] <= offset) { ans = mid; lo = mid + 1; } + else hi = mid - 1; + } + return ans < 0 ? null : pages[ans]; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java index c41b0b49..c2042632 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java @@ -59,8 +59,19 @@ public class HybridRetriever { /** * Chunk-level search result (semantic search). + *

+ * RFC-051 PR-1c: {@code pageNumber} and {@code headerBreadcrumb} are + * populated when the chunk has those columns set (lazy ingest with + * preprocessor on, or backfilled chunks). Both are nullable. */ - public record ChunkHit(Long chunkId, Long rawId, String snippet, float score) {} + public record ChunkHit(Long chunkId, Long rawId, String snippet, float score, + Integer pageNumber, String headerBreadcrumb) { + + /** Backwards-compatible factory for callers that don't yet pass metadata. */ + public ChunkHit(Long chunkId, Long rawId, String snippet, float score) { + this(chunkId, rawId, snippet, score, null, null); + } + } /** * RFC-032: Enhanced search returning PageSearchResult with snippet and matchedBy metadata. @@ -152,7 +163,8 @@ public class HybridRetriever { String snippet = c.getContent().length() > 300 ? c.getContent().substring(0, 300) + "..." : c.getContent(); - return new ChunkHit(c.getId(), c.getRawId(), snippet, score); + return new ChunkHit(c.getId(), c.getRawId(), snippet, score, + c.getPageNumber(), c.getHeaderBreadcrumb()); }) .sorted(Comparator.comparingDouble(ChunkHit::score).reversed()) .limit(topK) diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java new file mode 100644 index 00000000..7b9275bd --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java @@ -0,0 +1,104 @@ +package vip.mate.wiki.service; + +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.stereotype.Component; + +/** + * RFC-051 PR-1c: source-type-specific text normalization. + *

+ * Goal is structural cleanup before chunking — strip noise from HTML, + * normalize line endings, trim duplicate whitespace — without touching the + * actual semantic content. Heading and page-marker extraction lives in + * {@link DocumentPreprocessService} because it needs the offsets of the + * surviving text, not just the text itself. + * + *

Tika is intentionally not pulled in: existing {@code DocumentExtractTool} + * already covers PDF/Office via pdftotext / pdfplumber / Java fallbacks. + */ +@Slf4j +@Component +public class WikiContentNormalizer { + + private static final int MAX_HTML_LEN = 8 * 1024 * 1024; // 8 MB safety cap + + /** + * Normalize raw text by source type. Returns the input unchanged when the + * source type is unknown so we never lose content silently. + * + * @param sourceType {@code WikiRawMaterialEntity.sourceType} value + * (text / pdf / docx / xlsx / pptx / url / paste / markdown) + * @param rawText extracted text content + * @return cleaned text, never {@code null} + */ + public String normalize(String sourceType, String rawText) { + if (rawText == null) return ""; + String type = sourceType == null ? "" : sourceType.toLowerCase(); + return switch (type) { + case "url", "html" -> normalizeHtml(rawText); + // PDF text from DocumentExtractTool may already contain "--- Page N ---" + // markers; we keep them so the preprocessor can map char offsets to pages. + case "pdf" -> collapseBlankLines(rawText); + case "docx", "pptx", "xlsx" -> collapseBlankLines(rawText); + case "markdown", "md", "text", "paste" -> collapseBlankLines(rawText); + default -> collapseBlankLines(rawText); + }; + } + + /** + * Strip nav/footer/script/style/aside and ad-like classes from HTML, then + * return readable text. Falls through to the raw input when the document + * is too large to parse safely or jsoup throws. + */ + private String normalizeHtml(String rawHtml) { + if (rawHtml.length() > MAX_HTML_LEN) { + log.warn("[WikiContentNormalizer] HTML payload exceeds {} bytes, skipping cleanup", MAX_HTML_LEN); + return collapseBlankLines(rawHtml); + } + try { + Document doc = Jsoup.parse(rawHtml); + // Drop structural noise. + doc.select("script, style, noscript, nav, header, footer, aside, form, iframe").remove(); + // Drop common ad / share / cookie banners by class hint. + Elements adNodes = doc.select( + "[class*=ad-], [class*=ads], [class^=ad_], [id*=ads], " + + "[class*=cookie-banner], [class*=share-], [class*=related-posts]"); + adNodes.remove(); + // Strip aria-hidden / display:none nodes — these are usually skip links / overlays. + for (Element hidden : doc.select("[aria-hidden=true], [hidden]")) hidden.remove(); + + // Convert to text. Jsoup .text() collapses whitespace; we want headings on + // their own lines so the preprocessor can detect them. Walk children manually + // to preserve heading boundaries. + StringBuilder sb = new StringBuilder(Math.min(rawHtml.length(), 256 * 1024)); + for (Element el : doc.body() != null ? doc.body().getAllElements() : doc.getAllElements()) { + String tag = el.tagName(); + String text = el.ownText(); + if (text.isBlank()) continue; + if (tag.matches("h[1-6]")) { + int level = Integer.parseInt(tag.substring(1)); + sb.append('\n').append("#".repeat(level)).append(' ').append(text.trim()).append('\n'); + } else { + sb.append(text.trim()).append('\n'); + } + } + String out = sb.toString(); + return out.isBlank() ? collapseBlankLines(rawHtml) : collapseBlankLines(out); + } catch (Exception e) { + log.warn("[WikiContentNormalizer] HTML parse failed, falling back to raw text: {}", e.getMessage()); + return collapseBlankLines(rawHtml); + } + } + + /** + * Collapse 3+ consecutive blank lines down to 2, normalize CRLF to LF. + * Cheap, lossless cleanup that helps chunk boundary detection. + */ + private String collapseBlankLines(String text) { + String unified = text.replace("\r\n", "\n").replace('\r', '\n'); + return unified.replaceAll("\n{3,}", "\n\n"); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index a691c01f..f2926808 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -16,6 +16,7 @@ import vip.mate.agent.prompt.PromptLoader; import vip.mate.llm.model.ModelConfigEntity; import vip.mate.llm.service.ModelConfigService; import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.WikiChunkDraft; import vip.mate.wiki.job.WikiKbConfig; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; @@ -60,6 +61,15 @@ public class WikiProcessingService { @org.springframework.context.annotation.Lazy private vip.mate.wiki.job.WikiProcessingJobService wikiJobService; + /** + * RFC-051 PR-1c: optional preprocessor that fills chunk metadata + * (page_number / token_count / header_breadcrumb / source_section). + * Marked optional so unit tests that construct this service directly + * (without Spring) can opt out without exploding. + */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private DocumentPreprocessService preprocessService; + /** Parallel chunk / material processing executor (JDK 21 virtual threads) */ public static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor(); @@ -1650,6 +1660,34 @@ public class WikiProcessingService { } } + /** + * RFC-051 PR-1c: bridge from {@link DocumentPreprocessService.Chunker} to + * the existing sentence-boundary chunker. Returns {@code [start, end]} + * pairs over the supplied text. + */ + private List splitToOffsetPairs(String text) { + List windows = splitIntoChunksWithOffsets(text); + List out = new ArrayList<>(windows.size()); + for (ChunkWithOffset w : windows) { + out.add(new int[]{w.startOffset(), w.endOffset()}); + } + return out; + } + + /** + * Legacy chunk persistence path used by the lazy branch when the + * preprocessor is unavailable or yields nothing usable. Returns the + * persisted chunk count. + */ + private int persistLegacyLazy(Long kbId, Long rawId, String textContent) { + List chunksWithOffset = splitIntoChunksWithOffsets(textContent); + List chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList(); + List offsets = chunksWithOffset.stream() + .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList(); + chunkService.persistChunks(kbId, rawId, chunks, offsets); + return chunks.size(); + } + /** * RFC-051 PR-1b: lazy ingest — chunk + embed, no page generation. *

@@ -1677,12 +1715,21 @@ public class WikiProcessingService { return; } - List chunksWithOffset = splitIntoChunksWithOffsets(textContent); - List chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList(); - List offsets = chunksWithOffset.stream() - .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList(); - chunkService.persistChunks(kbId, rawId, chunks, offsets); - int totalChunks = chunks.size(); + int totalChunks; + // PR-1c: when the preprocessor is on the classpath, normalize + + // attach metadata; otherwise fall back to the legacy chunker. + if (preprocessService != null) { + List drafts = preprocessService.preprocess(raw, textContent, this::splitToOffsetPairs); + if (drafts.isEmpty()) { + log.warn("[Wiki] Lazy preprocess produced 0 drafts for raw={}, falling back to legacy split", rawId); + totalChunks = persistLegacyLazy(kbId, rawId, textContent); + } else { + chunkService.persistChunks(kbId, rawId, drafts); + totalChunks = drafts.size(); + } + } else { + totalChunks = persistLegacyLazy(kbId, rawId, textContent); + } log.info("[Wiki] Lazy ingest persisted {} chunks for raw={}", totalChunks, rawId); // Async embedding — mirror the eager path so a slow embedding model diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java index f9289edf..29b53b0f 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java @@ -260,11 +260,18 @@ public class WikiTool { JSONArray arr = new JSONArray(); for (HybridRetriever.ChunkHit hit : hits) { - arr.add(JSONUtil.createObj() + cn.hutool.json.JSONObject obj = JSONUtil.createObj() .set("chunkId", hit.chunkId()) .set("rawTitle", rawTitles.getOrDefault(hit.rawId(), "unknown")) .set("snippet", hit.snippet()) - .set("score", String.format("%.4f", hit.score()))); + .set("score", String.format("%.4f", hit.score())); + // RFC-051 PR-1c: surface chunk metadata when available so the agent + // can cite "page 12, section 'Setup / Linux'" rather than an opaque snippet. + if (hit.pageNumber() != null) obj.set("pageNumber", hit.pageNumber()); + if (hit.headerBreadcrumb() != null && !hit.headerBreadcrumb().isBlank()) { + obj.set("section", hit.headerBreadcrumb()); + } + arr.add(obj); } return JSONUtil.createObj()