diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java index 6a0703eb..205bb0fd 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java @@ -46,16 +46,16 @@ public class WikiContentNormalizer { if (rawText == null) return ""; String type = sourceType == null ? "" : sourceType.toLowerCase(); return switch (type) { - // 'url' is raw fetched web HTML and still carries markup, so it needs - // tag stripping here. 'html' file uploads are tag-stripped upstream by - // DocumentExtractTool (jsoup) before reaching the normalizer — running - // normalizeHtml again would re-collapse the recovered heading structure, - // so they fall through to the plain-text branch. - case "url" -> normalizeHtml(rawText); + // Web and HTML sources can still carry raw markup, so they go through + // normalizeHtml, which strips script/style/nav/footer noise. normalizeHtml + // also recognizes text that was already tag-stripped upstream (an extracted + // .html/.htm upload re-entering the normalizer) and leaves that text's line + // structure intact instead of re-collapsing its recovered headings. + case "url", "html", "htm" -> normalizeHtml(rawText); // PDF text from DocumentExtractTool may already contain "--- Page N ---" // markers; we keep them so the preprocessor can map char offsets to pages. case "pdf" -> collapseBlankLines(rawText); - case "docx", "doc", "pptx", "ppt", "xlsx", "xls", "html", "htm" -> collapseBlankLines(rawText); + case "docx", "doc", "pptx", "ppt", "xlsx", "xls" -> collapseBlankLines(rawText); case "markdown", "md", "text", "paste" -> collapseBlankLines(rawText); default -> collapseBlankLines(rawText); }; @@ -63,16 +63,38 @@ public class WikiContentNormalizer { /** * Strip nav/footer/script/style/aside and ad-like classes from HTML, then - * return readable text. Falls through to the raw input when the document - * is too large to parse safely or jsoup throws. + * return readable text. + *

+ * The output never carries markup: input that still contains tags is parsed + * and stripped; input that is already tag-free is returned with whitespace + * cleanup only. When the document cannot be parsed (too large, or jsoup + * throws) it is run through {@link #stripMarkupLossy(String)} so that + * script/style bodies and tags are dropped without a full parse — the raw + * markup is never passed through verbatim. */ private String normalizeHtml(String rawHtml) { if (rawHtml.length() > MAX_HTML_LEN) { - log.warn("[WikiContentNormalizer] HTML payload exceeds {} bytes, skipping cleanup", MAX_HTML_LEN); - return collapseBlankLines(rawHtml); + log.warn("[WikiContentNormalizer] HTML payload exceeds {} bytes, stripping markup without a full parse", MAX_HTML_LEN); + return stripMarkupLossy(rawHtml); } try { Document doc = Jsoup.parse(rawHtml); + + // Distinguish genuine HTML markup from text that was already tag-stripped + // upstream (an extracted .html/.htm upload re-entering the normalizer). + // jsoup synthesizes an // skeleton even for plain text, + // so the absence of any element children means there is no real markup. + // Plain text must skip the element walker below: that walker relies on + // Element.ownText(), which collapses newlines and would merge a heading + // line into the paragraph that follows it. + Element body = doc.body(); + Element head = doc.head(); + boolean hasMarkup = (body != null && !body.children().isEmpty()) + || (head != null && !head.children().isEmpty()); + if (!hasMarkup) { + return collapseBlankLines(rawHtml); + } + // Drop structural noise. doc.select("script, style, noscript, nav, header, footer, aside, form, iframe").remove(); // Drop common ad / share / cookie banners by class hint. @@ -99,13 +121,41 @@ public class WikiContentNormalizer { } } String out = sb.toString(); - return out.isBlank() ? collapseBlankLines(rawHtml) : collapseBlankLines(out); + if (!out.isBlank()) { + return collapseBlankLines(out); + } + // The walker produced nothing — the document was pure script/style/nav + // noise. Fall back to jsoup's plain-text extraction, never the raw markup: + // returning rawHtml here would carry