diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java index 32e07c99..b3fc54e2 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java @@ -20,7 +20,7 @@ import java.util.zip.ZipInputStream; /** * Document text extraction tool. - * Supports PDF, DOCX, XLSX, PPTX with format-specific fallback chains. + * Supports PDF, DOCX, XLSX, PPTX, HTML with format-specific fallback chains. * * Strategy by format: * - PDF: pdftotext -> pdfplumber/pypdf -> pdfbox -> OCR (scanned) -> Tika @@ -28,6 +28,8 @@ import java.util.zip.ZipInputStream; * - XLSX/PPTX: Tika directly (POI-based; correctly resolves the shared-strings * indirection table and walks SmartArt / chart / grouped-shape * text that a naive ZIP+XML scan misses). + * - HTML: jsoup parse -> drop script/style/nav/footer noise -> keep + * heading hierarchy as Markdown ATX lines. */ @Slf4j @Component @@ -46,11 +48,13 @@ public class DocumentExtractTool { - Word (.docx, .doc) - Excel (.xlsx, .xls) - 提取为文本表格 - PowerPoint (.pptx, .ppt) + - HTML (.html, .htm) - jsoup 清洗后提取正文 提取策略（按格式分链）： - PDF: pdftotext → pdfplumber/pypdf → pdfbox → OCR（扫描版） → Tika - DOCX: textutil / pandoc / libreoffice → ZIP-XML → Tika - XLSX/PPTX: 直接走 Tika（基于 POI，正确解析 sharedStrings 表与 SmartArt / 图表文本） + - HTML: jsoup 解析 → 去除 script/style/nav/footer 等噪音 → 保留标题层级 - 返回详细的提取过程和元数据 参数 options 可包含： @@ -134,6 +138,8 @@ public class DocumentExtractTool { content = extractXlsx(path, options, attempts); } else if (mimeType.contains("presentationml") || mimeType.contains("powerpoint")) { content = extractPptx(path, options, attempts); + } else if (mimeType.contains("html")) { + content = extractHtml(path, attempts); } else { return errorResult(filePath, "不支持的文档类型: " + mimeType, attempts); } @@ -869,6 +875,53 @@ public class DocumentExtractTool { return count; } + // ==================== HTML 提取 ==================== + + /** + * Extract readable text from an HTML file with jsoup. + *

+ * Drops structural noise (script / style / nav / header / footer / aside / + * form / iframe), then walks the surviving elements emitting headings as + * Markdown ATX lines ({@code # }, {@code ## } …) so the wiki preprocessor + * can still detect the document's heading hierarchy. The charset is + * auto-detected from the BOM / {@code } declaration. + */ + private ExtractedContent extractHtml(Path path, List attempts) throws Exception { + long t = System.currentTimeMillis(); + org.jsoup.nodes.Document doc; + try { + // charsetName = null lets jsoup sniff the encoding from BOM / meta tag. + doc = org.jsoup.Jsoup.parse(path.toFile(), null); + } catch (IOException e) { + attempts.add("jsoup: 读取失败 - " + e.getMessage()); + throw new Exception("HTML 文件读取失败: " + e.getMessage()); + } + + doc.select("script, style, noscript, nav, header, footer, aside, form, iframe").remove(); + + StringBuilder sb = new StringBuilder(); + org.jsoup.nodes.Element root = doc.body() != null ? doc.body() : doc; + for (org.jsoup.nodes.Element el : root.getAllElements()) { + String text = el.ownText(); + if (text.isBlank()) continue; + String tag = el.tagName(); + if (tag.length() == 2 && tag.charAt(0) == 'h' && tag.charAt(1) >= '1' && tag.charAt(1) <= '6') { + int level = tag.charAt(1) - '0'; + sb.append('\n').append("#".repeat(level)).append(' ').append(text.trim()).append('\n'); + } else { + sb.append(text.trim()).append('\n'); + } + } + + String out = sb.toString().strip(); + if (out.isBlank()) { + attempts.add("jsoup: 解析成功但无可读文本 (" + (System.currentTimeMillis() - t) + "ms)"); + throw new Exception("HTML 提取无文本（页面可能仅含脚本 / 样式）"); + } + attempts.add("jsoup: 成功 (" + (System.currentTimeMillis() - t) + "ms)"); + return new ExtractedContent(out, "jsoup", 0); + } + // ==================== 工具方法 ==================== /** @@ -938,6 +991,7 @@ public class DocumentExtractTool { if (fileName.endsWith(".xls")) return "application/vnd.ms-excel"; if (fileName.endsWith(".pptx")) return "application/vnd.openxmlformats-officedocument.presentationml.presentation"; if (fileName.endsWith(".ppt")) return "application/vnd.ms-powerpoint"; + if (fileName.endsWith(".html") || fileName.endsWith(".htm")) return "text/html"; return "application/octet-stream"; } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java index 42f50070..d41542ad 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java @@ -263,12 +263,17 @@ public class WikiController { : "txt"; // Resolve source type from extension. Image extensions route to the - // vision-in pipeline at extraction time; everything else falls through - // to the existing text / pdf / docx handling. + // vision-in pipeline at extraction time; Office / PDF / HTML extensions + // are staged on disk and extracted by DocumentExtractTool; plain-text + // formats (incl. CSV) are stored directly. Unknown extensions fall back + // to text so the upload never hard-fails. String sourceType = switch (extension) { case "pdf" -> "pdf"; case "docx", "doc" -> "docx"; - case "txt", "md" -> "text"; + case "xlsx", "xls" -> "xlsx"; + case "pptx", "ppt" -> "pptx"; + case "html", "htm" -> "html"; + case "txt", "md", "csv" -> "text"; case "png", "jpg", "jpeg", "webp", "gif", "bmp", "tiff", "tif" -> "image"; default -> "text"; }; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java index 010cfb30..6a0703eb 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java @@ -46,11 +46,16 @@ public class WikiContentNormalizer { if (rawText == null) return ""; String type = sourceType == null ? "" : sourceType.toLowerCase(); return switch (type) { - case "url", "html" -> normalizeHtml(rawText); + // 'url' is raw fetched web HTML and still carries markup, so it needs + // tag stripping here. 'html' file uploads are tag-stripped upstream by + // DocumentExtractTool (jsoup) before reaching the normalizer — running + // normalizeHtml again would re-collapse the recovered heading structure, + // so they fall through to the plain-text branch. + case "url" -> normalizeHtml(rawText); // PDF text from DocumentExtractTool may already contain "--- Page N ---" // markers; we keep them so the preprocessor can map char offsets to pages. case "pdf" -> collapseBlankLines(rawText); - case "docx", "pptx", "xlsx" -> collapseBlankLines(rawText); + case "docx", "doc", "pptx", "ppt", "xlsx", "xls", "html", "htm" -> collapseBlankLines(rawText); case "markdown", "md", "text", "paste" -> collapseBlankLines(rawText); default -> collapseBlankLines(rawText); }; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java index 8130618b..46640b44 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java @@ -31,10 +31,11 @@ public class WikiDirectoryScanService { private final WikiProperties properties; private static final Set SUPPORTED_EXTENSIONS = Set.of( - "txt", "md", "pdf", "docx", "doc", "pptx", "xlsx" + "txt", "md", "csv", "pdf", "docx", "doc", + "pptx", "ppt", "xlsx", "xls", "html", "htm" ); - private static final Set TEXT_EXTENSIONS = Set.of("txt", "md"); + private static final Set TEXT_EXTENSIONS = Set.of("txt", "md", "csv"); /** * 扫描结果 @@ -144,8 +145,9 @@ public class WikiDirectoryScanService { String sourceType = switch (ext) { case "pdf" -> "pdf"; case "docx", "doc" -> "docx"; - case "pptx" -> "pptx"; - case "xlsx" -> "xlsx"; + case "pptx", "ppt" -> "pptx"; + case "xlsx", "xls" -> "xlsx"; + case "html", "htm" -> "html"; default -> "text"; }; rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file)); diff --git a/mateclaw-server/src/main/resources/docs/en/wiki.md b/mateclaw-server/src/main/resources/docs/en/wiki.md index 83b997b3..683024ce 100644 --- a/mateclaw-server/src/main/resources/docs/en/wiki.md +++ b/mateclaw-server/src/main/resources/docs/en/wiki.md @@ -37,7 +37,7 @@ MateClaw's LLM Wiki **is the same idea, raised into a product**: A knowledge base is three layers stacked on top of each other: -1. **Raw material** — the files you dropped in. PDF, DOCX, plain text, markdown, or a whole local directory scanned in one go. The system keeps them intact; any claim in the Wiki traces back to the passage that produced it. +1. **Raw material** — the files you dropped in. PDF, Word, Excel, PowerPoint, HTML, markdown, plain text (incl. CSV), or a whole local directory scanned in one go. The system keeps them intact; any claim in the Wiki traces back to the passage that produced it. 2. **Wiki pages** — structured articles the AI writes from the raw material. Each page has a title, a summary, a body, bidirectional links to related pages (`[[like this]]`, plus the alias form `[[target|display text]]`), and provenance pointers back into the raw layer. 3. **Agent surface** — when an agent calls a wiki tool, the system auto-injects the summaries of relevant pages into the prompt. Bodies are fetched on demand. Agents don't read raw files. They read the library. @@ -51,7 +51,7 @@ This matters because the agent's context window stops getting wasted on re-readi Once it exists, add material: -- **Upload files** — drag PDFs, DOCX, markdown, or plain text into the upload area. Each file becomes a raw material row. +- **Upload files** — drag PDF, Word, Excel, PowerPoint, HTML, markdown, or plain-text (incl. CSV) files into the upload area. Each file becomes a raw material row. - **Scan a local directory** — desktop only. Point at a folder and MateClaw walks it recursively, respecting `.gitignore`, importing everything that looks like text. - **Paste text** — for short excerpts or conversation transcripts. diff --git a/mateclaw-server/src/main/resources/docs/zh/wiki.md b/mateclaw-server/src/main/resources/docs/zh/wiki.md index 7fba9844..0498b38a 100644 --- a/mateclaw-server/src/main/resources/docs/zh/wiki.md +++ b/mateclaw-server/src/main/resources/docs/zh/wiki.md @@ -37,7 +37,7 @@ MateClaw 的 LLM Wiki **是同一个想法长成的产品**： 一个知识库是三层结构叠起来的： -1. **原始材料层**——你扔进去的文件。PDF、DOCX、纯文本、Markdown，或者桌面端扫描整个本地目录。系统保留原文不动；Wiki 里的任何一句话都能回溯到它出自哪段原文。 +1. **原始材料层**——你扔进去的文件。PDF、Word、Excel、PowerPoint、HTML、Markdown、纯文本（含 CSV），或者桌面端扫描整个本地目录。系统保留原文不动；Wiki 里的任何一句话都能回溯到它出自哪段原文。 2. **Wiki 页面层**——AI 从原始材料里写出的结构化文章。每一页有标题、摘要、正文、指向相关页面的双向链接（`[[像这样]]`，也支持 `[[target|展示文字]]` alias 形式）、以及通往原文的来源指针。 3. **Agent 表层**——Agent 调用 wiki 工具时，系统会把相关页面的摘要自动注入 prompt，正文按需读取。Agent **不读原文**，它读这本书。 @@ -51,7 +51,7 @@ MateClaw 的 LLM Wiki **是同一个想法长成的产品**： 建好之后加材料： -- **上传文件**——把 PDF、DOCX、Markdown、纯文本拖进上传区。每个文件成为一条 raw material。 +- **上传文件**——把 PDF、Word、Excel、PowerPoint、HTML、Markdown、纯文本（含 CSV）拖进上传区。每个文件成为一条 raw material。 - **扫描本地目录**——桌面端专属。指一个文件夹，MateClaw 递归走完整个树，尊重 `.gitignore`，把能读出文本的全都导入。 - **粘贴文本**——适合短片段或对话记录。 diff --git a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue index 09c27e8f..31c238a4 100644 --- a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue +++ b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue @@ -32,10 +32,10 @@ - .txt, .md, .pdf, .docx + .txt .md .csv .pdf .docx .xlsx .pptx .html - +