feat(wiki): support HTML, Excel, PowerPoint and CSV raw materials

mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java

@@ -20,7 +20,7 @@ import java.util.zip.ZipInputStream;
 
 /**
  * Document text extraction tool.
- * Supports PDF, DOCX, XLSX, PPTX with format-specific fallback chains.
+ * Supports PDF, DOCX, XLSX, PPTX, HTML with format-specific fallback chains.
  *
  * Strategy by format:
  * - PDF:       pdftotext -> pdfplumber/pypdf -> pdfbox -> OCR (scanned) -> Tika
@@ -28,6 +28,8 @@ import java.util.zip.ZipInputStream;
  * - XLSX/PPTX: Tika directly (POI-based; correctly resolves the shared-strings
  *              indirection table and walks SmartArt / chart / grouped-shape
  *              text that a naive ZIP+XML scan misses).
+ * - HTML:      jsoup parse -> drop script/style/nav/footer noise -> keep
+ *              heading hierarchy as Markdown ATX lines.
  */
 @Slf4j
 @Component
@@ -46,11 +48,13 @@ public class DocumentExtractTool {
         - Word (.docx, .doc)
         - Excel (.xlsx, .xls) - 提取为文本表格
         - PowerPoint (.pptx, .ppt)
+        - HTML (.html, .htm) - jsoup 清洗后提取正文
 
         提取策略（按格式分链）：
         - PDF:       pdftotext → pdfplumber/pypdf → pdfbox → OCR（扫描版） → Tika
         - DOCX:      textutil / pandoc / libreoffice → ZIP-XML → Tika
         - XLSX/PPTX: 直接走 Tika（基于 POI，正确解析 sharedStrings 表与 SmartArt / 图表文本）
+        - HTML:      jsoup 解析 → 去除 script/style/nav/footer 等噪音 → 保留标题层级
         - 返回详细的提取过程和元数据
 
         参数 options 可包含：
@@ -134,6 +138,8 @@ public class DocumentExtractTool {
                 content = extractXlsx(path, options, attempts);
             } else if (mimeType.contains("presentationml") || mimeType.contains("powerpoint")) {
                 content = extractPptx(path, options, attempts);
+            } else if (mimeType.contains("html")) {
+                content = extractHtml(path, attempts);
             } else {
                 return errorResult(filePath, "不支持的文档类型: " + mimeType, attempts);
             }
@@ -869,6 +875,53 @@ public class DocumentExtractTool {
         return count;
     }
 
+    // ==================== HTML 提取 ====================
+
+    /**
+     * Extract readable text from an HTML file with jsoup.
+     * <p>
+     * Drops structural noise (script / style / nav / header / footer / aside /
+     * form / iframe), then walks the surviving elements emitting headings as
+     * Markdown ATX lines ({@code # }, {@code ## } …) so the wiki preprocessor
+     * can still detect the document's heading hierarchy. The charset is
+     * auto-detected from the BOM / {@code <meta charset>} declaration.
+     */
+    private ExtractedContent extractHtml(Path path, List<String> attempts) throws Exception {
+        long t = System.currentTimeMillis();
+        org.jsoup.nodes.Document doc;
+        try {
+            // charsetName = null lets jsoup sniff the encoding from BOM / meta tag.
+            doc = org.jsoup.Jsoup.parse(path.toFile(), null);
+        } catch (IOException e) {
+            attempts.add("jsoup: 读取失败 - " + e.getMessage());
+            throw new Exception("HTML 文件读取失败: " + e.getMessage());
+        }
+
+        doc.select("script, style, noscript, nav, header, footer, aside, form, iframe").remove();
+
+        StringBuilder sb = new StringBuilder();
+        org.jsoup.nodes.Element root = doc.body() != null ? doc.body() : doc;
+        for (org.jsoup.nodes.Element el : root.getAllElements()) {
+            String text = el.ownText();
+            if (text.isBlank()) continue;
+            String tag = el.tagName();
+            if (tag.length() == 2 && tag.charAt(0) == 'h' && tag.charAt(1) >= '1' && tag.charAt(1) <= '6') {
+                int level = tag.charAt(1) - '0';
+                sb.append('\n').append("#".repeat(level)).append(' ').append(text.trim()).append('\n');
+            } else {
+                sb.append(text.trim()).append('\n');
+            }
+        }
+
+        String out = sb.toString().strip();
+        if (out.isBlank()) {
+            attempts.add("jsoup: 解析成功但无可读文本 (" + (System.currentTimeMillis() - t) + "ms)");
+            throw new Exception("HTML 提取无文本（页面可能仅含脚本 / 样式）");
+        }
+        attempts.add("jsoup: 成功 (" + (System.currentTimeMillis() - t) + "ms)");
+        return new ExtractedContent(out, "jsoup", 0);
+    }
+
     // ==================== 工具方法 ====================
 
     /**
@@ -938,6 +991,7 @@ public class DocumentExtractTool {
         if (fileName.endsWith(".xls")) return "application/vnd.ms-excel";
         if (fileName.endsWith(".pptx")) return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
         if (fileName.endsWith(".ppt")) return "application/vnd.ms-powerpoint";
+        if (fileName.endsWith(".html") || fileName.endsWith(".htm")) return "text/html";
         return "application/octet-stream";
     }
 

mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java

@@ -263,12 +263,17 @@ public class WikiController {
                 : "txt";
 
         // Resolve source type from extension. Image extensions route to the
-        // vision-in pipeline at extraction time; everything else falls through
+        // vision-in pipeline at extraction time; Office / PDF / HTML extensions
-        // to the existing text / pdf / docx handling.
+        // are staged on disk and extracted by DocumentExtractTool; plain-text
+        // formats (incl. CSV) are stored directly. Unknown extensions fall back
+        // to text so the upload never hard-fails.
         String sourceType = switch (extension) {
             case "pdf" -> "pdf";
             case "docx", "doc" -> "docx";
-            case "txt", "md" -> "text";
+            case "xlsx", "xls" -> "xlsx";
+            case "pptx", "ppt" -> "pptx";
+            case "html", "htm" -> "html";
+            case "txt", "md", "csv" -> "text";
             case "png", "jpg", "jpeg", "webp", "gif", "bmp", "tiff", "tif" -> "image";
             default -> "text";
         };

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java

@@ -46,11 +46,16 @@ public class WikiContentNormalizer {
         if (rawText == null) return "";
         String type = sourceType == null ? "" : sourceType.toLowerCase();
         return switch (type) {
-            case "url", "html" -> normalizeHtml(rawText);
+            // 'url' is raw fetched web HTML and still carries markup, so it needs
+            // tag stripping here. 'html' file uploads are tag-stripped upstream by
+            // DocumentExtractTool (jsoup) before reaching the normalizer — running
+            // normalizeHtml again would re-collapse the recovered heading structure,
+            // so they fall through to the plain-text branch.
+            case "url" -> normalizeHtml(rawText);
             // PDF text from DocumentExtractTool may already contain "--- Page N ---"
             // markers; we keep them so the preprocessor can map char offsets to pages.
             case "pdf" -> collapseBlankLines(rawText);
-            case "docx", "pptx", "xlsx" -> collapseBlankLines(rawText);
+            case "docx", "doc", "pptx", "ppt", "xlsx", "xls", "html", "htm" -> collapseBlankLines(rawText);
             case "markdown", "md", "text", "paste" -> collapseBlankLines(rawText);
             default -> collapseBlankLines(rawText);
         };

mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java

@@ -31,10 +31,11 @@ public class WikiDirectoryScanService {
     private final WikiProperties properties;
 
     private static final Set<String> SUPPORTED_EXTENSIONS = Set.of(
-            "txt", "md", "pdf", "docx", "doc", "pptx", "xlsx"
+            "txt", "md", "csv", "pdf", "docx", "doc",
+            "pptx", "ppt", "xlsx", "xls", "html", "htm"
     );
 
-    private static final Set<String> TEXT_EXTENSIONS = Set.of("txt", "md");
+    private static final Set<String> TEXT_EXTENSIONS = Set.of("txt", "md", "csv");
 
     /**
      * 扫描结果
@@ -144,8 +145,9 @@ public class WikiDirectoryScanService {
                     String sourceType = switch (ext) {
                         case "pdf" -> "pdf";
                         case "docx", "doc" -> "docx";
-                        case "pptx" -> "pptx";
+                        case "pptx", "ppt" -> "pptx";
-                        case "xlsx" -> "xlsx";
+                        case "xlsx", "xls" -> "xlsx";
+                        case "html", "htm" -> "html";
                         default -> "text";
                     };
                     rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));

mateclaw-server/src/main/resources/docs/en/wiki.md

@@ -37,7 +37,7 @@ MateClaw's LLM Wiki **is the same idea, raised into a product**:
 
 A knowledge base is three layers stacked on top of each other:
 
-1. **Raw material** — the files you dropped in. PDF, DOCX, plain text, markdown, or a whole local directory scanned in one go. The system keeps them intact; any claim in the Wiki traces back to the passage that produced it.
+1. **Raw material** — the files you dropped in. PDF, Word, Excel, PowerPoint, HTML, markdown, plain text (incl. CSV), or a whole local directory scanned in one go. The system keeps them intact; any claim in the Wiki traces back to the passage that produced it.
 2. **Wiki pages** — structured articles the AI writes from the raw material. Each page has a title, a summary, a body, bidirectional links to related pages (`[[like this]]`, plus the alias form `[[target|display text]]`), and provenance pointers back into the raw layer.
 3. **Agent surface** — when an agent calls a wiki tool, the system auto-injects the summaries of relevant pages into the prompt. Bodies are fetched on demand. Agents don't read raw files. They read the library.
 
@@ -51,7 +51,7 @@ This matters because the agent's context window stops getting wasted on re-readi
 
 Once it exists, add material:
 
-- **Upload files** — drag PDFs, DOCX, markdown, or plain text into the upload area. Each file becomes a raw material row.
+- **Upload files** — drag PDF, Word, Excel, PowerPoint, HTML, markdown, or plain-text (incl. CSV) files into the upload area. Each file becomes a raw material row.
 - **Scan a local directory** — desktop only. Point at a folder and MateClaw walks it recursively, respecting `.gitignore`, importing everything that looks like text.
 - **Paste text** — for short excerpts or conversation transcripts.
 

mateclaw-server/src/main/resources/docs/zh/wiki.md

@@ -37,7 +37,7 @@ MateClaw 的 LLM Wiki **是同一个想法长成的产品**：
 
 一个知识库是三层结构叠起来的：
 
-1. **原始材料层**——你扔进去的文件。PDF、DOCX、纯文本、Markdown，或者桌面端扫描整个本地目录。系统保留原文不动；Wiki 里的任何一句话都能回溯到它出自哪段原文。
+1. **原始材料层**——你扔进去的文件。PDF、Word、Excel、PowerPoint、HTML、Markdown、纯文本（含 CSV），或者桌面端扫描整个本地目录。系统保留原文不动；Wiki 里的任何一句话都能回溯到它出自哪段原文。
 2. **Wiki 页面层**——AI 从原始材料里写出的结构化文章。每一页有标题、摘要、正文、指向相关页面的双向链接（`[[像这样]]`，也支持 `[[target|展示文字]]` alias 形式）、以及通往原文的来源指针。
 3. **Agent 表层**——Agent 调用 wiki 工具时，系统会把相关页面的摘要自动注入 prompt，正文按需读取。Agent **不读原文**，它读这本书。
 
@@ -51,7 +51,7 @@ MateClaw 的 LLM Wiki **是同一个想法长成的产品**：
 
 建好之后加材料：
 
-- **上传文件**——把 PDF、DOCX、Markdown、纯文本拖进上传区。每个文件成为一条 raw material。
+- **上传文件**——把 PDF、Word、Excel、PowerPoint、HTML、Markdown、纯文本（含 CSV）拖进上传区。每个文件成为一条 raw material。
 - **扫描本地目录**——桌面端专属。指一个文件夹，MateClaw 递归走完整个树，尊重 `.gitignore`，把能读出文本的全都导入。
 - **粘贴文本**——适合短片段或对话记录。
 

mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue

@@ -32,10 +32,10 @@
             <template v-else-if="isDragging">{{ t('wiki.dropToUpload') }}</template>
             <template v-else>{{ t('wiki.dropFiles') }}</template>
           </span>
-          <span class="upload-hint">.txt, .md, .pdf, .docx</span>
+          <span class="upload-hint">.txt .md .csv .pdf .docx .xlsx .pptx .html</span>
         </div>
       </div>
-      <input ref="fileInput" type="file" style="display:none" accept=".txt,.md,.pdf,.docx,.doc" multiple @change="handleFileSelect" />
+      <input ref="fileInput" type="file" style="display:none" accept=".txt,.md,.csv,.pdf,.docx,.doc,.xlsx,.xls,.pptx,.ppt,.html,.htm" multiple @change="handleFileSelect" />
       <button class="btn-secondary add-text-btn" @click="showAddText = true">
         <svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
           <line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/>

pom.xml

@@ -53,7 +53,10 @@
         <tyrus.version>2.2.2</tyrus.version>
 
         <!-- Document, wiki, and rendering toolchain -->
-        <poi.version>5.4.1</poi.version>
+        <!-- POI must stay aligned with the POI version tika-parser-microsoft-module
+             pulls in transitively (poi-ooxml-full / poi-scratchpad). A skew between
+             poi-ooxml-lite and poi-ooxml-full silently breaks XSSF (xlsx) parsing. -->
+        <poi.version>5.5.1</poi.version>
         <batik.version>1.19</batik.version>
         <jsoup.version>1.22.2</jsoup.version>
         <tika.version>3.3.0</tika.version>