opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(wiki): support HTML, Excel, PowerPoint and CSV raw materials

Browse Source
This commit is contained in:
matevip 2026-05-18 21:29:56 +08:00
parent b763022810
commit 33a40ad9d9
8 changed files with 86 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java
View File

@ -20,7 +20,7 @@ import java.util.zip.ZipInputStream;
/**
* Document text extraction tool.
* Supports PDF, DOCX, XLSX, PPTX with format-specific fallback chains.
* Supports PDF, DOCX, XLSX, PPTX, HTML with format-specific fallback chains.
*
* Strategy by format:
* - PDF: pdftotext -> pdfplumber/pypdf -> pdfbox -> OCR (scanned) -> Tika
@ -28,6 +28,8 @@ import java.util.zip.ZipInputStream;
* - XLSX/PPTX: Tika directly (POI-based; correctly resolves the shared-strings
* indirection table and walks SmartArt / chart / grouped-shape
* text that a naive ZIP+XML scan misses).
* - HTML: jsoup parse -> drop script/style/nav/footer noise -> keep
* heading hierarchy as Markdown ATX lines.
*/
@Slf4j
@Component
@ -46,11 +48,13 @@ public class DocumentExtractTool {
- Word (.docx, .doc)
- Excel (.xlsx, .xls) - 提取为文本表格
- PowerPoint (.pptx, .ppt)
- HTML (.html, .htm) - jsoup 清洗后提取正文
提取策略（按格式分链）：
- PDF: pdftotext → pdfplumber/pypdf → pdfbox → OCR（扫描版） → Tika
- DOCX: textutil / pandoc / libreoffice → ZIP-XML → Tika
- XLSX/PPTX: 直接走 Tika（基于 POI，正确解析 sharedStrings 表与 SmartArt / 图表文本）
- HTML: jsoup 解析 → 去除 script/style/nav/footer 等噪音 → 保留标题层级
- 返回详细的提取过程和元数据
参数 options 可包含：
@ -134,6 +138,8 @@ public class DocumentExtractTool {
content = extractXlsx(path, options, attempts);
} else if (mimeType.contains("presentationml") || mimeType.contains("powerpoint")) {
content = extractPptx(path, options, attempts);
} else if (mimeType.contains("html")) {
content = extractHtml(path, attempts);
} else {
return errorResult(filePath, "不支持的文档类型: " + mimeType, attempts);
}
@ -869,6 +875,53 @@ public class DocumentExtractTool {
return count;
}
// ==================== HTML 提取 ====================
/**
* Extract readable text from an HTML file with jsoup.
* <p>
* Drops structural noise (script / style / nav / header / footer / aside /
* form / iframe), then walks the surviving elements emitting headings as
* Markdown ATX lines ({@code # }, {@code ## } …) so the wiki preprocessor
* can still detect the document's heading hierarchy. The charset is
* auto-detected from the BOM / {@code <meta charset>} declaration.
*/
private ExtractedContent extractHtml(Path path, List<String> attempts) throws Exception {
long t = System.currentTimeMillis();
org.jsoup.nodes.Document doc;
try {
// charsetName = null lets jsoup sniff the encoding from BOM / meta tag.
doc = org.jsoup.Jsoup.parse(path.toFile(), null);
} catch (IOException e) {
attempts.add("jsoup: 读取失败 - " + e.getMessage());
throw new Exception("HTML 文件读取失败: " + e.getMessage());
}
doc.select("script, style, noscript, nav, header, footer, aside, form, iframe").remove();
StringBuilder sb = new StringBuilder();
org.jsoup.nodes.Element root = doc.body() != null ? doc.body() : doc;
for (org.jsoup.nodes.Element el : root.getAllElements()) {
String text = el.ownText();
if (text.isBlank()) continue;
String tag = el.tagName();
if (tag.length() == 2 && tag.charAt(0) == 'h' && tag.charAt(1) >= '1' && tag.charAt(1) <= '6') {
int level = tag.charAt(1) - '0';
sb.append('\n').append("#".repeat(level)).append(' ').append(text.trim()).append('\n');
} else {
sb.append(text.trim()).append('\n');
}
}
String out = sb.toString().strip();
if (out.isBlank()) {
attempts.add("jsoup: 解析成功但无可读文本 (" + (System.currentTimeMillis() - t) + "ms)");
throw new Exception("HTML 提取无文本（页面可能仅含脚本 / 样式）");
}
attempts.add("jsoup: 成功 (" + (System.currentTimeMillis() - t) + "ms)");
return new ExtractedContent(out, "jsoup", 0);
}
// ==================== 工具方法 ====================
/**
@ -938,6 +991,7 @@ public class DocumentExtractTool {
if (fileName.endsWith(".xls")) return "application/vnd.ms-excel";
if (fileName.endsWith(".pptx")) return "application/vnd.openxmlformats-officedocument.presentationml.presentation";
if (fileName.endsWith(".ppt")) return "application/vnd.ms-powerpoint";
if (fileName.endsWith(".html") || fileName.endsWith(".htm")) return "text/html";
return "application/octet-stream";
}

11
mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java
View File

@ -263,12 +263,17 @@ public class WikiController {
: "txt";
// Resolve source type from extension. Image extensions route to the
// vision-in pipeline at extraction time; everything else falls through
// to the existing text / pdf / docx handling.
// vision-in pipeline at extraction time; Office / PDF / HTML extensions
// are staged on disk and extracted by DocumentExtractTool; plain-text
// formats (incl. CSV) are stored directly. Unknown extensions fall back
// to text so the upload never hard-fails.
String sourceType = switch (extension) {
case "pdf" -> "pdf";
case "docx", "doc" -> "docx";
case "txt", "md" -> "text";
case "xlsx", "xls" -> "xlsx";
case "pptx", "ppt" -> "pptx";
case "html", "htm" -> "html";
case "txt", "md", "csv" -> "text";
case "png", "jpg", "jpeg", "webp", "gif", "bmp", "tiff", "tif" -> "image";
default -> "text";
};

9
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContentNormalizer.java
View File

@ -46,11 +46,16 @@ public class WikiContentNormalizer {
if (rawText == null) return "";
String type = sourceType == null ? "" : sourceType.toLowerCase();
return switch (type) {
case "url", "html" -> normalizeHtml(rawText);
// 'url' is raw fetched web HTML and still carries markup, so it needs
// tag stripping here. 'html' file uploads are tag-stripped upstream by
// DocumentExtractTool (jsoup) before reaching the normalizer — running
// normalizeHtml again would re-collapse the recovered heading structure,
// so they fall through to the plain-text branch.
case "url" -> normalizeHtml(rawText);
// PDF text from DocumentExtractTool may already contain "--- Page N ---"
// markers; we keep them so the preprocessor can map char offsets to pages.
case "pdf" -> collapseBlankLines(rawText);
case "docx", "pptx", "xlsx" -> collapseBlankLines(rawText);
case "docx", "doc", "pptx", "ppt", "xlsx", "xls", "html", "htm" -> collapseBlankLines(rawText);
case "markdown", "md", "text", "paste" -> collapseBlankLines(rawText);
default -> collapseBlankLines(rawText);
};

10
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java
View File

@ -31,10 +31,11 @@ public class WikiDirectoryScanService {
private final WikiProperties properties;
private static final Set<String> SUPPORTED_EXTENSIONS = Set.of(
"txt", "md", "pdf", "docx", "doc", "pptx", "xlsx"
"txt", "md", "csv", "pdf", "docx", "doc",
"pptx", "ppt", "xlsx", "xls", "html", "htm"
);
private static final Set<String> TEXT_EXTENSIONS = Set.of("txt", "md");
private static final Set<String> TEXT_EXTENSIONS = Set.of("txt", "md", "csv");
/**
* 扫描结果
@ -144,8 +145,9 @@ public class WikiDirectoryScanService {
String sourceType = switch (ext) {
case "pdf" -> "pdf";
case "docx", "doc" -> "docx";
case "pptx" -> "pptx";
case "xlsx" -> "xlsx";
case "pptx", "ppt" -> "pptx";
case "xlsx", "xls" -> "xlsx";
case "html", "htm" -> "html";
default -> "text";
};
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));

4
mateclaw-server/src/main/resources/docs/en/wiki.md
View File

@ -37,7 +37,7 @@ MateClaw's LLM Wiki **is the same idea, raised into a product**:
A knowledge base is three layers stacked on top of each other:
1. **Raw material** — the files you dropped in. PDF, DOCX, plain text, markdown, or a whole local directory scanned in one go. The system keeps them intact; any claim in the Wiki traces back to the passage that produced it.
1. **Raw material** — the files you dropped in. PDF, Word, Excel, PowerPoint, HTML, markdown, plain text (incl. CSV), or a whole local directory scanned in one go. The system keeps them intact; any claim in the Wiki traces back to the passage that produced it.
2. **Wiki pages** — structured articles the AI writes from the raw material. Each page has a title, a summary, a body, bidirectional links to related pages (`[[like this]]`, plus the alias form `[[target|display text]]`), and provenance pointers back into the raw layer.
3. **Agent surface** — when an agent calls a wiki tool, the system auto-injects the summaries of relevant pages into the prompt. Bodies are fetched on demand. Agents don't read raw files. They read the library.
@ -51,7 +51,7 @@ This matters because the agent's context window stops getting wasted on re-readi
Once it exists, add material:
- **Upload files** — drag PDFs, DOCX, markdown, or plain text into the upload area. Each file becomes a raw material row.
- **Upload files** — drag PDF, Word, Excel, PowerPoint, HTML, markdown, or plain-text (incl. CSV) files into the upload area. Each file becomes a raw material row.
- **Scan a local directory** — desktop only. Point at a folder and MateClaw walks it recursively, respecting `.gitignore`, importing everything that looks like text.
- **Paste text** — for short excerpts or conversation transcripts.

4
mateclaw-server/src/main/resources/docs/zh/wiki.md
View File

@ -37,7 +37,7 @@ MateClaw 的 LLM Wiki **是同一个想法长成的产品**：
一个知识库是三层结构叠起来的：
1. **原始材料层**——你扔进去的文件。PDF、DOCX、纯文本、Markdown，或者桌面端扫描整个本地目录。系统保留原文不动；Wiki 里的任何一句话都能回溯到它出自哪段原文。
1. **原始材料层**——你扔进去的文件。PDF、Word、Excel、PowerPoint、HTML、Markdown、纯文本（含 CSV），或者桌面端扫描整个本地目录。系统保留原文不动；Wiki 里的任何一句话都能回溯到它出自哪段原文。
2. **Wiki 页面层**——AI 从原始材料里写出的结构化文章。每一页有标题、摘要、正文、指向相关页面的双向链接（`[[像这样]]`，也支持 `[[target|展示文字]]` alias 形式）、以及通往原文的来源指针。
3. **Agent 表层**——Agent 调用 wiki 工具时，系统会把相关页面的摘要自动注入 prompt，正文按需读取。Agent **不读原文**，它读这本书。
@ -51,7 +51,7 @@ MateClaw 的 LLM Wiki **是同一个想法长成的产品**：
建好之后加材料：
- **上传文件**——把 PDF、DOCX、Markdown、纯文本拖进上传区。每个文件成为一条 raw material。
- **上传文件**——把 PDF、Word、Excel、PowerPoint、HTML、Markdown、纯文本（含 CSV）拖进上传区。每个文件成为一条 raw material。
- **扫描本地目录**——桌面端专属。指一个文件夹，MateClaw 递归走完整个树，尊重 `.gitignore`，把能读出文本的全都导入。
- **粘贴文本**——适合短片段或对话记录。

4
mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue
View File

@ -32,10 +32,10 @@
<template v-else-if="isDragging">{{ t('wiki.dropToUpload') }}</template>
<template v-else>{{ t('wiki.dropFiles') }}</template>
</span>
<span class="upload-hint">.txt, .md, .pdf, .docx</span>
<span class="upload-hint">.txt .md .csv .pdf .docx .xlsx .pptx .html</span>
</div>
</div>
<input ref="fileInput" type="file" style="display:none" accept=".txt,.md,.pdf,.docx,.doc" multiple @change="handleFileSelect" />
<input ref="fileInput" type="file" style="display:none" accept=".txt,.md,.csv,.pdf,.docx,.doc,.xlsx,.xls,.pptx,.ppt,.html,.htm" multiple @change="handleFileSelect" />
<button class="btn-secondary add-text-btn" @click="showAddText = true">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<line x1="12" y1="5" x2="12" y2="19"/><line x1="5" y1="12" x2="19" y2="12"/>

5
pom.xml
View File

@ -53,7 +53,10 @@
<tyrus.version>2.2.2</tyrus.version>
<!-- Document, wiki, and rendering toolchain -->
<poi.version>5.4.1</poi.version>
<!-- POI must stay aligned with the POI version tika-parser-microsoft-module
pulls in transitively (poi-ooxml-full / poi-scratchpad). A skew between
poi-ooxml-lite and poi-ooxml-full silently breaks XSSF (xlsx) parsing. -->
<poi.version>5.5.1</poi.version>
<batik.version>1.19</batik.version>
<jsoup.version>1.22.2</jsoup.version>
<tika.version>3.3.0</tika.version>