diff --git a/mateclaw-server/pom.xml b/mateclaw-server/pom.xml index 7bcd5cfd..ee3a6b3f 100644 --- a/mateclaw-server/pom.xml +++ b/mateclaw-server/pom.xml @@ -296,8 +296,6 @@ Used by WikiContentNormalizer to strip nav/footer/script/style/aside and ad-class nodes from URL/HTML uploads before chunking. Small (~430KB), no transitive deps, JVM-only — safe for the desktop bundle. - Tika is intentionally not pulled in: DocumentExtractTool already - covers PDF/Office via pdftotext, pdfplumber, and Java fallbacks. --> org.jsoup @@ -305,6 +303,35 @@ 1.18.3 + + + + org.apache.tika + tika-core + 3.0.0 + + + org.apache.tika + tika-parser-pdf-module + 3.0.0 + + + org.apache.tika + tika-parser-microsoft-module + 3.0.0 + + org.flywaydb diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java index eb7ae193..b75b9b1a 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java @@ -222,15 +222,26 @@ public class DocumentExtractTool { } // attempts 已由 tryOcrExtract 内部记录失败原因 + // 5. Tika 兜底(RFC-051 §5.2):所有命令行 / Python / PDFBox / OCR 都失败时 + // 用 Java 内置的 Tika 再试一次。主要服务于 Windows 没装 Poppler / Python 的桌面用户。 + long t4 = System.currentTimeMillis(); + content = TikaExtractor.extract(path); + if (content != null && !content.isBlank()) { + attempts.add("tika: 成功 (" + (System.currentTimeMillis() - t4) + "ms)"); + int pages = realPageCount > 0 ? realPageCount : estimatePages(content); + return new ExtractedContent(content, "tika", pages); + } + attempts.add("tika: 失败或不可用"); + // 返回之前级别的部分结果(如果有) if (bestContent != null) { - log.warn("[DocumentExtract] OCR 不可用,返回部分文本结果: method={}, length={}", + log.warn("[DocumentExtract] OCR/Tika 不可用,返回部分文本结果: method={}, length={}", bestMethod, bestContent.strip().length()); int pages = realPageCount > 0 ? realPageCount : estimatePages(bestContent); return new ExtractedContent(bestContent, bestMethod + "_partial", pages); } - throw new Exception("所有 PDF 提取方法都失败(包括 OCR)"); + throw new Exception("所有 PDF 提取方法都失败(包括 OCR 与 Tika)"); } /** @@ -556,7 +567,17 @@ public class DocumentExtractTool { } attempts.add("java_zip_xml: 失败"); - throw new Exception("所有 DOCX 提取方法都失败"); + // 5. Tika 兜底(RFC-051 §5.2)—— 当 textutil/pandoc/libreoffice/ZIP-XML 全失败时。 + // Tika 的 Microsoft 模块覆盖到 .docx 内嵌 SmartArt、批注、复杂表格等场景,正好填补 + // 我们手写的 ZIP XML 解析器的盲区。 + content = TikaExtractor.extract(path); + if (content != null && !content.isBlank()) { + attempts.add("tika: 成功"); + return new ExtractedContent(content, "tika", 0); + } + attempts.add("tika: 失败或不可用"); + + throw new Exception("所有 DOCX 提取方法都失败(包括 Tika)"); } private String tryTextutil(Path path) { @@ -687,6 +708,17 @@ public class DocumentExtractTool { } } + // Our ZIP-XML extractor only reads tags and skips the shared-strings table, + // so cells full of text labels look "empty". When that happens, fall through to + // Tika which knows how to resolve the shared-strings indirection. + if (text.toString().replaceAll("---.*?---", "").strip().isEmpty()) { + String fallback = TikaExtractor.extract(path); + if (fallback != null && !fallback.isBlank()) { + attempts.add("tika: 成功(ZIP-XML 仅有数字 / 共享字符串未解析)"); + return new ExtractedContent(fallback, "tika", 0); + } + } + attempts.add("java_zip_xml: 成功"); return new ExtractedContent(text.toString(), "java_zip_xml", 0); } @@ -721,6 +753,17 @@ public class DocumentExtractTool { } } + // Slide layouts with text inside SmartArt / charts / grouped shapes don't surface + // through the simple grep — Tika walks the full DrawingML graph and pulls + // them out. Only invoke when our walker produced nothing useful. + if (text.toString().replaceAll("---.*?---", "").strip().isEmpty()) { + String fallback = TikaExtractor.extract(path); + if (fallback != null && !fallback.isBlank()) { + attempts.add("tika: 成功(ZIP-XML 未抓到正文,可能是 SmartArt / 图表)"); + return new ExtractedContent(fallback, "tika", Math.max(0, slideNum - 1)); + } + } + attempts.add("java_zip_xml: 成功"); return new ExtractedContent(text.toString(), "java_zip_xml", Math.max(0, slideNum - 1)); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/TikaExtractor.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/TikaExtractor.java new file mode 100644 index 00000000..1965adc0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/TikaExtractor.java @@ -0,0 +1,91 @@ +package vip.mate.tool.builtin; + +import lombok.extern.slf4j.Slf4j; +import org.apache.tika.exception.WriteLimitReachedException; +import org.apache.tika.metadata.Metadata; +import org.apache.tika.parser.AutoDetectParser; +import org.apache.tika.parser.ParseContext; +import org.apache.tika.sax.BodyContentHandler; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * RFC-051 §5.2: Apache Tika as the last-resort document extractor. + *

+ * Used by {@link DocumentExtractTool} only after every other path + * (pdftotext / pdfplumber / pdfbox / OCR for PDFs, and the system-command + * + ZIP-XML chain for Office formats) has failed. Tika ships its own + * PDFBox + POI internals, so it works on Windows installs without Python + * or Poppler — which is the actual scenario the RFC §13.1 pointed to. + * + *

Safety

+ *
    + *
  • {@link BodyContentHandler} caps output at {@code maxChars}; when the + * cap is hit Tika throws {@link WriteLimitReachedException}, which we + * treat as a successful (truncated) extract rather than a failure.
  • + *
  • Tika 3.x has built-in zip-bomb defenses on its zip readers (POI's + * {@code ZipSecureFile}); we don't disable them.
  • + *
  • Any other parse failure returns {@code null} so the caller can fall + * through to its existing structured-error path.
  • + *
+ * + * The extractor is deliberately stateless and synchronous: callers drive + * concurrency externally. + */ +@Slf4j +public final class TikaExtractor { + + /** + * Reasonable default for a single-document parse. 5MB of text is + * well above any source we'd actually feed into the wiki pipeline, + * and well below what would OOM a typical desktop install. + */ + public static final int DEFAULT_MAX_CHARS = 5_000_000; + + private TikaExtractor() {} + + /** Extract with the default cap. */ + public static String extract(Path path) { + return extract(path, DEFAULT_MAX_CHARS); + } + + /** + * Extract text from {@code path} using Tika's {@link AutoDetectParser}, + * capping output at {@code maxChars}. Returns the extracted text on + * success (possibly truncated), or {@code null} on any failure. + */ + public static String extract(Path path, int maxChars) { + if (path == null) return null; + if (!Files.isRegularFile(path)) { + log.debug("[Tika] Path is not a regular file: {}", path); + return null; + } + int cap = maxChars <= 0 ? DEFAULT_MAX_CHARS : maxChars; + + BodyContentHandler handler = new BodyContentHandler(cap); + AutoDetectParser parser = new AutoDetectParser(); + Metadata metadata = new Metadata(); + ParseContext context = new ParseContext(); + + try (InputStream is = Files.newInputStream(path)) { + parser.parse(is, handler, metadata, context); + return handler.toString(); + } catch (WriteLimitReachedException truncated) { + // Cap hit — Tika filled the handler before parsing finished. The + // partial text is still useful, especially since callers will chunk + // anyway and only want the leading prose for routing/embedding. + String partial = handler.toString(); + log.info("[Tika] Output cap reached at {} chars for {}; returning partial", + partial.length(), path.getFileName()); + return partial.isBlank() ? null : partial; + } catch (Throwable t) { + // Catching Throwable on purpose: Tika can throw NoClassDefFoundError / + // LinkageError when an obscure transitive parser is missing on a + // minimal classpath, and that should not crash the extract chain. + log.warn("[Tika] Parse failed for {}: {}", path.getFileName(), t.getMessage()); + return null; + } + } +}