From 4dc280a3d104a38da2110bfe34ba88883efb4857 Mon Sep 17 00:00:00 2001 From: matevip Date: Sat, 2 May 2026 19:04:39 +0800 Subject: [PATCH] feat(wiki): wire image uploads to vision-in pipeline --- .../mate/wiki/controller/WikiController.java | 6 +- .../wiki/model/WikiRawMaterialEntity.java | 7 +- .../wiki/service/WikiRawMaterialService.java | 99 ++++++++++++++++++- .../h2/V80__wiki_raw_material_mime_type.sql | 6 ++ .../V80__wiki_raw_material_mime_type.sql | 21 ++++ 5 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V80__wiki_raw_material_mime_type.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V80__wiki_raw_material_mime_type.sql diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java index 9b6a6dd3..b765f46d 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java @@ -239,11 +239,14 @@ public class WikiController { ? originalName.substring(originalName.lastIndexOf(".") + 1).toLowerCase() : "txt"; - // 确定 sourceType + // Resolve source type from extension. Image extensions route to the + // vision-in pipeline at extraction time; everything else falls through + // to the existing text / pdf / docx handling. String sourceType = switch (extension) { case "pdf" -> "pdf"; case "docx", "doc" -> "docx"; case "txt", "md" -> "text"; + case "png", "jpg", "jpeg", "webp", "gif", "bmp", "tiff", "tif" -> "image"; default -> "text"; }; @@ -258,6 +261,7 @@ public class WikiController { Path targetPath = uploadDir.resolve(System.currentTimeMillis() + "_" + originalName); file.transferTo(targetPath); return R.ok(rawService.addFile(kbId, originalName, sourceType, + file.getContentType(), targetPath.toString(), file.getSize())); } } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java index c875524c..06bcdbba 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java @@ -23,10 +23,13 @@ public class WikiRawMaterialEntity { /** 材料标题 */ private String title; - /** 来源类型：text / pdf / docx / url / paste */ + /** Source type: text / pdf / docx / image / url / paste. */ private String sourceType; - /** 原始文件路径（二进制文件） */ + /** Original Content-Type from the upload (e.g. {@code image/png}); null for text. */ + private String mimeType; + + /** Original file path on disk (binary uploads only). */ private String sourcePath; /** 原始文本内容（文本类型） */ diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java index a94d071b..4baaf2d5 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java @@ -9,6 +9,9 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import vip.mate.tool.builtin.DocumentExtractTool; +import vip.mate.tool.image.vision.ImageVisionService; +import vip.mate.tool.image.vision.VisionRequest; +import vip.mate.tool.image.vision.VisionResult; import vip.mate.wiki.WikiProperties; import vip.mate.wiki.event.WikiProcessingEvent; import vip.mate.wiki.model.WikiRawMaterialEntity; @@ -38,6 +41,7 @@ public class WikiRawMaterialService { private final DocumentExtractTool documentExtractTool; /** RFC-013：删除时级联清理 chunk */ private final WikiChunkService chunkService; + private final ImageVisionService imageVisionService; /** * RFC-012 follow-up #3：从 partial 状态触发的 reprocess 会在此 set 中打标， @@ -119,15 +123,33 @@ public class WikiRawMaterialService { } /** - * 添加文件类型的原始材料（PDF/DOCX 等） + * Adds a file-type raw material (PDF / DOCX / image / ...). + * + *

Backwards-compatible overload that omits the MIME type. Callers + * with the upload Content-Type in hand should prefer the four-argument + * variant — image-routing in particular needs an authoritative MIME so + * downstream vision providers know what they are decoding. */ @Transactional public WikiRawMaterialEntity addFile(Long kbId, String title, String sourceType, String sourcePath, long fileSize) { + return addFile(kbId, title, sourceType, null, sourcePath, fileSize); + } + + /** + * Adds a file-type raw material with explicit MIME type. + * + * @param mimeType Content-Type string from the upload (e.g. {@code image/png}); + * may be null if unknown + */ + @Transactional + public WikiRawMaterialEntity addFile(Long kbId, String title, String sourceType, + String mimeType, String sourcePath, long fileSize) { WikiRawMaterialEntity entity = new WikiRawMaterialEntity(); entity.setKbId(kbId); entity.setTitle(title); entity.setSourceType(sourceType); + entity.setMimeType(mimeType); entity.setSourcePath(sourcePath); entity.setFileSize(fileSize); entity.setProcessingStatus("pending"); @@ -314,6 +336,13 @@ public class WikiRawMaterialService { if ("text".equals(entity.getSourceType())) { return entity.getOriginalContent(); } + // Image source: route through the vision-in pipeline. Failures (feature + // flag off, no provider configured, all providers failed) degrade to an + // empty caption rather than blocking the upload — the user keeps the + // raw row and can retry once vision is configured. + if ("image".equals(entity.getSourceType())) { + return extractTextFromImage(entity); + } // 二进制文件：调用 DocumentExtractTool 提取 if (entity.getSourcePath() != null && !entity.getSourcePath().isBlank()) { try { @@ -344,6 +373,74 @@ public class WikiRawMaterialService { return entity.getOriginalContent(); } + /** + * Routes an image-typed raw material through the vision-in pipeline, + * caches the resulting caption into {@code extracted_text} so the next + * call short-circuits, and degrades gracefully on failure. + * + *

Failure modes (feature flag off, no provider, all providers failed, + * IO errors reading the image bytes) are intentionally swallowed and + * surfaced as the empty string. The upload still succeeded; the raw + * row remains and downstream code is expected to tolerate "no + * extracted text yet" — calling this method again later (e.g. after + * an operator enables the feature flag) re-runs the pipeline. + */ + private String extractTextFromImage(WikiRawMaterialEntity entity) { + if (entity.getSourcePath() == null || entity.getSourcePath().isBlank()) { + log.warn("[Wiki] Image raw material missing sourcePath: id={}", entity.getId()); + return ""; + } + byte[] imageBytes; + try { + imageBytes = java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(entity.getSourcePath())); + } catch (Exception e) { + log.error("[Wiki] Failed to read image bytes for id={}: {}", entity.getId(), e.getMessage()); + return ""; + } + + VisionRequest request = VisionRequest.builder() + .imageBytes(imageBytes) + .mimeType(resolveMimeType(entity)) + .build(); + VisionResult result; + try { + result = imageVisionService.caption(request); + } catch (Exception e) { + log.warn("[Wiki] Vision pipeline failed for raw id={}: {}", entity.getId(), e.getMessage()); + return ""; + } + + StringBuilder text = new StringBuilder(result.getCaption() == null ? "" : result.getCaption()); + if (result.getVisibleText() != null && !result.getVisibleText().isBlank()) { + text.append("\n\n--- Visible text ---\n").append(result.getVisibleText()); + } + String combined = text.toString(); + updateExtractedText(entity.getId(), combined); + log.info("[Wiki] Image vision captioned raw id={} provider={} model={} chars={}", + entity.getId(), result.getProviderId(), result.getModel(), combined.length()); + return combined; + } + + /** Best-effort MIME resolution: prefer the persisted column, fall back to file extension. */ + private static String resolveMimeType(WikiRawMaterialEntity entity) { + if (entity.getMimeType() != null && !entity.getMimeType().isBlank()) { + return entity.getMimeType(); + } + String path = entity.getSourcePath() == null ? "" : entity.getSourcePath().toLowerCase(); + int dot = path.lastIndexOf('.'); + if (dot < 0) return "application/octet-stream"; + String ext = path.substring(dot + 1); + return switch (ext) { + case "png" -> "image/png"; + case "jpg", "jpeg" -> "image/jpeg"; + case "webp" -> "image/webp"; + case "gif" -> "image/gif"; + case "bmp" -> "image/bmp"; + case "tiff", "tif" -> "image/tiff"; + default -> "application/octet-stream"; + }; + } + /** * Recover raw materials stuck in 'processing' status after a server restart. * Resets them to 'pending', clears stale progress fields, and optionally diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V80__wiki_raw_material_mime_type.sql b/mateclaw-server/src/main/resources/db/migration/h2/V80__wiki_raw_material_mime_type.sql new file mode 100644 index 00000000..7a00d0b7 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V80__wiki_raw_material_mime_type.sql @@ -0,0 +1,6 @@ +-- Adds the MIME type column to mate_wiki_raw_material so the upload pipeline +-- can route uploads to the right downstream extractor. Image source types +-- in particular need the original Content-Type to pick a vision provider +-- and to render previews correctly without re-sniffing the file. + +ALTER TABLE mate_wiki_raw_material ADD COLUMN IF NOT EXISTS mime_type VARCHAR(64); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V80__wiki_raw_material_mime_type.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V80__wiki_raw_material_mime_type.sql new file mode 100644 index 00000000..5bc34f29 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V80__wiki_raw_material_mime_type.sql @@ -0,0 +1,21 @@ +-- Adds the MIME type column to mate_wiki_raw_material so the upload pipeline +-- can route uploads to the right downstream extractor. Image source types +-- in particular need the original Content-Type to pick a vision provider +-- and to render previews correctly without re-sniffing the file. +-- +-- MySQL does not support `ADD COLUMN IF NOT EXISTS`, so we guard via +-- INFORMATION_SCHEMA + a prepared statement to keep the migration +-- idempotent across re-runs and partial failures. + +SET @sql = ( + SELECT IF(COUNT(*) = 0, + 'ALTER TABLE mate_wiki_raw_material ADD COLUMN mime_type VARCHAR(64)', + 'SELECT 1') + FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_schema = DATABASE() + AND table_name = 'mate_wiki_raw_material' + AND column_name = 'mime_type' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt;