From 088ffcaee19a63ec1cf3145a8e96805dd627172e Mon Sep 17 00:00:00 2001 From: matevip Date: Sat, 2 May 2026 19:04:32 +0800 Subject: [PATCH] feat(wiki): add image-to-text vision SPI with DashScope provider --- .../vip/mate/tool/image/ImageCapability.java | 7 +- .../image/vision/ImageVisionProvider.java | 59 +++++ .../tool/image/vision/ImageVisionService.java | 185 +++++++++++++++ .../mate/tool/image/vision/VisionContext.java | 30 +++ .../mate/tool/image/vision/VisionRequest.java | 35 +++ .../mate/tool/image/vision/VisionResult.java | 45 ++++ .../provider/DashScopeVisionProvider.java | 221 ++++++++++++++++++ .../src/main/resources/messages.properties | 6 + .../src/main/resources/messages_en.properties | 6 + .../wiki/vision-caption-context-aware.txt | 11 + .../prompts/wiki/vision-caption-factual.txt | 1 + 11 files changed, 604 insertions(+), 2 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionService.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionContext.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionRequest.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionResult.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java create mode 100644 mateclaw-server/src/main/resources/prompts/wiki/vision-caption-context-aware.txt create mode 100644 mateclaw-server/src/main/resources/prompts/wiki/vision-caption-factual.txt diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java index 93eaa4e4..273e12e4 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageCapability.java @@ -1,7 +1,7 @@ package vip.mate.tool.image; /** - * 图片生成能力枚举 + * 图片相关能力枚举（生成方向 + 解析方向）。 * * @author MateClaw Team */ @@ -11,5 +11,8 @@ public enum ImageCapability { TEXT_TO_IMAGE, /** 图片编辑 / 风格转换 */ - IMAGE_EDIT + IMAGE_EDIT, + + /** Image to text — vision-in pipeline (factual caption + best-effort visible text). */ + IMAGE_TO_TEXT } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionProvider.java new file mode 100644 index 00000000..3bd04175 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionProvider.java @@ -0,0 +1,59 @@ +package vip.mate.tool.image.vision; + +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.ImageCapability; + +import java.util.Set; + +/** + * SPI for image-to-text providers. + * + *

Symmetric counterpart of {@link vip.mate.tool.image.ImageGenerationProvider}. + * Implementations are auto-discovered as Spring beans and selected by + * {@link ImageVisionService} according to {@link #autoDetectOrder()} + * (lower wins) — typically: regional default first, premium fallbacks last. + * + * @author MateClaw Team + */ +public interface ImageVisionProvider { + + /** + * Stable identifier for the SPI registry. + * + *

Convention: {@code -vision} (e.g. {@code dashscope-vision}, + * {@code openai-vision}, {@code claude-vision}). + */ + String id(); + + /** Display label for admin UI. */ + String label(); + + /** Whether the provider needs an API key configured. */ + boolean requiresCredential(); + + /** + * Auto-detect priority. Lower wins. Conventional bands: + *

+ */ + int autoDetectOrder(); + + /** Capabilities advertised; today only {@link ImageCapability#IMAGE_TO_TEXT}. */ + Set capabilities(); + + /** Reports whether the provider is currently usable (key configured + reachable). */ + boolean isAvailable(SystemSettingsDTO settings); + + /** + * Synchronously caption an image. + * + *

Throws on failure — the caller (typically {@link ImageVisionService}) + * decides whether to retry / fall back / surface to user. + * + * @return non-null populated {@link VisionResult} + */ + VisionResult caption(VisionRequest request, SystemSettingsDTO settings); +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionService.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionService.java new file mode 100644 index 00000000..5477e6bf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/ImageVisionService.java @@ -0,0 +1,185 @@ +package vip.mate.tool.image.vision; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.exception.MateClawException; +import vip.mate.system.featureflag.FeatureFlagService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.wiki.metrics.WikiMetrics; +import vip.mate.wiki.model.WikiImageCaptionCacheEntity; +import vip.mate.wiki.service.WikiImageCaptionCacheService; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +/** + * Routing + caching coordinator for the image-to-text pipeline. + * + *

End-to-end flow: + *

    + *
  1. Short-circuit when the {@code wiki.ocr.enabled} feature flag is off.
    2. + *
  2. Hash the image bytes (SHA-256) and probe the caption cache. A cache + * hit returns immediately and bumps the row's {@code hit_count} as a + * side effect.
    3. + *
  3. On miss, walk providers in {@link ImageVisionProvider#autoDetectOrder()} + * order. The first available provider that returns a non-null caption + * wins. Failures are logged and the next provider is tried.
    4. + *
  4. Persist the winning caption to the cache (race-tolerant) and emit + * success metrics.
    5. + *
  5. If every provider fails, throw a {@link MateClawException} carrying + * the i18n key {@code err.wiki.vision.all_failed}.
    6. + *
+ * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class ImageVisionService { + + private static final String FEATURE_FLAG = "wiki.ocr.enabled"; + + private final List providers; + private final WikiImageCaptionCacheService cacheService; + private final SystemSettingService systemSettingService; + private final FeatureFlagService featureFlagService; + private final WikiMetrics metrics; + + /** + * Captions an image, going through cache → provider chain. + * + * @throws MateClawException with key {@code err.wiki.vision.disabled} when the feature flag is off + * @throws MateClawException with key {@code err.wiki.vision.no_provider} when no provider is available + * @throws MateClawException with key {@code err.wiki.vision.all_failed} when every provider failed + */ + public VisionResult caption(VisionRequest request) { + if (request == null || request.getImageBytes() == null + || request.getImageBytes().length == 0) { + throw new IllegalArgumentException("VisionRequest with image bytes is required"); + } + if (!featureFlagService.isEnabled(FEATURE_FLAG)) { + throw new MateClawException("err.wiki.vision.disabled", + "Image vision pipeline is currently disabled"); + } + + String sha256 = sha256Hex(request.getImageBytes()); + + // 1. Cache check. + Optional cached = cacheService.lookup(sha256); + if (cached.isPresent()) { + metrics.recordVisionCacheHit(true); + log.debug("[Vision] cache hit sha={}", shortSha(sha256)); + return fromEntity(cached.get()); + } + metrics.recordVisionCacheHit(false); + + // 2. Provider walk. + SystemSettingsDTO settings = systemSettingService.getSettings(); + List available = providers.stream() + .filter(p -> { + try { + return p.isAvailable(settings); + } catch (Exception e) { + log.debug("[Vision] availability check threw for provider={}: {}", + p.id(), e.getMessage()); + return false; + } + }) + .sorted(Comparator.comparingInt(ImageVisionProvider::autoDetectOrder)) + .toList(); + if (available.isEmpty()) { + throw new MateClawException("err.wiki.vision.no_provider", + "No image vision provider is configured"); + } + + VisionResult winner = null; + Throwable lastError = null; + for (ImageVisionProvider provider : available) { + long startNanos = System.nanoTime(); + try { + winner = provider.caption(request, settings); + metrics.recordVisionCall(provider.id(), true, + Duration.ofNanos(System.nanoTime() - startNanos)); + if (winner != null) { + break; + } + } catch (Exception e) { + metrics.recordVisionCall(provider.id(), false, + Duration.ofNanos(System.nanoTime() - startNanos)); + log.warn("[Vision] provider={} failed sha={}: {}", + provider.id(), shortSha(sha256), e.getMessage()); + lastError = e; + } + } + + if (winner == null) { + String why = lastError != null ? lastError.getMessage() : "all providers returned null"; + throw new MateClawException("err.wiki.vision.all_failed", + "All image vision providers failed: " + why); + } + + // 3. Persist to cache. + cacheService.persist(toEntity(sha256, winner, request.getMimeType())); + + return winner; + } + + // ==================== internal ==================== + + private static VisionResult fromEntity(WikiImageCaptionCacheEntity row) { + Instant captured = row.getCapturedAt() == null + ? Instant.now() + : row.getCapturedAt().atZone(ZoneId.systemDefault()).toInstant(); + return VisionResult.builder() + .caption(row.getCaption()) + .visibleText(row.getVisibleText()) + .providerId(row.getProviderId()) + .model(row.getCaptureModel()) + .capturedAt(captured) + .durationMs(row.getDurationMs() != null ? row.getDurationMs() : 0L) + .build(); + } + + private static WikiImageCaptionCacheEntity toEntity(String sha256, VisionResult result, + String mimeType) { + WikiImageCaptionCacheEntity row = new WikiImageCaptionCacheEntity(); + row.setImageSha256(sha256); + row.setCaption(result.getCaption()); + row.setVisibleText(result.getVisibleText()); + row.setMimeType(mimeType); + row.setCaptureModel(result.getModel()); + row.setProviderId(result.getProviderId()); + row.setDurationMs(result.getDurationMs()); + row.setHitCount(0L); + row.setCapturedAt(result.getCapturedAt() != null + ? LocalDateTime.ofInstant(result.getCapturedAt(), ZoneId.systemDefault()) + : LocalDateTime.now()); + return row; + } + + static String sha256Hex(byte[] bytes) { + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(bytes); + StringBuilder sb = new StringBuilder(64); + for (byte b : digest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable on this JVM", e); + } + } + + private static String shortSha(String sha) { + return sha.substring(0, Math.min(8, sha.length())); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionContext.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionContext.java new file mode 100644 index 00000000..cd3f3805 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionContext.java @@ -0,0 +1,30 @@ +package vip.mate.tool.image.vision; + +import lombok.Builder; +import lombok.Data; + +/** + * Surrounding text fed to the vision provider for context-aware captioning. + * + *

When at least one side is non-blank the provider switches from the + * factual-only prompt to the context-aware prompt, which asks the model + * to flag images that look unrelated to the surrounding text (so callers + * can mark them off-topic in downstream pipelines). + * + * @author MateClaw Team + */ +@Data +@Builder +public class VisionContext { + + /** Up to ~500 chars of text that appeared immediately before the image. */ + private String beforeText; + + /** Up to ~500 chars of text that appeared immediately after the image. */ + private String afterText; + + public boolean hasContext() { + return (beforeText != null && !beforeText.isBlank()) + || (afterText != null && !afterText.isBlank()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionRequest.java new file mode 100644 index 00000000..c1c09caf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionRequest.java @@ -0,0 +1,35 @@ +package vip.mate.tool.image.vision; + +import lombok.Builder; +import lombok.Data; + +/** + * Input to one image-to-text call. + * + *

The image is supplied as raw bytes plus a MIME type; callers are + * responsible for any format conversion before invoking. The optional + * {@link VisionContext} carries surrounding text so the provider can + * decide whether the image is on-topic and produce a context-aware + * caption. + * + * @author MateClaw Team + */ +@Data +@Builder +public class VisionRequest { + + /** Raw image bytes (PNG / JPEG / WebP / ...). Required. */ + private byte[] imageBytes; + + /** MIME type, e.g. {@code image/png}. Required. */ + private String mimeType; + + /** Optional surrounding-text context for context-aware captioning. */ + private VisionContext context; + + /** + * Caller hint: when set, ask the provider for this specific model name. + * Provider may ignore the hint if the model is unsupported. + */ + private String preferModel; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionResult.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionResult.java new file mode 100644 index 00000000..e5aa558e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/VisionResult.java @@ -0,0 +1,45 @@ +package vip.mate.tool.image.vision; + +import lombok.Builder; +import lombok.Data; + +import java.time.Instant; + +/** + * Outcome of one successful image-to-text call. + * + *

Producers must populate {@link #caption}, {@link #providerId}, + * {@link #model} and {@link #capturedAt} at minimum. The other fields + * are optional and allow callers to render richer UI. + * + * @author MateClaw Team + */ +@Data +@Builder +public class VisionResult { + + /** 2-4 sentence factual description of the image (primary output). */ + private String caption; + + /** Best-effort recovery of any text rendered inside the image; may be null. */ + private String visibleText; + + /** + * True when the context-aware prompt determined the image is unrelated + * to the surrounding text. Caller decides what to do with off-topic + * captions (downrank / hide / annotate). False (or null) when context + * was not supplied. + */ + private boolean offTopic; + + /** SPI-registry id of the provider that produced this result. */ + private String providerId; + + /** Vendor-specific model identifier (e.g. {@code qwen-vl-max}). */ + private String model; + + private Instant capturedAt; + + /** Wall-clock duration of the round-trip, in milliseconds. */ + private long durationMs; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java new file mode 100644 index 00000000..fbf6f805 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java @@ -0,0 +1,221 @@ +package vip.mate.tool.image.vision.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.agent.prompt.PromptLoader; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.vision.ImageVisionProvider; +import vip.mate.tool.image.vision.VisionContext; +import vip.mate.tool.image.vision.VisionRequest; +import vip.mate.tool.image.vision.VisionResult; + +import java.time.Duration; +import java.time.Instant; +import java.util.Base64; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * DashScope vision provider — uses qwen-vl-max via the OpenAI-compatible + * endpoint at {@code /compatible-mode/v1/chat/completions}. + * + *

Default for the regional production rollout: API keys are typically + * available (DASHSCOPE_API_KEY is mandatory for the rest of the platform) + * and per-image cost is the lowest of the supported vendors. + * + *

Implementation deliberately uses Hutool {@code HttpRequest} rather + * than building a Spring AI {@code ChatClient} — the call shape is a + * one-shot request/response that doesn't benefit from the heavyweight + * agent-style observability / retry / cache machinery in + * {@code AgentOpenAiCompatibleChatModelBuilder}. + * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DashScopeVisionProvider implements ImageVisionProvider { + + private static final String PROVIDER_ID = "dashscope-vision"; + private static final String DASHSCOPE_PROVIDER_KEY = "dashscope"; + private static final String DEFAULT_MODEL = "qwen-vl-max"; + private static final String DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; + private static final int CALL_TIMEOUT_MS = 60_000; + private static final Pattern OFF_TOPIC_PREFIX = Pattern.compile("^\\s*\\[OFF-TOPIC\\]\\s*", Pattern.CASE_INSENSITIVE); + + private final ModelProviderService modelProviderService; + private final ObjectMapper objectMapper; + + @Override + public String id() { + return PROVIDER_ID; + } + + @Override + public String label() { + return "DashScope qwen-vl"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 10; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.IMAGE_TO_TEXT); + } + + @Override + public boolean isAvailable(SystemSettingsDTO settings) { + try { + return modelProviderService.isProviderConfigured(DASHSCOPE_PROVIDER_KEY); + } catch (Exception e) { + log.debug("[Vision][dashscope] availability check failed: {}", e.getMessage()); + return false; + } + } + + @Override + public VisionResult caption(VisionRequest request, SystemSettingsDTO settings) { + ModelProviderEntity provider; + try { + provider = modelProviderService.getProviderConfig(DASHSCOPE_PROVIDER_KEY); + } catch (Exception e) { + throw new MateClawException("err.wiki.vision.no_provider", + "DashScope provider is not configured"); + } + String apiKey = provider.getApiKey(); + if (apiKey == null || apiKey.isBlank()) { + throw new MateClawException("err.wiki.vision.no_provider", + "DashScope API key is missing"); + } + String baseUrl = (provider.getBaseUrl() == null || provider.getBaseUrl().isBlank()) + ? DEFAULT_BASE_URL + : provider.getBaseUrl(); + String model = (request.getPreferModel() == null || request.getPreferModel().isBlank()) + ? DEFAULT_MODEL + : request.getPreferModel(); + + String prompt = buildPrompt(request.getContext()); + String imageDataUrl = buildDataUrl(request.getMimeType(), request.getImageBytes()); + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("temperature", 0.0); + body.put("max_tokens", 4096); + + ArrayNode messages = body.putArray("messages"); + ObjectNode userMsg = messages.addObject(); + userMsg.put("role", "user"); + ArrayNode content = userMsg.putArray("content"); + content.addObject().put("type", "text").put("text", prompt); + ObjectNode imagePart = content.addObject(); + imagePart.put("type", "image_url"); + imagePart.putObject("image_url").put("url", imageDataUrl); + + long startNanos = System.nanoTime(); + HttpResponse response; + try { + response = HttpRequest.post(baseUrl + "/chat/completions") + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(CALL_TIMEOUT_MS) + .execute(); + } catch (Exception e) { + throw new MateClawException("err.wiki.vision.provider_failed", + "DashScope vision call failed: " + e.getMessage()); + } + long durationMs = Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); + + if (response.getStatus() != 200) { + String errBody = response.body(); + log.warn("[Vision][dashscope] HTTP {} body={}", response.getStatus(), truncate(errBody, 400)); + throw new MateClawException("err.wiki.vision.provider_failed", + "DashScope vision returned HTTP " + response.getStatus()); + } + + String text; + try { + JsonNode root = objectMapper.readTree(response.body()); + text = root.path("choices").path(0).path("message").path("content").asText("").trim(); + } catch (Exception e) { + throw new MateClawException("err.wiki.vision.provider_failed", + "DashScope vision response was unparseable: " + e.getMessage()); + } + if (text.isEmpty()) { + throw new MateClawException("err.wiki.vision.provider_failed", + "DashScope vision returned an empty caption"); + } + + Matcher offTopic = OFF_TOPIC_PREFIX.matcher(text); + boolean isOffTopic = offTopic.find(); + String caption = isOffTopic ? offTopic.replaceFirst("").trim() : text; + + return VisionResult.builder() + .caption(caption) + .visibleText(extractVisibleTextHeuristic(caption)) + .offTopic(isOffTopic) + .providerId(PROVIDER_ID) + .model(model) + .capturedAt(Instant.now()) + .durationMs(durationMs) + .build(); + } + + private String buildPrompt(VisionContext context) { + if (context != null && context.hasContext()) { + String template = PromptLoader.loadPrompt("wiki/vision-caption-context-aware"); + String before = (context.getBeforeText() == null || context.getBeforeText().isBlank()) + ? "(none)" : context.getBeforeText().trim(); + String after = (context.getAfterText() == null || context.getAfterText().isBlank()) + ? "(none)" : context.getAfterText().trim(); + return template.replace("{before}", before).replace("{after}", after); + } + return PromptLoader.loadPrompt("wiki/vision-caption-factual"); + } + + private static String buildDataUrl(String mimeType, byte[] bytes) { + String mime = (mimeType == null || mimeType.isBlank()) ? "image/png" : mimeType; + return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(bytes); + } + + /** + * Heuristic: pull obvious quoted strings as "visible text". This is a + * placeholder until a stronger OCR-grade extraction lands; for the + * factual prompt the model already inlines text verbatim into the + * caption, so the cache row's caption alone is enough for retrieval. + */ + private static String extractVisibleTextHeuristic(String caption) { + Matcher m = Pattern.compile("\"([^\"]{3,})\"").matcher(caption); + StringBuilder sb = new StringBuilder(); + while (m.find()) { + if (sb.length() > 0) sb.append('\n'); + sb.append(m.group(1)); + } + return sb.length() == 0 ? null : sb.toString(); + } + + private static String truncate(String s, int max) { + if (s == null) return ""; + return s.length() > max ? s.substring(0, max) + "…" : s; + } +} diff --git a/mateclaw-server/src/main/resources/messages.properties b/mateclaw-server/src/main/resources/messages.properties index d05a471e..51fb2682 100644 --- a/mateclaw-server/src/main/resources/messages.properties +++ b/mateclaw-server/src/main/resources/messages.properties @@ -273,3 +273,9 @@ agent.limit_exceeded.empty_context=\uff08\u5c1a\u672a\u6536\u96c6\u5230\u5de5\u5 cron.tasks_conversation.title=📋 定时任务 cron.run_header.scheduled=定时触发 cron.run_header.manual=手动触发 + +# --- Wiki vision-in pipeline --- +err.wiki.vision.disabled=图片识别功能未启用 +err.wiki.vision.no_provider=未配置可用的图片识别 provider +err.wiki.vision.provider_failed=图片识别 provider 调用失败 +err.wiki.vision.all_failed=所有图片识别 provider 调用失败 diff --git a/mateclaw-server/src/main/resources/messages_en.properties b/mateclaw-server/src/main/resources/messages_en.properties index 69fcb9d9..3802bce9 100644 --- a/mateclaw-server/src/main/resources/messages_en.properties +++ b/mateclaw-server/src/main/resources/messages_en.properties @@ -280,3 +280,9 @@ agent.limit_exceeded.empty_context=(No tool call results collected yet.) cron.tasks_conversation.title=📋 Scheduled Tasks cron.run_header.scheduled=scheduled cron.run_header.manual=manual + +# --- Wiki vision-in pipeline --- +err.wiki.vision.disabled=Image vision pipeline is currently disabled +err.wiki.vision.no_provider=No image vision provider is configured +err.wiki.vision.provider_failed=Image vision provider call failed +err.wiki.vision.all_failed=All image vision providers failed diff --git a/mateclaw-server/src/main/resources/prompts/wiki/vision-caption-context-aware.txt b/mateclaw-server/src/main/resources/prompts/wiki/vision-caption-context-aware.txt new file mode 100644 index 00000000..64ac2903 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/vision-caption-context-aware.txt @@ -0,0 +1,11 @@ +Describe this image factually for a knowledge-base index. Surrounding text below tells you what topic the image relates to. + +--- Text before image --- +{before} + +--- Text after image --- +{after} + +--- End surrounding text --- + +If the image is clearly NOT relevant to the surrounding text, prepend "[OFF-TOPIC] " to your response. Otherwise, describe factually: visible text verbatim, chart axes and values, diagram structure, key visual elements. 2 to 4 sentences. Plain text only — no markdown, no preamble. diff --git a/mateclaw-server/src/main/resources/prompts/wiki/vision-caption-factual.txt b/mateclaw-server/src/main/resources/prompts/wiki/vision-caption-factual.txt new file mode 100644 index 00000000..60b21d79 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/vision-caption-factual.txt @@ -0,0 +1 @@ +Describe this image factually for a knowledge-base index. Include: any visible text verbatim, chart axes and values, diagram structure (boxes/arrows/labels), key visual elements. Do NOT speculate or editorialize. 2 to 4 sentences. Output plain text only — no markdown, no preamble.