From ac654f3957accb437bb6e527eac4421fc024f4d7 Mon Sep 17 00:00:00 2001 From: matevip Date: Sat, 2 May 2026 21:46:15 +0800 Subject: [PATCH] feat(wiki): add Zhipu GLM-V and Volcano Doubao vision providers --- .../provider/DashScopeVisionProvider.java | 197 ++-------------- .../vision/provider/DoubaoVisionProvider.java | 59 +++++ .../OpenAiCompatibleVisionProvider.java | 221 ++++++++++++++++++ .../vision/provider/ZhipuVisionProvider.java | 59 +++++ 4 files changed, 356 insertions(+), 180 deletions(-) create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DoubaoVisionProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/OpenAiCompatibleVisionProvider.java create mode 100644 mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/ZhipuVisionProvider.java diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java index fbf6f805..3650918d 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java @@ -1,62 +1,30 @@ package vip.mate.tool.image.vision.provider; -import cn.hutool.http.HttpRequest; -import cn.hutool.http.HttpResponse; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import vip.mate.agent.prompt.PromptLoader; -import vip.mate.exception.MateClawException; -import vip.mate.llm.model.ModelProviderEntity; import vip.mate.llm.service.ModelProviderService; -import vip.mate.system.model.SystemSettingsDTO; -import vip.mate.tool.image.ImageCapability; -import vip.mate.tool.image.vision.ImageVisionProvider; -import vip.mate.tool.image.vision.VisionContext; -import vip.mate.tool.image.vision.VisionRequest; -import vip.mate.tool.image.vision.VisionResult; - -import java.time.Duration; -import java.time.Instant; -import java.util.Base64; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; /** - * DashScope vision provider — uses qwen-vl-max via the OpenAI-compatible - * endpoint at {@code /compatible-mode/v1/chat/completions}. + * DashScope vision provider — uses {@code qwen-vl-max} via the + * OpenAI-compatible endpoint at {@code /compatible-mode/v1/chat/completions}. * - *

Default for the regional production rollout: API keys are typically - * available (DASHSCOPE_API_KEY is mandatory for the rest of the platform) - * and per-image cost is the lowest of the supported vendors. - * - *

Implementation deliberately uses Hutool {@code HttpRequest} rather - * than building a Spring AI {@code ChatClient} — the call shape is a - * one-shot request/response that doesn't benefit from the heavyweight - * agent-style observability / retry / cache machinery in - * {@code AgentOpenAiCompatibleChatModelBuilder}. - * - * @author MateClaw Team + *

Default for the Chinese cloud rollout: API keys are typically + * available (DASHSCOPE_API_KEY is mandatory for the rest of the + * platform) and per-image cost is the lowest of the supported vendors, + * so this provider sits at the front of the auto-detect chain. */ -@Slf4j @Component -@RequiredArgsConstructor -public class DashScopeVisionProvider implements ImageVisionProvider { +public class DashScopeVisionProvider extends OpenAiCompatibleVisionProvider { private static final String PROVIDER_ID = "dashscope-vision"; private static final String DASHSCOPE_PROVIDER_KEY = "dashscope"; private static final String DEFAULT_MODEL = "qwen-vl-max"; private static final String DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; - private static final int CALL_TIMEOUT_MS = 60_000; - private static final Pattern OFF_TOPIC_PREFIX = Pattern.compile("^\\s*\\[OFF-TOPIC\\]\\s*", Pattern.CASE_INSENSITIVE); - private final ModelProviderService modelProviderService; - private final ObjectMapper objectMapper; + public DashScopeVisionProvider(ModelProviderService modelProviderService, + ObjectMapper objectMapper) { + super(modelProviderService, objectMapper); + } @Override public String id() { @@ -68,154 +36,23 @@ public class DashScopeVisionProvider implements ImageVisionProvider { return "DashScope qwen-vl"; } - @Override - public boolean requiresCredential() { - return true; - } - @Override public int autoDetectOrder() { return 10; } @Override - public Set capabilities() { - return Set.of(ImageCapability.IMAGE_TO_TEXT); + protected String providerKey() { + return DASHSCOPE_PROVIDER_KEY; } @Override - public boolean isAvailable(SystemSettingsDTO settings) { - try { - return modelProviderService.isProviderConfigured(DASHSCOPE_PROVIDER_KEY); - } catch (Exception e) { - log.debug("[Vision][dashscope] availability check failed: {}", e.getMessage()); - return false; - } + protected String defaultBaseUrl() { + return DEFAULT_BASE_URL; } @Override - public VisionResult caption(VisionRequest request, SystemSettingsDTO settings) { - ModelProviderEntity provider; - try { - provider = modelProviderService.getProviderConfig(DASHSCOPE_PROVIDER_KEY); - } catch (Exception e) { - throw new MateClawException("err.wiki.vision.no_provider", - "DashScope provider is not configured"); - } - String apiKey = provider.getApiKey(); - if (apiKey == null || apiKey.isBlank()) { - throw new MateClawException("err.wiki.vision.no_provider", - "DashScope API key is missing"); - } - String baseUrl = (provider.getBaseUrl() == null || provider.getBaseUrl().isBlank()) - ? DEFAULT_BASE_URL - : provider.getBaseUrl(); - String model = (request.getPreferModel() == null || request.getPreferModel().isBlank()) - ? DEFAULT_MODEL - : request.getPreferModel(); - - String prompt = buildPrompt(request.getContext()); - String imageDataUrl = buildDataUrl(request.getMimeType(), request.getImageBytes()); - - ObjectNode body = objectMapper.createObjectNode(); - body.put("model", model); - body.put("temperature", 0.0); - body.put("max_tokens", 4096); - - ArrayNode messages = body.putArray("messages"); - ObjectNode userMsg = messages.addObject(); - userMsg.put("role", "user"); - ArrayNode content = userMsg.putArray("content"); - content.addObject().put("type", "text").put("text", prompt); - ObjectNode imagePart = content.addObject(); - imagePart.put("type", "image_url"); - imagePart.putObject("image_url").put("url", imageDataUrl); - - long startNanos = System.nanoTime(); - HttpResponse response; - try { - response = HttpRequest.post(baseUrl + "/chat/completions") - .header("Authorization", "Bearer " + apiKey) - .header("Content-Type", "application/json") - .body(body.toString()) - .timeout(CALL_TIMEOUT_MS) - .execute(); - } catch (Exception e) { - throw new MateClawException("err.wiki.vision.provider_failed", - "DashScope vision call failed: " + e.getMessage()); - } - long durationMs = Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); - - if (response.getStatus() != 200) { - String errBody = response.body(); - log.warn("[Vision][dashscope] HTTP {} body={}", response.getStatus(), truncate(errBody, 400)); - throw new MateClawException("err.wiki.vision.provider_failed", - "DashScope vision returned HTTP " + response.getStatus()); - } - - String text; - try { - JsonNode root = objectMapper.readTree(response.body()); - text = root.path("choices").path(0).path("message").path("content").asText("").trim(); - } catch (Exception e) { - throw new MateClawException("err.wiki.vision.provider_failed", - "DashScope vision response was unparseable: " + e.getMessage()); - } - if (text.isEmpty()) { - throw new MateClawException("err.wiki.vision.provider_failed", - "DashScope vision returned an empty caption"); - } - - Matcher offTopic = OFF_TOPIC_PREFIX.matcher(text); - boolean isOffTopic = offTopic.find(); - String caption = isOffTopic ? offTopic.replaceFirst("").trim() : text; - - return VisionResult.builder() - .caption(caption) - .visibleText(extractVisibleTextHeuristic(caption)) - .offTopic(isOffTopic) - .providerId(PROVIDER_ID) - .model(model) - .capturedAt(Instant.now()) - .durationMs(durationMs) - .build(); - } - - private String buildPrompt(VisionContext context) { - if (context != null && context.hasContext()) { - String template = PromptLoader.loadPrompt("wiki/vision-caption-context-aware"); - String before = (context.getBeforeText() == null || context.getBeforeText().isBlank()) - ? "(none)" : context.getBeforeText().trim(); - String after = (context.getAfterText() == null || context.getAfterText().isBlank()) - ? "(none)" : context.getAfterText().trim(); - return template.replace("{before}", before).replace("{after}", after); - } - return PromptLoader.loadPrompt("wiki/vision-caption-factual"); - } - - private static String buildDataUrl(String mimeType, byte[] bytes) { - String mime = (mimeType == null || mimeType.isBlank()) ? "image/png" : mimeType; - return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(bytes); - } - - /** - * Heuristic: pull obvious quoted strings as "visible text". This is a - * placeholder until a stronger OCR-grade extraction lands; for the - * factual prompt the model already inlines text verbatim into the - * caption, so the cache row's caption alone is enough for retrieval. - */ - private static String extractVisibleTextHeuristic(String caption) { - Matcher m = Pattern.compile("\"([^\"]{3,})\"").matcher(caption); - StringBuilder sb = new StringBuilder(); - while (m.find()) { - if (sb.length() > 0) sb.append('\n'); - sb.append(m.group(1)); - } - return sb.length() == 0 ? null : sb.toString(); - } - - private static String truncate(String s, int max) { - if (s == null) return ""; - return s.length() > max ? s.substring(0, max) + "…" : s; + protected String defaultModel() { + return DEFAULT_MODEL; } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DoubaoVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DoubaoVisionProvider.java new file mode 100644 index 00000000..85ca7cd9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DoubaoVisionProvider.java @@ -0,0 +1,59 @@ +package vip.mate.tool.image.vision.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; + +/** + * Volcengine Ark Doubao vision provider — uses + * {@code doubao-seed-1-8-251228} (the seeded multimodal flagship) + * via the OpenAI-compatible endpoint at {@code /api/v3/chat/completions}. + * + *

Attaches to the {@code volcengine} model provider; the + * {@code volcengine-plan} sibling (coding subscription tier) is + * intentionally not consumed here because the multimodal model + * isn't part of that plan. + */ +@Component +public class DoubaoVisionProvider extends OpenAiCompatibleVisionProvider { + + private static final String PROVIDER_ID = "doubao-vision"; + private static final String VOLCENGINE_PROVIDER_KEY = "volcengine"; + private static final String DEFAULT_MODEL = "doubao-seed-1-8-251228"; + private static final String DEFAULT_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; + + public DoubaoVisionProvider(ModelProviderService modelProviderService, + ObjectMapper objectMapper) { + super(modelProviderService, objectMapper); + } + + @Override + public String id() { + return PROVIDER_ID; + } + + @Override + public String label() { + return "Volcano Doubao Vision"; + } + + @Override + public int autoDetectOrder() { + return 30; + } + + @Override + protected String providerKey() { + return VOLCENGINE_PROVIDER_KEY; + } + + @Override + protected String defaultBaseUrl() { + return DEFAULT_BASE_URL; + } + + @Override + protected String defaultModel() { + return DEFAULT_MODEL; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/OpenAiCompatibleVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/OpenAiCompatibleVisionProvider.java new file mode 100644 index 00000000..2659f5f5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/OpenAiCompatibleVisionProvider.java @@ -0,0 +1,221 @@ +package vip.mate.tool.image.vision.provider; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import vip.mate.agent.prompt.PromptLoader; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.vision.ImageVisionProvider; +import vip.mate.tool.image.vision.VisionContext; +import vip.mate.tool.image.vision.VisionRequest; +import vip.mate.tool.image.vision.VisionResult; + +import java.time.Duration; +import java.time.Instant; +import java.util.Base64; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Shared base for vision providers that hit OpenAI-compatible + * {@code /chat/completions} endpoints with image-content blocks. + * + *

DashScope (qwen-vl), Zhipu (glm-v) and Volcengine Ark (doubao + * vision) all accept the same request shape — a {@code messages} array + * whose user message contains a {@code text} part and an + * {@code image_url} part with a base64 data URL. Extracting that + * shape into one place lets each concrete provider supply only its + * identity and routing constants. + * + *

Subclass contract: implement {@link #id()}, {@link #label()}, + * {@link #autoDetectOrder()}, {@link #providerKey()}, + * {@link #defaultBaseUrl()}, and {@link #defaultModel()}. Override + * {@link #completionsPath()} when the endpoint differs from the + * default {@code /chat/completions}. + */ +@Slf4j +public abstract class OpenAiCompatibleVisionProvider implements ImageVisionProvider { + + protected static final int CALL_TIMEOUT_MS = 60_000; + private static final Pattern OFF_TOPIC_PREFIX = + Pattern.compile("^\\s*\\[OFF-TOPIC\\]\\s*", Pattern.CASE_INSENSITIVE); + + protected final ModelProviderService modelProviderService; + protected final ObjectMapper objectMapper; + + protected OpenAiCompatibleVisionProvider(ModelProviderService modelProviderService, + ObjectMapper objectMapper) { + this.modelProviderService = modelProviderService; + this.objectMapper = objectMapper; + } + + /** {@code mate_model_provider.provider_id} this vision provider attaches to. */ + protected abstract String providerKey(); + + protected abstract String defaultBaseUrl(); + + protected abstract String defaultModel(); + + /** Override when the endpoint isn't the OpenAI-standard suffix. */ + protected String completionsPath() { + return "/chat/completions"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public Set capabilities() { + return Set.of(ImageCapability.IMAGE_TO_TEXT); + } + + @Override + public boolean isAvailable(SystemSettingsDTO settings) { + try { + return modelProviderService.isProviderConfigured(providerKey()); + } catch (Exception e) { + log.debug("[Vision][{}] availability check failed: {}", id(), e.getMessage()); + return false; + } + } + + @Override + public VisionResult caption(VisionRequest request, SystemSettingsDTO settings) { + ModelProviderEntity provider = loadProvider(); + String apiKey = provider.getApiKey(); + if (apiKey == null || apiKey.isBlank()) { + throw new MateClawException("err.wiki.vision.no_provider", + label() + " API key is missing"); + } + String baseUrl = (provider.getBaseUrl() == null || provider.getBaseUrl().isBlank()) + ? defaultBaseUrl() + : provider.getBaseUrl(); + String model = (request.getPreferModel() == null || request.getPreferModel().isBlank()) + ? defaultModel() + : request.getPreferModel(); + + String prompt = buildPrompt(request.getContext()); + String imageDataUrl = buildDataUrl(request.getMimeType(), request.getImageBytes()); + + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", model); + body.put("temperature", 0.0); + body.put("max_tokens", 4096); + + ArrayNode messages = body.putArray("messages"); + ObjectNode userMsg = messages.addObject(); + userMsg.put("role", "user"); + ArrayNode content = userMsg.putArray("content"); + content.addObject().put("type", "text").put("text", prompt); + ObjectNode imagePart = content.addObject(); + imagePart.put("type", "image_url"); + imagePart.putObject("image_url").put("url", imageDataUrl); + + long startNanos = System.nanoTime(); + HttpResponse response; + try { + response = HttpRequest.post(baseUrl + completionsPath()) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(CALL_TIMEOUT_MS) + .execute(); + } catch (Exception e) { + throw new MateClawException("err.wiki.vision.provider_failed", + label() + " vision call failed: " + e.getMessage()); + } + long durationMs = Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); + + if (response.getStatus() != 200) { + String errBody = response.body(); + log.warn("[Vision][{}] HTTP {} body={}", id(), response.getStatus(), truncate(errBody, 400)); + throw new MateClawException("err.wiki.vision.provider_failed", + label() + " vision returned HTTP " + response.getStatus()); + } + + String text; + try { + JsonNode root = objectMapper.readTree(response.body()); + text = root.path("choices").path(0).path("message").path("content").asText("").trim(); + } catch (Exception e) { + throw new MateClawException("err.wiki.vision.provider_failed", + label() + " vision response was unparseable: " + e.getMessage()); + } + if (text.isEmpty()) { + throw new MateClawException("err.wiki.vision.provider_failed", + label() + " vision returned an empty caption"); + } + + Matcher offTopic = OFF_TOPIC_PREFIX.matcher(text); + boolean isOffTopic = offTopic.find(); + String caption = isOffTopic ? offTopic.replaceFirst("").trim() : text; + + return VisionResult.builder() + .caption(caption) + .visibleText(extractVisibleTextHeuristic(caption)) + .offTopic(isOffTopic) + .providerId(id()) + .model(model) + .capturedAt(Instant.now()) + .durationMs(durationMs) + .build(); + } + + private ModelProviderEntity loadProvider() { + try { + return modelProviderService.getProviderConfig(providerKey()); + } catch (Exception e) { + throw new MateClawException("err.wiki.vision.no_provider", + label() + " provider is not configured"); + } + } + + private String buildPrompt(VisionContext context) { + if (context != null && context.hasContext()) { + String template = PromptLoader.loadPrompt("wiki/vision-caption-context-aware"); + String before = (context.getBeforeText() == null || context.getBeforeText().isBlank()) + ? "(none)" : context.getBeforeText().trim(); + String after = (context.getAfterText() == null || context.getAfterText().isBlank()) + ? "(none)" : context.getAfterText().trim(); + return template.replace("{before}", before).replace("{after}", after); + } + return PromptLoader.loadPrompt("wiki/vision-caption-factual"); + } + + private static String buildDataUrl(String mimeType, byte[] bytes) { + String mime = (mimeType == null || mimeType.isBlank()) ? "image/png" : mimeType; + return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(bytes); + } + + /** + * Heuristic: pull obvious quoted strings as "visible text". This is a + * placeholder until a stronger OCR-grade extraction lands; for the + * factual prompt the model already inlines text verbatim into the + * caption, so the cache row's caption alone is enough for retrieval. + */ + private static String extractVisibleTextHeuristic(String caption) { + Matcher m = Pattern.compile("\"([^\"]{3,})\"").matcher(caption); + StringBuilder sb = new StringBuilder(); + while (m.find()) { + if (sb.length() > 0) sb.append('\n'); + sb.append(m.group(1)); + } + return sb.length() == 0 ? null : sb.toString(); + } + + private static String truncate(String s, int max) { + if (s == null) return ""; + return s.length() > max ? s.substring(0, max) + "…" : s; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/ZhipuVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/ZhipuVisionProvider.java new file mode 100644 index 00000000..ef284456 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/ZhipuVisionProvider.java @@ -0,0 +1,59 @@ +package vip.mate.tool.image.vision.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelProviderService; + +/** + * Zhipu vision provider — uses {@code glm-5v-turbo} via the + * OpenAI-compatible endpoint at {@code /api/paas/v4/chat/completions}. + * + *

Attaches to the {@code zhipu-cn} model provider. Operators with + * the international tier should configure the {@code zhipu-cn} entry + * with the international base URL or wait for an explicit intl + * sibling provider — the seeded {@code zhipu-intl} differs only in + * base URL. + */ +@Component +public class ZhipuVisionProvider extends OpenAiCompatibleVisionProvider { + + private static final String PROVIDER_ID = "zhipu-vision"; + private static final String ZHIPU_PROVIDER_KEY = "zhipu-cn"; + private static final String DEFAULT_MODEL = "glm-5v-turbo"; + private static final String DEFAULT_BASE_URL = "https://open.bigmodel.cn/api/paas/v4"; + + public ZhipuVisionProvider(ModelProviderService modelProviderService, + ObjectMapper objectMapper) { + super(modelProviderService, objectMapper); + } + + @Override + public String id() { + return PROVIDER_ID; + } + + @Override + public String label() { + return "Zhipu GLM-V"; + } + + @Override + public int autoDetectOrder() { + return 20; + } + + @Override + protected String providerKey() { + return ZHIPU_PROVIDER_KEY; + } + + @Override + protected String defaultBaseUrl() { + return DEFAULT_BASE_URL; + } + + @Override + protected String defaultModel() { + return DEFAULT_MODEL; + } +}